def getClusterJobData(self):
if self.clusterjobdata is not None:
return self.clusterjobdata
if not 'commit' in self.params or self.params['commit'] is False:
return None
pathq = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
clustq = appiondata.ApAppionJobData()
clustq['path'] = pathq
clustq['jobtype'] = self.functionname.lower()
clustdatas = clustq.query()
if not clustdatas:
### insert a cluster job
clustq['name'] = self.params['runname'] + ".appionsub.job"
clustq['clusterpath'] = pathq
clustq['user'] = apParam.getUsername()
clustq['cluster'] = apParam.getHostname()
clustq['status'] = "R"
clustq['session'] = self.getSessionData()
### need a proper way to create a jobtype
clustq['jobtype'] = self.params['jobtype']
if not clustq['jobtype']:
clustq['jobtype'] = self.functionname.lower()
clustq.insert()
self.clusterjobdata = clustq
return clustq
elif len(clustdatas) == 1:
### we have an entry
### we need to say that we are running
apWebScript.setJobToRun(clustdatas[0].dbid)
self.clusterjobdata = clustdatas[0]
return clustdatas[0]
else:
### special case: more than one job with given path
apDisplay.printWarning("More than one cluster job has this path")
self.clusterjobdata = clustdatas[0]
return clustdatas[0]
def getClusterJobData(self):
if self.clusterjobdata is not None:
return self.clusterjobdata
if not 'commit' in self.params or self.params['commit'] is False:
return None
pathq = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
clustq = appiondata.ApAppionJobData()
clustq['path'] = pathq
clustq['jobtype'] = self.functionname.lower()
clustdatas = clustq.query()
if not clustdatas:
### insert a cluster job
clustq['name'] = self.params['runname']+".appionsub.job"
clustq['clusterpath'] = pathq
clustq['user'] = apParam.getUsername()
clustq['cluster'] = apParam.getHostname()
clustq['status'] = "R"
clustq['session'] = self.getSessionData()
### need a proper way to create a jobtype
clustq['jobtype']=self.params['jobtype']
if not clustq['jobtype']:
clustq['jobtype'] = self.functionname.lower()
clustq.insert()
self.clusterjobdata = clustq
return clustq
elif len(clustdatas) == 1:
### we have an entry
### we need to say that we are running
apWebScript.setJobToRun(clustdatas[0].dbid)
self.clusterjobdata = clustdatas[0]
return clustdatas[0]
else:
### special case: more than one job with given path
apDisplay.printWarning("More than one cluster job has this path")
self.clusterjobdata = clustdatas[0]
return clustdatas[0]
def uploadScriptData(self):
"""
Using tables to track program run parameters in a generic fashion
inspired by Roberto Marabini and Carlos Oscar Sanchez Sorzano from the Xmipp team/Carazo lab
"""
apDisplay.printMsg("Uploading ScriptData....")
prognameq = appiondata.ScriptProgramName()
prognameq['name'] = self.functionname
userq = appiondata.ScriptUserName()
userdict = apParam.getUserDict()
if userdict:
userq['name'] = userdict['username']
userq['uid'] = userdict['uid']
userq['gid'] = userdict['gid']
userq['fullname'] = userdict['fullname']
unixshell = userdict['unixshell']
else:
userq['name'] = "unknown"
unixshell = None
hostq = appiondata.ScriptHostName()
hostq['name'] = apParam.getHostname()
hostq['ip'] = apParam.getHostIP()
hostq['system'] = apParam.getSystemName()
hostq['distro'] = apParam.getLinuxDistro()
hostq['nproc'] = apParam.getNumProcessors()
hostq['memory'] = apParam.getTotalMemory()
hostq['cpu_vendor'] = apParam.getCPUVendor()
hostq['gpu_vendor'] = apParam.getGPUVendor()
hostq['arch'] = apParam.getMachineArch()
progrunq = appiondata.ScriptProgramRun()
progrunq['runname'] = self.params['runname']
progrunq['progname'] = prognameq
progrunq['username'] = userq
progrunq['hostname'] = hostq
progrunq['unixshell'] = unixshell
progrunq['rundir'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
progrunq['job'] = self.getClusterJobData()
appiondir = apParam.getAppionDirectory()
### get appion version/subversion revision
progrunq['revision'] = None
versionfile = os.path.join(appiondir, "appionlib/version.txt")
if os.path.isfile(versionfile):
f = open(versionfile, 'r')
line = f.readline()
f.close()
sline = line.strip()
progrunq['revision'] = sline
if os.path.isdir(os.path.join(appiondir, ".svn")):
if progrunq['revision'] is None:
progrunq['revision'] = version.getSubversionRevision(appiondir)
else:
progrunq['revision'] += "-" + version.getSubversionRevision(
appiondir)
if not progrunq['revision']:
progrunq['revision'] = 'unknown'
apDisplay.printMsg("Running Appion version '%s'" %
(progrunq['revision']))
progrunq['appion_path'] = appiondata.ApPathData(
path=os.path.abspath(appiondir))
for paramname in self.params.keys():
paramnameq = appiondata.ScriptParamName()
paramnameq['name'] = paramname
paramnameq['progname'] = prognameq
paramvalueq = appiondata.ScriptParamValue()
paramvalueq['value'] = str(self.params[paramname])
usage = self.usageFromParamDest(paramname, self.params[paramname])
#print "usage: ", usage
paramvalueq['usage'] = usage
paramvalueq['paramname'] = paramnameq
paramvalueq['progrun'] = progrunq
if usage is not None:
paramvalueq.insert()
def __init__(self,optargs=sys.argv[1:],quiet=False,useglobalparams=True,maxnproc=None):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.successful_run = False
self.clusterjobdata = None
self.params = {}
sys.stdout.write("\n\n")
self.quiet = quiet
self.maxnproc = maxnproc
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.timestamp = apParam.makeTimestamp()
self.argdict = {}
self.optdict = {}
apDisplay.printMsg("Time stamp: "+self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
apDisplay.printMsg("Function name: "+self.functionname)
self.appiondir = apParam.getAppionDirectory()
apDisplay.printMsg("Appion directory: "+self.appiondir)
hostname = apParam.getHostname()
apDisplay.printMsg("Processing hostname: "+hostname)
self.parsePythonPath()
# loadavg = os.getloadavg()[0]
# if loadavg > 2.0:
# apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
# loadsquared = loadavg*loadavg
# time.sleep(loadavg)
# apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2)))
self.setLockname('lock')
### setup default parser: run directory, etc.
self.setParams(optargs,useglobalparams)
#if 'outdir' in self.params and self.params['outdir'] is not None:
# self.params['rundir'] = self.params['outdir']
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
### check if user wants to print help message
if 'commit' in self.params and self.params['commit'] is True:
apDisplay.printMsg("Committing data to database")
else:
apDisplay.printWarning("Not committing data to database")
self.checkConflicts()
if useglobalparams is True:
self.checkGlobalConflicts()
### setup run directory
self.setProcessingDirName()
self.setupRunDirectory()
### Start pool of threads to run subprocesses.
### Later you will use self.process_launcher.launch(...) to
### put commands into the queue.
### There is currently a timeout built into it that will cause
### the threads to die if they have no tasks after 10 seconds.
self.process_launcher = apThread.ProcessLauncher(2, self.params['rundir'])
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### upload command line parameters to database
self.uploadScriptData()
### any custom init functions go here
self.onInit()
def uploadScriptData(self):
"""
Using tables to track program run parameters in a generic fashion
inspired by Roberto Marabini and Carlos Oscar Sanchez Sorzano from the Xmipp team/Carazo lab
"""
apDisplay.printMsg("Uploading ScriptData....")
prognameq = appiondata.ScriptProgramName()
prognameq['name'] = self.functionname
userq = appiondata.ScriptUserName()
userdict = apParam.getUserDict()
if userdict:
userq['name'] = userdict['username']
userq['uid'] = userdict['uid']
userq['gid'] = userdict['gid']
userq['fullname'] = userdict['fullname']
unixshell = userdict['unixshell']
else:
userq['name'] = "unknown"
unixshell = None
hostq = appiondata.ScriptHostName()
hostq['name'] = apParam.getHostname()
hostq['ip'] = apParam.getHostIP()
hostq['system'] = apParam.getSystemName()
hostq['distro'] = apParam.getLinuxDistro()
hostq['nproc'] = apParam.getNumProcessors()
hostq['memory'] = apParam.getTotalMemory()
hostq['cpu_vendor'] = apParam.getCPUVendor()
hostq['gpu_vendor'] = apParam.getGPUVendor()
hostq['arch'] = apParam.getMachineArch()
progrunq = appiondata.ScriptProgramRun()
progrunq['runname'] = self.params['runname']
progrunq['progname'] = prognameq
progrunq['username'] = userq
progrunq['hostname'] = hostq
progrunq['unixshell'] = unixshell
progrunq['rundir'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
progrunq['job'] = self.getClusterJobData()
appiondir = apParam.getAppionDirectory()
### get appion version/subversion revision
progrunq['revision'] = None
versionfile = os.path.join(appiondir, "appionlib/version.txt")
if os.path.isfile(versionfile):
f = open(versionfile, 'r')
line = f.readline()
f.close()
sline = line.strip()
progrunq['revision'] = sline
if os.path.isdir(os.path.join(appiondir, ".svn")):
if progrunq['revision'] is None:
progrunq['revision'] = version.getSubversionRevision(appiondir)
else:
progrunq['revision'] += "-"+version.getSubversionRevision(appiondir)
if not progrunq['revision']:
progrunq['revision'] = 'unknown'
apDisplay.printMsg("Running Appion version '%s'"%(progrunq['revision']))
progrunq['appion_path'] = appiondata.ApPathData(path=os.path.abspath(appiondir))
for paramname in self.params.keys():
paramnameq = appiondata.ScriptParamName()
paramnameq['name'] = paramname
paramnameq['progname'] = prognameq
paramvalueq = appiondata.ScriptParamValue()
paramvalueq['value'] = str(self.params[paramname])
usage = self.usageFromParamDest(paramname, self.params[paramname])
#print "usage: ", usage
paramvalueq['usage'] = usage
paramvalueq['paramname'] = paramnameq
paramvalueq['progrun'] = progrunq
if usage is not None:
paramvalueq.insert()
