def getClusterJobData(self): if self.clusterjobdata is not None: return self.clusterjobdata if not 'commit' in self.params or self.params['commit'] is False: return None pathq = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) clustq = appiondata.ApAppionJobData() clustq['path'] = pathq clustq['jobtype'] = self.functionname.lower() clustdatas = clustq.query() if not clustdatas: ### insert a cluster job clustq['name'] = self.params['runname'] + ".appionsub.job" clustq['clusterpath'] = pathq clustq['user'] = apParam.getUsername() clustq['cluster'] = apParam.getHostname() clustq['status'] = "R" clustq['session'] = self.getSessionData() ### need a proper way to create a jobtype clustq['jobtype'] = self.params['jobtype'] if not clustq['jobtype']: clustq['jobtype'] = self.functionname.lower() clustq.insert() self.clusterjobdata = clustq return clustq elif len(clustdatas) == 1: ### we have an entry ### we need to say that we are running apWebScript.setJobToRun(clustdatas[0].dbid) self.clusterjobdata = clustdatas[0] return clustdatas[0] else: ### special case: more than one job with given path apDisplay.printWarning("More than one cluster job has this path") self.clusterjobdata = clustdatas[0] return clustdatas[0]
def getClusterJobData(self): if self.clusterjobdata is not None: return self.clusterjobdata if not 'commit' in self.params or self.params['commit'] is False: return None pathq = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) clustq = appiondata.ApAppionJobData() clustq['path'] = pathq clustq['jobtype'] = self.functionname.lower() clustdatas = clustq.query() if not clustdatas: ### insert a cluster job clustq['name'] = self.params['runname']+".appionsub.job" clustq['clusterpath'] = pathq clustq['user'] = apParam.getUsername() clustq['cluster'] = apParam.getHostname() clustq['status'] = "R" clustq['session'] = self.getSessionData() ### need a proper way to create a jobtype clustq['jobtype']=self.params['jobtype'] if not clustq['jobtype']: clustq['jobtype'] = self.functionname.lower() clustq.insert() self.clusterjobdata = clustq return clustq elif len(clustdatas) == 1: ### we have an entry ### we need to say that we are running apWebScript.setJobToRun(clustdatas[0].dbid) self.clusterjobdata = clustdatas[0] return clustdatas[0] else: ### special case: more than one job with given path apDisplay.printWarning("More than one cluster job has this path") self.clusterjobdata = clustdatas[0] return clustdatas[0]
def uploadScriptData(self): """ Using tables to track program run parameters in a generic fashion inspired by Roberto Marabini and Carlos Oscar Sanchez Sorzano from the Xmipp team/Carazo lab """ apDisplay.printMsg("Uploading ScriptData....") prognameq = appiondata.ScriptProgramName() prognameq['name'] = self.functionname userq = appiondata.ScriptUserName() userdict = apParam.getUserDict() if userdict: userq['name'] = userdict['username'] userq['uid'] = userdict['uid'] userq['gid'] = userdict['gid'] userq['fullname'] = userdict['fullname'] unixshell = userdict['unixshell'] else: userq['name'] = "unknown" unixshell = None hostq = appiondata.ScriptHostName() hostq['name'] = apParam.getHostname() hostq['ip'] = apParam.getHostIP() hostq['system'] = apParam.getSystemName() hostq['distro'] = apParam.getLinuxDistro() hostq['nproc'] = apParam.getNumProcessors() hostq['memory'] = apParam.getTotalMemory() hostq['cpu_vendor'] = apParam.getCPUVendor() hostq['gpu_vendor'] = apParam.getGPUVendor() hostq['arch'] = apParam.getMachineArch() progrunq = appiondata.ScriptProgramRun() progrunq['runname'] = self.params['runname'] progrunq['progname'] = prognameq progrunq['username'] = userq progrunq['hostname'] = hostq progrunq['unixshell'] = unixshell progrunq['rundir'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) progrunq['job'] = self.getClusterJobData() appiondir = apParam.getAppionDirectory() ### get appion version/subversion revision progrunq['revision'] = None versionfile = os.path.join(appiondir, "appionlib/version.txt") if os.path.isfile(versionfile): f = open(versionfile, 'r') line = f.readline() f.close() sline = line.strip() progrunq['revision'] = sline if os.path.isdir(os.path.join(appiondir, ".svn")): if progrunq['revision'] is None: progrunq['revision'] = version.getSubversionRevision(appiondir) else: progrunq['revision'] += "-" + version.getSubversionRevision( appiondir) if not progrunq['revision']: progrunq['revision'] = 'unknown' apDisplay.printMsg("Running Appion version '%s'" % (progrunq['revision'])) progrunq['appion_path'] = appiondata.ApPathData( path=os.path.abspath(appiondir)) for paramname in self.params.keys(): paramnameq = appiondata.ScriptParamName() paramnameq['name'] = paramname paramnameq['progname'] = prognameq paramvalueq = appiondata.ScriptParamValue() paramvalueq['value'] = str(self.params[paramname]) usage = self.usageFromParamDest(paramname, self.params[paramname]) #print "usage: ", usage paramvalueq['usage'] = usage paramvalueq['paramname'] = paramnameq paramvalueq['progrun'] = progrunq if usage is not None: paramvalueq.insert()
def __init__(self,optargs=sys.argv[1:],quiet=False,useglobalparams=True,maxnproc=None): """ Starts a new function and gets all the parameters """ ### setup some expected values self.successful_run = False self.clusterjobdata = None self.params = {} sys.stdout.write("\n\n") self.quiet = quiet self.maxnproc = maxnproc self.startmem = mem.active() self.t0 = time.time() self.createDefaultStats() self.timestamp = apParam.makeTimestamp() self.argdict = {} self.optdict = {} apDisplay.printMsg("Time stamp: "+self.timestamp) self.functionname = apParam.getFunctionName(sys.argv[0]) apDisplay.printMsg("Function name: "+self.functionname) self.appiondir = apParam.getAppionDirectory() apDisplay.printMsg("Appion directory: "+self.appiondir) hostname = apParam.getHostname() apDisplay.printMsg("Processing hostname: "+hostname) self.parsePythonPath() # loadavg = os.getloadavg()[0] # if loadavg > 2.0: # apDisplay.printMsg("Load average is high "+str(round(loadavg,2))) # loadsquared = loadavg*loadavg # time.sleep(loadavg) # apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2))) self.setLockname('lock') ### setup default parser: run directory, etc. self.setParams(optargs,useglobalparams) #if 'outdir' in self.params and self.params['outdir'] is not None: # self.params['rundir'] = self.params['outdir'] ### setup correct database after we have read the project id if 'projectid' in self.params and self.params['projectid'] is not None: apDisplay.printMsg("Using split database") # use a project database newdbname = apProject.getAppionDBFromProjectId(self.params['projectid']) sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '"+newdbname+"'", "green") ### check if user wants to print help message if 'commit' in self.params and self.params['commit'] is True: apDisplay.printMsg("Committing data to database") else: apDisplay.printWarning("Not committing data to database") self.checkConflicts() if useglobalparams is True: self.checkGlobalConflicts() ### setup run directory self.setProcessingDirName() self.setupRunDirectory() ### Start pool of threads to run subprocesses. ### Later you will use self.process_launcher.launch(...) to ### put commands into the queue. ### There is currently a timeout built into it that will cause ### the threads to die if they have no tasks after 10 seconds. self.process_launcher = apThread.ProcessLauncher(2, self.params['rundir']) ### write function log self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet)) ### upload command line parameters to database self.uploadScriptData() ### any custom init functions go here self.onInit()
def uploadScriptData(self): """ Using tables to track program run parameters in a generic fashion inspired by Roberto Marabini and Carlos Oscar Sanchez Sorzano from the Xmipp team/Carazo lab """ apDisplay.printMsg("Uploading ScriptData....") prognameq = appiondata.ScriptProgramName() prognameq['name'] = self.functionname userq = appiondata.ScriptUserName() userdict = apParam.getUserDict() if userdict: userq['name'] = userdict['username'] userq['uid'] = userdict['uid'] userq['gid'] = userdict['gid'] userq['fullname'] = userdict['fullname'] unixshell = userdict['unixshell'] else: userq['name'] = "unknown" unixshell = None hostq = appiondata.ScriptHostName() hostq['name'] = apParam.getHostname() hostq['ip'] = apParam.getHostIP() hostq['system'] = apParam.getSystemName() hostq['distro'] = apParam.getLinuxDistro() hostq['nproc'] = apParam.getNumProcessors() hostq['memory'] = apParam.getTotalMemory() hostq['cpu_vendor'] = apParam.getCPUVendor() hostq['gpu_vendor'] = apParam.getGPUVendor() hostq['arch'] = apParam.getMachineArch() progrunq = appiondata.ScriptProgramRun() progrunq['runname'] = self.params['runname'] progrunq['progname'] = prognameq progrunq['username'] = userq progrunq['hostname'] = hostq progrunq['unixshell'] = unixshell progrunq['rundir'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) progrunq['job'] = self.getClusterJobData() appiondir = apParam.getAppionDirectory() ### get appion version/subversion revision progrunq['revision'] = None versionfile = os.path.join(appiondir, "appionlib/version.txt") if os.path.isfile(versionfile): f = open(versionfile, 'r') line = f.readline() f.close() sline = line.strip() progrunq['revision'] = sline if os.path.isdir(os.path.join(appiondir, ".svn")): if progrunq['revision'] is None: progrunq['revision'] = version.getSubversionRevision(appiondir) else: progrunq['revision'] += "-"+version.getSubversionRevision(appiondir) if not progrunq['revision']: progrunq['revision'] = 'unknown' apDisplay.printMsg("Running Appion version '%s'"%(progrunq['revision'])) progrunq['appion_path'] = appiondata.ApPathData(path=os.path.abspath(appiondir)) for paramname in self.params.keys(): paramnameq = appiondata.ScriptParamName() paramnameq['name'] = paramname paramnameq['progname'] = prognameq paramvalueq = appiondata.ScriptParamValue() paramvalueq['value'] = str(self.params[paramname]) usage = self.usageFromParamDest(paramname, self.params[paramname]) #print "usage: ", usage paramvalueq['usage'] = usage paramvalueq['paramname'] = paramnameq paramvalueq['progrun'] = progrunq if usage is not None: paramvalueq.insert()