def getJobDataFromPath(path):
pathq = appiondata.ApPathData()
pathq['path'] = os.path.abspath(path)
jobq = appiondata.ApAppionJobData()
jobq['path'] = pathq
jobdatas = jobq.query()
return jobdatas
def tryToGetJobID(self):
jobname = self.params['runname'] + '.job'
jobtype = 'recon'
jobpath = self.params['rundir']
qpath = appiondata.ApPathData(path=os.path.abspath(jobpath))
q = appiondata.ApAppionJobData(name=jobname,
jobtype=jobtype,
path=qpath)
results = q.query()
if len(results) == 1:
### success, only one job id found
return results[0].dbid
elif len(results) > 1:
### fail because too many job ids
jobids = [result.dbid for result in results]
apDisplay.printWarning(
"Several job IDs found for this run %s: You should manually specify a jobid, if it exists"
% jobids)
return False
else:
### no job found
apDisplay.printWarning(
"No job IDs found for this run, you should manually specify a jobid, if it exists"
)
return False
def getJobDataFromPathAndType(path, jobtype):
pathq = appiondata.ApPathData()
pathq['path'] = os.path.abspath(path)
jobq = appiondata.ApAppionJobData()
jobq['path'] = pathq
jobq['jobtype'] = jobtype
jobdatas = jobq.query(results=1)
if not jobdatas:
return None
return jobdatas[0]
def prepareForCluster(self):
#package data for transfer to cluster
apFile.removeFile(self.params['runname'] + ".tar")
cmd = "tar --exclude=*.tar -cf %s.tar *" % (self.params['runname'])
print cmd
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
needf = open("files-needed.txt", "w")
needf.write(
"%s.tar\n" %
(os.path.join(self.params['rundir'], self.params['runname'])))
needf.write("%s\n" %
(os.path.splitext(self.refinestackfile)[0] + ".hed"))
needf.write("%s\n" %
(os.path.splitext(self.refinestackfile)[0] + ".img"))
needf.close()
### find cluster job based on path
partq = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
jobq = appiondata.ApAppionJobData()
jobq['path'] = partq
jobq['jobtype'] = 'prepfrealign'
jobdatas = jobq.query(results=1)
if not jobdatas:
apDisplay.printError("Could not find job data for prepFrealign")
jobdata = jobdatas[0]
### create a frealign table
frealignq = appiondata.ApFrealignPrepareData()
frealignq['name'] = self.params['runname']
frealignq['ppn'] = self.params['ppn']
frealignq['rpn'] = self.params['rpn']
frealignq['nodes'] = self.params['nodes']
frealignq['memory'] = self.calcMemNeeded() / 1e9 #memory in gigabytes
frealignq['hidden'] = False
frealignq['tarfile'] = "%s.tar" % (self.params['runname'])
frealignq['path'] = partq
frealignq['stack'] = appiondata.ApStackData.direct_query(
self.params['stackid'])
if self.params['reconstackid'] is not None:
frealignq['reconstack'] = appiondata.ApStackData.direct_query(
self.params['reconstackid'])
frealignq['model'] = appiondata.ApInitialModelData.direct_query(
self.params['modelid'])
frealignq['job'] = jobdata
frealignq['symmetry'] = self.symmdata
if self.params['reconiterid'] is not None:
frealignq['refineIter'] = appiondata.ApRefineIterData.direct_query(
self.params['reconiterid'])
frealignq.insert()
def tryToGetJobID(self): jobname = self.params['runname'] + '.job' jobtype = 'recon' jobpath = self.params['rundir'] qpath = appiondata.ApPathData(path=os.path.abspath(jobpath)) q = appiondata.ApAppionJobData(name=jobname, jobtype=jobtype, path=qpath) results = q.query() if len(results) == 1: ## success, only one job id found self.params['jobid'] = results[0].dbid return '' elif len(results) > 1: ## fail because too many job ids jobids = [result.dbid for result in results] return 'Several Job IDs found for this run: %s\nYou will have to manually specify a jobid' % (jobids,) else: ## no job found self.params['jobid'] = None return ''
def getClusterJobData(self):
if self.clusterjobdata is not None:
return self.clusterjobdata
if not 'commit' in self.params or self.params['commit'] is False:
return None
pathq = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
clustq = appiondata.ApAppionJobData()
clustq['path'] = pathq
clustq['jobtype'] = self.functionname.lower()
clustdatas = clustq.query()
if not clustdatas:
### insert a cluster job
clustq['name'] = self.params['runname'] + ".appionsub.job"
clustq['clusterpath'] = pathq
clustq['user'] = apParam.getUsername()
clustq['cluster'] = apParam.getHostname()
clustq['status'] = "R"
clustq['session'] = self.getSessionData()
### need a proper way to create a jobtype
clustq['jobtype'] = self.params['jobtype']
if not clustq['jobtype']:
clustq['jobtype'] = self.functionname.lower()
clustq.insert()
self.clusterjobdata = clustq
return clustq
elif len(clustdatas) == 1:
### we have an entry
### we need to say that we are running
apWebScript.setJobToRun(clustdatas[0].dbid)
self.clusterjobdata = clustdatas[0]
return clustdatas[0]
else:
### special case: more than one job with given path
apDisplay.printWarning("More than one cluster job has this path")
self.clusterjobdata = clustdatas[0]
return clustdatas[0]
def getJobDataFromType(jobtype):
jobq = appiondata.ApAppionJobData()
jobq['jobtype'] = jobtype
jobdatas = jobq.query()
return jobdatas
#!/bin/python
'''
Find a recon job that died before finishing up. Perform the last lines of
the job file to move the contents of recon subdir up one level and run
updateAppionDB to change status to D.
Note: does not use project selection yet.
'''
import sys
from appionlib import appiondata
import subprocess
import os
if __name__ == "__main__":
rjobs = appiondata.ApAppionJobData(status='R', jobtype='recon')
rjobs = rjobs.query()
print 'Recon jobs listed as "Running" in the database:'
for i, job in enumerate(rjobs):
print ' %d: %s' % (i, job['name'])
print ''
response = raw_input(
'Choose a number from the list (or just hit enter to cancel): ')
try:
i = int(response)
myjob = rjobs[i]
except:
sys.exit()
print 'Checking if job is actually running on cluster...'
retValue = projDB
cursor.close()
dbConnection.close()
except MySQLdb.DatabaseError, e:
sys.stderr.write("Warning: Failure determining project database: %s \n" % (e))
#if jobId is not set, assume there is no entry in ApAppionJobData for this run
if not jobObject.getJobId():
### insert a cluster job
# TODO: what happens when this runs remotely???
rundir = jobObject.getRundir()
pathq = appiondata.ApPathData(path=os.path.abspath(rundir))
clustq = appiondata.ApAppionJobData()
clustq['path'] = pathq
clustq['jobtype'] = jobObject.getJobType()
clustq['name'] = jobObject.getJobName()
remoterundir = jobObject.getOutputDir()
remoterundirq = appiondata.ApPathData(path=os.path.abspath(remoterundir))
clustq['clusterpath'] = remoterundirq
clustq['session'] = apDatabase.getSessionDataFromSessionId(jobObject.getExpId())
clustq['user'] = os.getlogin()
clustq['cluster'] = os.uname()[1]
clustq['clusterjobid'] = job
clustq['status'] = "Q"
clustq.insert()
return retValue