def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(
self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '" + newdbname + "'",
"green")
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(
self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(
self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(
self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
### get stack data
self.stack = {}
if self.params['stackid'] is not None:
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
else:
self.stack['data'] = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
self.stack['apix'] = self.stack['pixelsize']
self.stack['boxsize'] = self.stack['boxsize']
self.stack['file'] = self.stack['imagicfile']
### check conflicts
if self.params['stackid'] is None and self.params['alignstackid'] is None:
apDisplay.printError("stack id OR alignstack id was not defined")
if self.params['stackid'] is not None and self.params['alignstackid'] is not None:
apDisplay.printError("either specify stack id OR alignstack id, not both")
if self.params['generations'] is None:
apDisplay.printError("number of generations was not provided")
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: "
+ str(maxparticles) + " requested: " + str(self.params['numpart']))
if self.params['numpart'] > apFile.numImagesInStack(self.stack['file']):
apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
+" than available "+str(apFile.numImagesInStack(self.stack['file'])))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(self.stack['file'])
boxsize = apStack.getStackBoxsize(self.params['stackid'])
if self.params['ou'] is None:
self.params['ou'] = (boxsize / 2.0) - 2
self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d"
%(self.params['clipsize'],self.clipsize))
self.clipsize = self.params['clipsize']
self.mpirun = self.checkMPI()
if self.mpirun is None:
apDisplay.printError("There is no MPI installed")
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(
self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '" + newdbname + "'",
"green")
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['expweight'] is False:
apDisplay.printWarning(
"Exposure weighting is turned off, make sure this is what you want"
)
if self.params['localavg'] is False:
apDisplay.printWarning(
"Trajectory local averaging is turned off, make sure this is what you want"
)
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(
self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(
self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(
self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# pixel size info
self.params['apix'] = apStack.getMicrographPixelSizeFromStackId(
self.params['stackid'])
self.params['box'] = self.stackdata['boxsize']
self.params['particleradius'] = self.params[
'particleradius'] / self.params['apix']
if self.params['particleradius'] > self.params['box'] / 2.0:
apDisplay.printWarning(
"specified particle radius greater than box radius, \
setting particle radius to 0.8 * boxsize")
# micrograph & frame info
frames = qimage['use frames']
nframes = len(frames)
if self.params['framelastali'] is None:
self.params['framelastali'] = frames[-1]
if self.params['framelastave'] is None:
self.params['framelastave'] = frames[-1]
# microscope kV
self.params['kv'] = qimage['scope']['high tension'] / 1000.0
# query exposure per frame, if not set here
if self.params['total_dose'] is not None:
dose = self.params['total_dose']
else:
try:
dose = apDatabase.getDoseFromImageData(qimage)
except:
apDisplay.printError(
"dose not specified and not in database, please specify explicitly"
)
if self.params['expperframe'] is None and self.params[
'expweight'] is True:
if dose is not None:
self.params['expperframe'] = dose / nframes
else:
apDisplay.printError(
"exposure per frame needs to be specified, cannot find in database"
)
# dimensions
self.params['framex'] = int(
apDatabase.getDimensionsFromImageData(qimage)['x'])
self.params['framey'] = int(
apDatabase.getDimensionsFromImageData(qimage)['y'])
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
#==================
#==================
#==================
if __name__ == '__main__':
print "Usage: apCoranPlot.py <reconid> <projectid>"
### setup correct database after we have read the project id
if len(sys.argv) > 2:
projectid = int(sys.argv[2])
else:
projectid = None
if projectid is not None:
apDisplay.printWarning("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(projectid)
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
### run the program
if len(sys.argv) > 1:
reconid = int(sys.argv[1])
makeCoranKeepPlot(reconid)
else:
reconids = getAllCoranRecons()
for reconid in reconids:
recondata = apRecon.getRefineRunDataFromID(reconid)
reconpath = recondata['path']['path']
print reconpath
os.chdir(reconpath)
try:
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['expweight'] is False:
apDisplay.printWarning("Exposure weighting is turned off, make sure this is what you want")
if self.params['localavg'] is False:
apDisplay.printWarning("Trajectory local averaging is turned off, make sure this is what you want")
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# pixel size info
self.params['apix'] = apStack.getMicrographPixelSizeFromStackId(self.params['stackid'])
self.params['box'] = self.stackdata['boxsize']
self.params['particleradius'] = self.params['particleradius'] / self.params['apix']
if self.params['particleradius'] > self.params['box'] / 2.0:
apDisplay.printWarning("specified particle radius greater than box radius, \
setting particle radius to 0.8 * boxsize")
# micrograph & frame info
frames = qimage['use frames']
nframes = len(frames)
if self.params['framelastali'] is None:
self.params['framelastali'] = frames[-1]
if self.params['framelastave'] is None:
self.params['framelastave'] = frames[-1]
# microscope kV
self.params['kv'] = qimage['scope']['high tension']/1000.0
# query exposure per frame, if not set here
if self.params['total_dose'] is not None:
dose = self.params['total_dose']
else:
try:
dose = apDatabase.getDoseFromImageData(qimage)
except:
apDisplay.printError("dose not specified and not in database, please specify explicitly")
if self.params['expperframe'] is None and self.params['expweight'] is True:
if dose is not None:
self.params['expperframe'] = dose / nframes
else:
apDisplay.printError("exposure per frame needs to be specified, cannot find in database")
# dimensions
self.params['framex'] = int(apDatabase.getDimensionsFromImageData(qimage)['x'])
self.params['framey'] = int(apDatabase.getDimensionsFromImageData(qimage)['y'])
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(
self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '" + newdbname + "'",
"green")
### get stack data
self.stack = {}
if self.params['stackid'] is not None:
self.stack['data'] = apStack.getOnlyStackData(
self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(
self.params['stackid'])
self.stack['file'] = os.path.join(
self.stack['data']['path']['path'], self.stack['data']['name'])
else:
self.stack['data'] = appiondata.ApAlignStackData.direct_query(
self.params['alignstackid'])
self.stack['apix'] = self.stack['pixelsize']
self.stack['boxsize'] = self.stack['boxsize']
self.stack['file'] = self.stack['imagicfile']
### check conflicts
if self.params['stackid'] is None and self.params[
'alignstackid'] is None:
apDisplay.printError("stack id OR alignstack id was not defined")
if self.params['stackid'] is not None and self.params[
'alignstackid'] is not None:
apDisplay.printError(
"either specify stack id OR alignstack id, not both")
if self.params['generations'] is None:
apDisplay.printError("number of generations was not provided")
maxparticles = 500000
if self.params['numpart'] > maxparticles:
apDisplay.printError("too many particles requested, max: " +
str(maxparticles) + " requested: " +
str(self.params['numpart']))
if self.params['numpart'] > apFile.numImagesInStack(
self.stack['file']):
apDisplay.printError(
"trying to use more particles " + str(self.params['numpart']) +
" than available " +
str(apFile.numImagesInStack(self.stack['file'])))
if self.params['numpart'] is None:
self.params['numpart'] = apFile.numImagesInStack(
self.stack['file'])
boxsize = apStack.getStackBoxsize(self.params['stackid'])
if self.params['ou'] is None:
self.params['ou'] = (boxsize / 2.0) - 2
self.clipsize = int(math.floor(
boxsize / float(self.params['bin'] * 2))) * 2
if self.params['clipsize'] is not None:
if self.params['clipsize'] > self.clipsize:
apDisplay.printError("requested clipsize is too big %d > %d" %
(self.params['clipsize'], self.clipsize))
self.clipsize = self.params['clipsize']
self.mpirun = self.checkMPI()
if self.mpirun is None:
apDisplay.printError("There is no MPI installed")
def upgradeProjectDB(projectdb,backup=True):
### set version of database
selectq = " SELECT * FROM `install` WHERE `key`='version'"
values = projectdb.returnCustomSQL(selectq)
if values:
projectdb.updateColumn("install", "value", "'2.0'",
"install.key = 'version'",timestamp=False)
else:
insertq = "INSERT INTO `install` (`key`, `value`) VALUES ('version', '2.0')"
projectdb.executeCustomSQL(insertq)
if __name__ == "__main__":
projectids = getProjectIds()
projectdb = dbupgrade.DBUpgradeTools('projectdata', drop=True)
for projectid in projectids:
appiondbname = apProject.getAppionDBFromProjectId(projectid, die=False)
if appiondbname is None:
continue
elif not projectdb.databaseExists(appiondbname):
print "\033[31merror database %s does not exist\033[0m"%(appiondbname)
time.sleep(1)
continue
if apProject.setDBfromProjectId(projectid, die=False):
reconrunids = getReconRunsIds()
for reconrunid in reconrunids:
apRecon.setGoodBadParticlesFromReconId(reconrunid)
# Tomography upgrades
createExcludedColumnByFakeInsert()
allalignrundata = getTomoAlignmentRuns()
for alignrun in allalignrundata:
insertTiltsInAlign(alignrun)
#===============
#===============
if __name__ == "__main__":
print "Usage: apStackMeanPlot.py <stackid> <#points> <projectid>"
if len(sys.argv) > 1:
stackid = int(sys.argv[1])
else:
#stackid=1279
stackid=1291
if len(sys.argv) > 2:
gridpoints = int(sys.argv[2])
else:
gridpoints=16
if len(sys.argv) > 3:
projectid = int(sys.argv[3])
else:
projectid=None
### setup correct database after we have read the project id
if projectid is not None:
apDisplay.printWarning("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(projectid)
import sinedon
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
makeStackMeanPlot(stackid, gridpoints)
def __init__(self,
optargs=sys.argv[1:],
quiet=False,
useglobalparams=True,
maxnproc=None):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.successful_run = False
self.clusterjobdata = None
self.params = {}
sys.stdout.write("\n\n")
self.quiet = quiet
self.maxnproc = maxnproc
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.timestamp = apParam.makeTimestamp()
self.argdict = {}
self.optdict = {}
apDisplay.printMsg("Time stamp: " + self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
apDisplay.printMsg("Function name: " + self.functionname)
self.appiondir = apParam.getAppionDirectory()
apDisplay.printMsg("Appion directory: " + self.appiondir)
self.parsePythonPath()
loadavg = os.getloadavg()[0]
if loadavg > 2.0:
apDisplay.printMsg("Load average is high " +
str(round(loadavg, 2)))
loadsquared = loadavg * loadavg
time.sleep(loadavg)
apDisplay.printMsg("New load average " +
str(round(os.getloadavg()[0], 2)))
self.setLockname('lock')
### setup default parser: run directory, etc.
self.setParams(optargs, useglobalparams)
#if 'outdir' in self.params and self.params['outdir'] is not None:
# self.params['rundir'] = self.params['outdir']
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(
self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '" + newdbname + "'",
"green")
### check if user wants to print help message
if 'commit' in self.params and self.params['commit'] is True:
apDisplay.printMsg("Committing data to database")
else:
apDisplay.printWarning("Not committing data to database")
self.checkConflicts()
if useglobalparams is True:
self.checkGlobalConflicts()
### setup run directory
self.setProcessingDirName()
self.setupRunDirectory()
### Start pool of threads to run subprocesses.
### Later you will use self.process_launcher.launch(...) to
### put commands into the queue.
### There is currently a timeout built into it that will cause
### the threads to die if they have no tasks after 10 seconds.
self.process_launcher = apThread.ProcessLauncher(
2, self.params['rundir'])
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### upload command line parameters to database
self.uploadScriptData()
### any custom init functions go here
self.onInit()
if values:
projectdb.updateColumn("install",
"value",
"'2.0'",
"install.key = 'version'",
timestamp=False)
else:
insertq = "INSERT INTO `install` (`key`, `value`) VALUES ('version', '2.0')"
projectdb.executeCustomSQL(insertq)
if __name__ == "__main__":
projectids = getProjectIds()
projectdb = dbupgrade.DBUpgradeTools('projectdata', drop=True)
for projectid in projectids:
appiondbname = apProject.getAppionDBFromProjectId(projectid, die=False)
if appiondbname is None:
continue
elif not projectdb.databaseExists(appiondbname):
print "\033[31merror database %s does not exist\033[0m" % (
appiondbname)
time.sleep(1)
continue
if apProject.setDBfromProjectId(projectid, die=False):
reconrunids = getReconRunsIds()
for reconrunid in reconrunids:
apRecon.setGoodBadParticlesFromReconId(reconrunid)
# Tomography upgrades
createExcludedColumnByFakeInsert()
allalignrundata = getTomoAlignmentRuns()
for alignrun in allalignrundata:
def __init__(self,optargs=sys.argv[1:],quiet=False,useglobalparams=True,maxnproc=None):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.successful_run = False
self.clusterjobdata = None
self.params = {}
sys.stdout.write("\n\n")
self.quiet = quiet
self.maxnproc = maxnproc
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.timestamp = apParam.makeTimestamp()
self.argdict = {}
self.optdict = {}
apDisplay.printMsg("Time stamp: "+self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
apDisplay.printMsg("Function name: "+self.functionname)
self.appiondir = apParam.getAppionDirectory()
apDisplay.printMsg("Appion directory: "+self.appiondir)
hostname = apParam.getHostname()
apDisplay.printMsg("Processing hostname: "+hostname)
self.parsePythonPath()
# loadavg = os.getloadavg()[0]
# if loadavg > 2.0:
# apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
# loadsquared = loadavg*loadavg
# time.sleep(loadavg)
# apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2)))
self.setLockname('lock')
### setup default parser: run directory, etc.
self.setParams(optargs,useglobalparams)
#if 'outdir' in self.params and self.params['outdir'] is not None:
# self.params['rundir'] = self.params['outdir']
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
### check if user wants to print help message
if 'commit' in self.params and self.params['commit'] is True:
apDisplay.printMsg("Committing data to database")
else:
apDisplay.printWarning("Not committing data to database")
self.checkConflicts()
if useglobalparams is True:
self.checkGlobalConflicts()
### setup run directory
self.setProcessingDirName()
self.setupRunDirectory()
### Start pool of threads to run subprocesses.
### Later you will use self.process_launcher.launch(...) to
### put commands into the queue.
### There is currently a timeout built into it that will cause
### the threads to die if they have no tasks after 10 seconds.
self.process_launcher = apThread.ProcessLauncher(2, self.params['rundir'])
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### upload command line parameters to database
self.uploadScriptData()
### any custom init functions go here
self.onInit()