def setRunDir(self):
"""
this function only runs if no rundir is defined at the command line
"""
if self.params['rundir'] is None:
if ('sessionname' in self.params and self.params['sessionname'] is not None ):
# command line users may use sessionname rather than expId
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']])
else:
if ('expId' in self.params and self.params['expId']):
# expId should always be included from appionwrapper derived appionscript
sessiondata = apDatabase.getSessionDataFromSessionId(self.params['expId'])
self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']])
# The rest should not be needed with appionwrapper format
from appionlib import apStack
if ( self.params['rundir'] is None
and 'reconid' in self.params
and self.params['reconid'] is not None ):
self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid'], msg=False)
if ( self.params['rundir'] is None
and 'stackid' in self.params
and self.params['stackid'] is not None ):
#auto set the run directory
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
path = os.path.abspath(stackdata['path']['path'])
path = os.path.dirname(path)
path = os.path.dirname(path)
self.params['rundir'] = os.path.join(path, self.processdirname, self.params['runname'])
self.params['outdir'] = self.params['rundir']
def checkGlobalConflicts(self):
"""
put in any conflicting parameters
"""
appionScript.AppionScript.checkGlobalConflicts(self)
if self.params['runname'] is None:
apDisplay.printError("please enter a runname, example: 'runname=run1'")
if self.params['runname'] == 'templates':
apDisplay.printError("templates is a reserved runname, please use another runname")
if self.params['runname'] == 'models':
apDisplay.printError("models is a reserved runname, please use another runname")
if self.params['rundir']is not None and self.params['runname'] != os.path.basename(self.params['rundir']):
apDisplay.printError("runname and rundir basename are different: "
+self.params['runname']+" vs. "+os.path.basename(self.params['rundir']))
if self.params['mrcnames'] and self.params['preset']:
apDisplay.printError("preset can not be specified if particular images have been specified")
if (self.params['sessionname'] is None and self.params['expid'] is None) and self.params['mrcnames'] is None:
apDisplay.printError("please specify an mrc name or session")
if self.params['sessionname'] is None and self.params['expid'] is not None:
self.params['sessionname'] = apDatabase.getSessionDataFromSessionId(self.params['expid'])['name']
if self.params['sessionname'] is not None and self.params['projectid'] is not None:
### Check that project and images are in sync
imgproject = apProject.getProjectIdFromSessionName(self.params['sessionname'])
if imgproject and imgproject != self.params['projectid']:
apDisplay.printError("project id and session do not correlate")
def undoCTFCorrect(self, newstackroot):
# At this point, the stack needs to be remade un-ctf-corrected, and possibly normalized and/or inverted
apDisplay.printWarning('Relion needs a stack without ctf correction. A new stack is being made....')
# Gather all the makestack parameters
totalpart = self.originalStackData.numpart
numpart = totalpart if not self.params['last'] else min(self.params['last'],totalpart)
stackpathname = os.path.basename( self.originalStackData.path )
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir,'start.hed')
presetname = self.originalStackData.preset
# binning is combination of the original binning of the stack and the preparation binnning
bin = self.originalStackData.bin * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin
lowpasstext = setArgText( 'lowpass', ( self.params['lowpass'], self.originalStackData.lowpass ), False)
highpasstext = setArgText( 'highpass', ( self.params['highpass'], self.originalStackData.highpass ), True)
partlimittext = setArgText('partlimit',(numpart,),False)
xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'],), True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
reversetext = '--reverse' if self.originalStackData.reverse else ''
defoctext = '--defocpair' if self.originalStackData.defocpair else ''
inverttext = '--no-invert' if not self.invert else ''
# Build the makestack2 command
cmd = "makestack2.py "
cmd += (" --single=%s --fromstackid=%d %s %s %s %s %s %s "%
(os.path.basename(newstackimagicfile),selectionid,stackid,lowpasstext,
highpasstext,partlimittext,reversetext,defoctext,inverttext))
cmd += (" --normalized %s --boxsize=%d --bin=%d "%
(xmipp_normtext,unbinnedboxsize,bin))
cmd += (" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "%
(stackpathname,stackid,projectid))
cmd += (" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d "%
(presetname,newstackrunname,newstackrundir,sessionname,sessionid))
cmd += (" --no-wait --no-commit --no-continue --jobtype=makestack2 ")
# Run the command
logfilepath = os.path.join(newstackrundir,'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd,logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Make sure our new stack params reflects the changes made
# Use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
self.stack['file'] = newstackroot+'.hed'
# Clean up
boxfiles = glob.glob("*.box")
for boxfile in boxfiles:
apFile.removeFile(boxfile)
def checkGlobalConflicts(self):
"""
put in any conflicting parameters
"""
appionScript.AppionScript.checkGlobalConflicts(self)
if self.params['runname'] is None:
apDisplay.printError(
"please enter a runname, example: 'runname=run1'")
if self.params['runname'] == 'templates':
apDisplay.printError(
"templates is a reserved runname, please use another runname")
if self.params['runname'] == 'models':
apDisplay.printError(
"models is a reserved runname, please use another runname")
if self.params['rundir'] is not None and self.params[
'runname'] != os.path.basename(self.params['rundir']):
apDisplay.printError(
"runname and rundir basename are different: " +
self.params['runname'] + " vs. " +
os.path.basename(self.params['rundir']))
if self.params['mrcnames'] and self.params['preset']:
apDisplay.printError(
"preset can not be specified if particular images have been specified"
)
if (self.params['sessionname'] is None and self.params['expid'] is None
) and self.params['mrcnames'] is None:
apDisplay.printError("please specify an mrc name or session")
if self.params['sessionname'] is None and self.params[
'expid'] is not None:
self.params[
'sessionname'] = apDatabase.getSessionDataFromSessionId(
self.params['expid'])['name']
if self.params['sessionname'] is not None and self.params[
'projectid'] is not None:
### Check that project and images are in sync
imgproject = apProject.getProjectIdFromSessionName(
self.params['sessionname'])
if imgproject and imgproject != self.params['projectid']:
apDisplay.printError("project id and session do not correlate")
def convertToRefineStack(self):
'''
The stack is remaked without ctf correction and inverted and normalized if needed
'''
newstackroot = os.path.join(self.params['rundir'],os.path.basename(self.stack['file'])[:-4])
self.stack['phaseflipped'] = False
self.stack['format'] = 'relion' #TODO: Where is this used?
# If we just want the frealign param file, skip this function
if self.params['paramonly'] is True:
print 'newstackroot',newstackroot
return
# If we just need to normalize, run xmipp_normalize
if self.normalize and not self.un_ctf_correct:
extname,addformat = self.proc2dFormatConversion()
outstack = os.path.join(self.params['rundir'], "start.%s" % extname)
self.xmippNormStack(self.stack['file'], outstack)
self.stack['file'] = outstack
# If we don't need to un-ctf-correct, we are done
if not self.un_ctf_correct:
return
# At this point, the stack needs to be remade un-ctf-corrected, and possibly normalized and/or inverted
apDisplay.printWarning('Relion needs a stack without ctf correction. A new stack is being made....')
# Gather all the makestack parameters
totalpart = self.originalStackData.numpart
numpart = totalpart if not self.params['last'] else min(self.params['last'],totalpart)
stackpathname = os.path.basename( self.originalStackData.path )
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir,'start.hed')
presetname = self.originalStackData.preset
# binning is combination of the original binning of the stack and the preparation binnning
bin = self.originalStackData.bin * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin
lowpasstext = setArgText( 'lowpass', ( self.params['lowpass'], self.originalStackData.lowpass ), False)
highpasstext = setArgText( 'highpass', ( self.params['highpass'], self.originalStackData.highpass ), True)
partlimittext = setArgText('partlimit',(numpart,),False)
xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'],), True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
reversetext = '--reverse' if self.originalStackData.reverse else ''
defoctext = '--defocpair' if self.originalStackData.defocpair else ''
inverttext = '--no-invert' if not self.invert else ''
# Build the makestack2 command
cmd = '''
makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s %s --normalized %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2
''' % (os.path.basename(newstackimagicfile),stackid,lowpasstext,highpasstext,partlimittext,reversetext,defoctext,inverttext,xmipp_normtext,unbinnedboxsize,bin,stackpathname,stackid,projectid,presetname,newstackrunname,newstackrundir,sessionname,sessionid)
# Run the command
logfilepath = os.path.join(newstackrundir,'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd,logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Convert the imagic stack to an MRC stack format
# Relion preferes an mrc stack with .mrcs extension
#self.ImagicStackToMrcStack(newstackimagicfile)
#os.rename(newstackimagicfile[:-4]+'.mrc',newstackroot+'.mrc')
# Make sure our new stack params reflects the changes made
# Use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
self.stack['file'] = newstackroot+'.hed'
# Clean up
rmfiles = glob.glob("*.box")
for rmfile in rmfiles:
apFile.removeFile(rmfile)
def convertToRefineStack(self):
'''
The stack is remaked without ctf correction and without invertion (ccd)
'''
newstackroot = os.path.join(self.params['rundir'],
os.path.basename(self.stack['file'])[:-4])
if self.params['paramonly'] is True:
print 'newstackroot', newstackroot
self.setFrealignStack(newstackroot)
return
# copy existing refined stack if possible
existing_refine_stack = apStack.getExistingRefineStack(
self.stack['data'],
'frealign',
False,
self.params['last'],
self.params['bin'],
lowpass=self.params['lowpass'],
highpass=self.params['highpass'])
try:
if existing_refine_stack:
if existing_refine_stack != os.path.join(
self.params['rundir'], newstackroot + '.mrc'):
shutil.copyfile(existing_refine_stack,
newstackroot + '.mrc')
new_partorderfile = os.path.join(self.params['rundir'],
'stackpartorder.list')
existing_partorderfile = os.path.join(
os.path.dirname(existing_refine_stack),
'stackpartorder.list')
# particle order list also need to be copied
if not os.path.isfile(new_partorderfile) and os.path.isfile(
existing_partorderfile):
shutil.copyfile(existing_partorderfile, new_partorderfile)
self.setFrealignStack(newstackroot)
return
except:
# make one from scratch if there is error
apDisplay.printMsg(
'Error copying existing refine stack, will regenerate')
if self.no_ctf_correction:
self.ImagicStackToFrealignMrcStack(self.stack['file'])
self.setFrealignStack(newstackroot)
return
# stack need to be remade without ctf correction
apDisplay.printWarning(
'Frealign needs a stack without ctf correction. A new stack is being made....'
)
stackdata = self.stack['data']
madestackdata = self.getOriginalStackMadeByMakeStack()
stackid = stackdata.dbid
stackruns = apStack.getStackRunsFromStack(madestackdata)
# To Do: need to handle combined stack
stackrun = stackruns[0]
stackpathname = os.path.basename(stackdata['path']['path'])
totalpart = apStack.getNumberStackParticlesFromId(stackid)
if not self.params['last']:
numpart = totalpart
else:
numpart = min(self.params['last'], totalpart)
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir, 'temp.hed')
# use first particle image to get presetname
oneparticle = apStack.getOneParticleFromStackId(stackid,
particlenumber=1)
preset = oneparticle['particle']['image']['preset']
if preset:
presetname = preset['name']
else:
presetname = 'manual'
# use first stack run to get parameters
paramdata = stackrun['stackParams']
# binning is combination of the original binning of the stack and the preparation binnning
bin = paramdata['bin'] * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * paramdata['bin']
lowpasstext = self.setArgText(
'lowpass', (self.params['lowpass'], paramdata['lowpass']), False)
highpasstext = self.setArgText(
'highpass', (self.params['highpass'], paramdata['highpass']), True)
partlimittext = self.setArgText('partlimit', (numpart, ), False)
#normalization
normtext = ''
if 'xmipp-norm' in paramdata.keys():
# myami-3.1 or before
normtext = self.setArgText('xmipp-normalize',
(paramdata['xmipp-norm'], ), True)
normtext = '--normalized ' + normtext
elif 'normalizemethod' in paramdata.keys():
# myami-3.2 or newer
normtext = '--normalize-method=%s' % (
paramdata['normalizemethod'], )
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
# The assumption is that the image is from ice grid and digital camera (black particles on white background
if 'reverse' in self.stackrunlogparams.keys():
reversetext = '--reverse'
else:
reversetext = ''
if 'defocpair' in self.stackrunlogparams.keys():
defoctext = '--defocpair'
else:
defoctext = ''
cmd = '''
makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s --no-invert %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2
''' % (os.path.basename(newstackimagicfile), stackid, lowpasstext,
highpasstext, partlimittext, reversetext, defoctext, normtext,
unbinnedboxsize, bin, stackpathname, stackid, projectid, presetname,
newstackrunname, newstackrundir, sessionname, sessionid)
logfilepath = os.path.join(newstackrundir, 'frealignstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd, logfilepath)
if returncode > 0:
apDisplay.printError('Error in Frealign specific stack making')
self.ImagicStackToFrealignMrcStack(newstackimagicfile)
os.rename(newstackimagicfile[:-4] + '.mrc', newstackroot + '.mrc')
self.setFrealignStack(newstackroot)
# use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
#clean up
rmfiles = glob.glob("*.box")
for rmfile in rmfiles:
apFile.removeFile(rmfile)
def runRelionPreprocess(self, newstackroot):
'''
1. Use stackIntoPicks.py to extract the particle locations from the selected stack.
2. Run makestack2.py without ctf correction or normalization using the stackIntoPicks result as the Particles run.
3. Run relion_preprocess with rescale and norm. Outputs .mrcs file.
Neil: most of these steps could be done more generally
'''
apDisplay.printWarning("Making a new stack from original images")
# Build the stackIntoPicks command
apDisplay.printMsg('Extracting the particle locations from your selected stack.')
newstackrunname = self.params['runname']+"_particles"
newstackrundir = self.params['rundir']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
sessionid = int(self.params['expid'])
cmd = "stackIntoPicks.py "
cmd += (" --stackid=%d --projectid=%d --runname=%s --rundir=%s "%
(stackid,projectid,newstackrunname,newstackrundir))
cmd += (" --commit --expId=%d --jobtype=stackintopicks "%
(sessionid))
# Run the command
logfilepath = os.path.join(newstackrundir,'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd,logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Build the makestack2 command
'''
This is the command we want to make a stack from a stackIntoPicks run
makestack2.py
--single=start.hed --selectionid=130 --invert --boxsize=320 --bin=2
--description="made from stackrun1 using stackintopicks"
--runname=stack66 --rundir=/ami/data00/appion/zz07jul25b/stacks/stack66
--commit --preset=en --projectid=303 --session=zz07jul25b
--no-rejects --no-wait --continue --expid=8556 --jobtype=makestack2
--ppn=1 --nodes=1 --walltime=240 --jobid=2016
'''
apDisplay.printMsg('Using selected stack particle locations to make a Relion ready stack....')
# Get the ID of the stackIntoPicks run we just created
#apParticle.getSelectionIdFromName(runname, sessionname)
runq = appiondata.ApSelectionRunData()
runq['name'] = newstackrunname
runq['session'] = leginondata.SessionData.direct_query(self.params['expid'])
rundatas = runq.query(results=1)
print rundatas
if rundatas:
selectionid = rundatas[0].dbid
else:
apDisplay.printError("Error creating Relion ready stack. Could not find stackIntoPicks.py data in database.\n")
# Gather all the makestack parameters
totalpart = self.originalStackData.numpart
numpart = totalpart if not self.params['last'] else min(self.params['last'],totalpart)
stackpathname = os.path.basename( self.originalStackData.path )
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir,'start.hed')
presetname = self.originalStackData.preset
# binning is combination of the original binning of the stack and the preparation binnning
bin = self.originalStackData.bin * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin
lowpasstext = setArgText( 'lowpass', ( self.params['lowpass'], self.originalStackData.lowpass ), False)
highpasstext = setArgText( 'highpass', ( self.params['highpass'], self.originalStackData.highpass ), True)
partlimittext = setArgText('partlimit',(numpart,),False)
xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'],), True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
reversetext = '--reverse' if self.originalStackData.reverse else ''
defoctext = '--defocpair' if self.originalStackData.defocpair else ''
inverttext = '--no-invert' if not self.invert else ''
# Build the makestack2 command
cmd = "makestack2.py "
cmd += (" --single=%s --selectionid=%d %s --boxsize=%d --bin=%d "%
(os.path.basename(newstackimagicfile),selectionid,inverttext,
unbinnedboxsize,bin))
cmd += (" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "%
(stackpathname,stackid,projectid))
cmd += (" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d "%
(presetname,newstackrunname,newstackrundir,sessionname,sessionid))
cmd += " --no-wait --no-commit --no-continue --jobtype=makestack2 "
# Run the command
logfilepath = os.path.join(newstackrundir,'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd,logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Clean up
boxfiles = glob.glob("*.box")
for boxfile in boxfiles:
apFile.removeFile(boxfile)
# Make sure our new stack params reflects the changes made
# Use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
self.stack['file'] = newstackroot+'.hed'
# Run Relion pre-process command
'''
Setup the follow relion preprocess command to normalize the new stack:
relion_preprocess \
--o particles --norm --bg_radius 60 --white_dust -1 --black_dust -1 \
--operate_on /ami/data00/appion/zz07jul25b/stacks/stack63_no_xmipp_norm/start.hed
'''
apDisplay.printMsg('Running Relion preprocessing to normalize your Relion Ready stack....')
bg_radius = math.floor((self.stack['boxsize'] / 2) - 1)
relioncmd = "relion_preprocess "
relioncmd += " --o particles --norm --bg_radius %d "%(bg_radius)
relioncmd += " --white_dust -1 --black_dust -1 --operate_on %s "%(newstackimagicfile)
apParam.runCmd(relioncmd, package="Relion", verbose=True, showcmd=True)
self.stack['file'] = 'particles.mrcs'
def undoCTFCorrect(self, newstackroot):
# At this point, the stack needs to be remade un-ctf-corrected, and possibly normalized and/or inverted
apDisplay.printWarning(
'Relion needs a stack without ctf correction. A new stack is being made....'
)
# Gather all the makestack parameters
totalpart = self.originalStackData.numpart
numpart = totalpart if not self.params['last'] else min(
self.params['last'], totalpart)
stackpathname = os.path.basename(self.originalStackData.path)
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir, 'start.hed')
presetname = self.originalStackData.preset
# binning is combination of the original binning of the stack and the preparation binnning
bin = self.originalStackData.bin * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin
lowpasstext = setArgText(
'lowpass',
(self.params['lowpass'], self.originalStackData.lowpass), False)
highpasstext = setArgText(
'highpass',
(self.params['highpass'], self.originalStackData.highpass), True)
partlimittext = setArgText('partlimit', (numpart, ), False)
xmipp_normtext = setArgText('xmipp-normalize',
(self.params['xmipp-norm'], ), True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
reversetext = '--reverse' if self.originalStackData.reverse else ''
defoctext = '--defocpair' if self.originalStackData.defocpair else ''
inverttext = '--no-invert' if not self.invert else ''
# Build the makestack2 command
cmd = "makestack2.py "
cmd += (" --single=%s --fromstackid=%d %s %s %s %s %s %s " %
(os.path.basename(newstackimagicfile), selectionid, stackid,
lowpasstext, highpasstext, partlimittext, reversetext,
defoctext, inverttext))
cmd += (" --normalized %s --boxsize=%d --bin=%d " %
(xmipp_normtext, unbinnedboxsize, bin))
cmd += (
" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "
% (stackpathname, stackid, projectid))
cmd += (
" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d " %
(presetname, newstackrunname, newstackrundir, sessionname,
sessionid))
cmd += (" --no-wait --no-commit --no-continue --jobtype=makestack2 ")
# Run the command
logfilepath = os.path.join(newstackrundir, 'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd, logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Make sure our new stack params reflects the changes made
# Use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
self.stack['file'] = newstackroot + '.hed'
# Clean up
boxfiles = glob.glob("*.box")
for boxfile in boxfiles:
apFile.removeFile(boxfile)
def runRelionPreprocess(self, newstackroot):
'''
1. Use stackIntoPicks.py to extract the particle locations from the selected stack.
2. Run makestack2.py without ctf correction or normalization using the stackIntoPicks result as the Particles run.
3. Run relion_preprocess with rescale and norm. Outputs .mrcs file.
Neil: most of these steps could be done more generally
'''
apDisplay.printWarning("Making a new stack from original images")
# Build the stackIntoPicks command
apDisplay.printMsg(
'Extracting the particle locations from your selected stack.')
newstackrunname = self.params['runname'] + "_particles"
newstackrundir = self.params['rundir']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
sessionid = int(self.params['expid'])
cmd = "stackIntoPicks.py "
cmd += (" --stackid=%d --projectid=%d --runname=%s --rundir=%s " %
(stackid, projectid, newstackrunname, newstackrundir))
cmd += (" --commit --expId=%d --jobtype=stackintopicks " % (sessionid))
# Run the command
logfilepath = os.path.join(newstackrundir, 'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd, logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Build the makestack2 command
'''
This is the command we want to make a stack from a stackIntoPicks run
makestack2.py
--single=start.hed --selectionid=130 --invert --boxsize=320 --bin=2
--description="made from stackrun1 using stackintopicks"
--runname=stack66 --rundir=/ami/data00/appion/zz07jul25b/stacks/stack66
--commit --preset=en --projectid=303 --session=zz07jul25b
--no-rejects --no-wait --continue --expid=8556 --jobtype=makestack2
--ppn=1 --nodes=1 --walltime=240 --jobid=2016
'''
apDisplay.printMsg(
'Using selected stack particle locations to make a Relion ready stack....'
)
# Get the ID of the stackIntoPicks run we just created
#apParticle.getSelectionIdFromName(runname, sessionname)
runq = appiondata.ApSelectionRunData()
runq['name'] = newstackrunname
runq['session'] = leginondata.SessionData.direct_query(
self.params['expid'])
rundatas = runq.query(results=1)
print rundatas
if rundatas:
selectionid = rundatas[0].dbid
else:
apDisplay.printError(
"Error creating Relion ready stack. Could not find stackIntoPicks.py data in database.\n"
)
# Gather all the makestack parameters
totalpart = self.originalStackData.numpart
numpart = totalpart if not self.params['last'] else min(
self.params['last'], totalpart)
stackpathname = os.path.basename(self.originalStackData.path)
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir, 'start.hed')
presetname = self.originalStackData.preset
# binning is combination of the original binning of the stack and the preparation binnning
bin = self.originalStackData.bin * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin
lowpasstext = setArgText(
'lowpass',
(self.params['lowpass'], self.originalStackData.lowpass), False)
highpasstext = setArgText(
'highpass',
(self.params['highpass'], self.originalStackData.highpass), True)
partlimittext = setArgText('partlimit', (numpart, ), False)
xmipp_normtext = setArgText('xmipp-normalize',
(self.params['xmipp-norm'], ), True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
reversetext = '--reverse' if self.originalStackData.reverse else ''
defoctext = '--defocpair' if self.originalStackData.defocpair else ''
inverttext = '--no-invert' if not self.invert else ''
# Build the makestack2 command
cmd = "makestack2.py "
cmd += (" --single=%s --selectionid=%d %s --boxsize=%d --bin=%d " %
(os.path.basename(newstackimagicfile), selectionid, inverttext,
unbinnedboxsize, bin))
cmd += (
" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "
% (stackpathname, stackid, projectid))
cmd += (
" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d " %
(presetname, newstackrunname, newstackrundir, sessionname,
sessionid))
cmd += " --no-wait --no-commit --no-continue --jobtype=makestack2 "
# Run the command
logfilepath = os.path.join(newstackrundir, 'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd, logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Clean up
boxfiles = glob.glob("*.box")
for boxfile in boxfiles:
apFile.removeFile(boxfile)
# Make sure our new stack params reflects the changes made
# Use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
self.stack['file'] = newstackroot + '.hed'
# Run Relion pre-process command
'''
Setup the follow relion preprocess command to normalize the new stack:
relion_preprocess \
--o particles --norm --bg_radius 60 --white_dust -1 --black_dust -1 \
--operate_on /ami/data00/appion/zz07jul25b/stacks/stack63_no_xmipp_norm/start.hed
'''
apDisplay.printMsg(
'Running Relion preprocessing to normalize your Relion Ready stack....'
)
bg_radius = math.floor((self.stack['boxsize'] / 2) - 1)
relioncmd = "relion_preprocess "
relioncmd += " --o particles --norm --bg_radius %d " % (bg_radius)
relioncmd += " --white_dust -1 --black_dust -1 --operate_on %s " % (
newstackimagicfile)
apParam.runCmd(relioncmd, package="Relion", verbose=True, showcmd=True)
self.stack['file'] = 'particles.mrcs'
def convertToRefineStack(self):
'''
The stack is remaked without ctf correction and without invertion (ccd)
'''
newstackroot = os.path.join(self.params['rundir'],os.path.basename(self.stack['file'])[:-4])
if self.params['paramonly'] is True:
print 'newstackroot',newstackroot
self.setFrealignStack(newstackroot)
return
# copy existing refined stack if possible
existing_refine_stack = apStack.getExistingRefineStack(self.stack['data'],'frealign',False,self.params['last'],self.params['bin'],lowpass=self.params['lowpass'],highpass=self.params['highpass'])
if existing_refine_stack:
if existing_refine_stack != os.path.join(self.params['rundir'],newstackroot+'.mrc'):
shutil.copyfile(existing_refine_stack,newstackroot+'.mrc')
new_partorderfile = os.path.join(self.params['rundir'],'stackpartorder.list')
existing_partorderfile = os.path.join(os.path.dirname(existing_refine_stack),'stackpartorder.list')
# particle order list also need to be copied
if not os.path.isfile(new_partorderfile) and os.path.isfile(existing_partorderfile):
shutil.copyfile(existing_partorderfile,new_partorderfile)
self.setFrealignStack(newstackroot)
return
if self.no_ctf_correction:
self.ImagicStackToFrealignMrcStack(self.stack['file'])
self.setFrealignStack(newstackroot)
return
# stack need to be remade without ctf correction
apDisplay.printWarning('Frealign needs a stack without ctf correction. A new stack is being made....')
stackdata = self.stack['data']
madestackdata = self.getOriginalStackMadeByMakeStack()
stackid = stackdata.dbid
stackruns = apStack.getStackRunsFromStack(madestackdata)
# To Do: need to handle combined stack
stackrun = stackruns[0]
stackpathname = os.path.basename(stackdata['path']['path'])
totalpart = apStack.getNumberStackParticlesFromId(stackid)
if not self.params['last']:
numpart = totalpart
else:
numpart = min(self.params['last'],totalpart)
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir,'temp.hed')
# use first particle image to get presetname
oneparticle = apStack.getOneParticleFromStackId(stackid, particlenumber=1)
preset =oneparticle['particle']['image']['preset']
if preset:
presetname = preset['name']
else:
presetname = 'manual'
# use first stack run to get parameters
paramdata = stackrun['stackParams']
# binning is combination of the original binning of the stack and the preparation binnning
bin = paramdata['bin']*self.params['bin']
unbinnedboxsize = self.stack['boxsize']*paramdata['bin']
lowpasstext = self.setArgText('lowpass',(self.params['lowpass'],paramdata['lowpass']),False)
highpasstext = self.setArgText('highpass',(self.params['highpass'],paramdata['highpass']),True)
partlimittext = self.setArgText('partlimit',(numpart,),False)
xmipp_normtext = self.setArgText('xmipp-normalize',(paramdata['xmipp-norm'],),True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
# The assumption is that the image is from ice grid and digital camera (black particles on white background
if 'reverse' in self.stackrunlogparams.keys():
reversetext = '--reverse'
else:
reversetext = ''
if 'defocpair' in self.stackrunlogparams.keys():
defoctext = '--defocpair'
else:
defoctext = ''
cmd = '''
makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s --no-invert --normalized %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2
''' % (os.path.basename(newstackimagicfile),stackid,lowpasstext,highpasstext,partlimittext,reversetext,defoctext,xmipp_normtext,unbinnedboxsize,bin,stackpathname,stackid,projectid,presetname,newstackrunname,newstackrundir,sessionname,sessionid)
logfilepath = os.path.join(newstackrundir,'frealignstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd,logfilepath)
if returncode > 0:
apDisplay.printError('Error in Frealign specific stack making')
self.ImagicStackToFrealignMrcStack(newstackimagicfile)
os.rename(newstackimagicfile[:-4]+'.mrc',newstackroot+'.mrc')
self.setFrealignStack(newstackroot)
# use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
#clean up
rmfiles = glob.glob("*.box")
for rmfile in rmfiles:
apFile.removeFile(rmfile)
cursor.close()
dbConnection.close()
except MySQLdb.DatabaseError, e:
sys.stderr.write("Warning: Failure determining project database: %s \n" % (e))
#if jobId is not set, assume there is no entry in ApAppionJobData for this run
if not jobObject.getJobId():
### insert a cluster job
# TODO: what happens when this runs remotely???
rundir = jobObject.getRundir()
pathq = appiondata.ApPathData(path=os.path.abspath(rundir))
clustq = appiondata.ApAppionJobData()
clustq['path'] = pathq
clustq['jobtype'] = jobObject.getJobType()
clustq['name'] = jobObject.getJobName()
remoterundir = jobObject.getOutputDir()
remoterundirq = appiondata.ApPathData(path=os.path.abspath(remoterundir))
clustq['clusterpath'] = remoterundirq
clustq['session'] = apDatabase.getSessionDataFromSessionId(jobObject.getExpId())
clustq['user'] = os.getlogin()
clustq['cluster'] = os.uname()[1]
clustq['clusterjobid'] = job
clustq['status'] = "Q"
clustq.insert()
return retValue
def convertToRefineStack(self):
'''
The stack is remaked without ctf correction and inverted and normalized if needed
'''
newstackroot = os.path.join(self.params['rundir'],
os.path.basename(self.stack['file'])[:-4])
self.stack['phaseflipped'] = False
self.stack['format'] = 'relion' #TODO: Where is this used?
# If we just want the frealign param file, skip this function
if self.params['paramonly'] is True:
print 'newstackroot', newstackroot
return
# If we just need to normalize, run xmipp_normalize
if self.normalize and not self.un_ctf_correct:
extname, addformat = self.proc2dFormatConversion()
outstack = os.path.join(self.params['rundir'],
"start.%s" % extname)
self.xmippNormStack(self.stack['file'], outstack)
self.stack['file'] = outstack
# If we don't need to un-ctf-correct, we are done
if not self.un_ctf_correct:
return
# At this point, the stack needs to be remade un-ctf-corrected, and possibly normalized and/or inverted
apDisplay.printWarning(
'Relion needs a stack without ctf correction. A new stack is being made....'
)
# Gather all the makestack parameters
totalpart = self.originalStackData.numpart
numpart = totalpart if not self.params['last'] else min(
self.params['last'], totalpart)
stackpathname = os.path.basename(self.originalStackData.path)
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir, 'start.hed')
presetname = self.originalStackData.preset
# binning is combination of the original binning of the stack and the preparation binnning
bin = self.originalStackData.bin * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin
lowpasstext = setArgText(
'lowpass',
(self.params['lowpass'], self.originalStackData.lowpass), False)
highpasstext = setArgText(
'highpass',
(self.params['highpass'], self.originalStackData.highpass), True)
partlimittext = setArgText('partlimit', (numpart, ), False)
xmipp_normtext = setArgText('xmipp-normalize',
(self.params['xmipp-norm'], ), True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
reversetext = '--reverse' if self.originalStackData.reverse else ''
defoctext = '--defocpair' if self.originalStackData.defocpair else ''
inverttext = '--no-invert' if not self.invert else ''
# Build the makestack2 command
cmd = '''
makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s %s --normalized %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2
''' % (os.path.basename(newstackimagicfile), stackid,
lowpasstext, highpasstext, partlimittext, reversetext,
defoctext, inverttext, xmipp_normtext, unbinnedboxsize,
bin, stackpathname, stackid, projectid, presetname,
newstackrunname, newstackrundir, sessionname, sessionid)
# Run the command
logfilepath = os.path.join(newstackrundir, 'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd, logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Convert the imagic stack to an MRC stack format
# Relion preferes an mrc stack with .mrcs extension
#self.ImagicStackToMrcStack(newstackimagicfile)
#os.rename(newstackimagicfile[:-4]+'.mrc',newstackroot+'.mrc')
# Make sure our new stack params reflects the changes made
# Use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
self.stack['file'] = newstackroot + '.hed'
# Clean up
rmfiles = glob.glob("*.box")
for rmfile in rmfiles:
apFile.removeFile(rmfile)
cursor.close()
dbConnection.close()
except MySQLdb.DatabaseError, e:
sys.stderr.write("Warning: Failure determining project database: %s \n" % (e))
#if jobId is not set, assume there is no entry in ApAppionJobData for this run
if not jobObject.getJobId():
### insert a cluster job
# TODO: what happens when this runs remotely???
rundir = jobObject.getRundir()
pathq = appiondata.ApPathData(path=os.path.abspath(rundir))
clustq = appiondata.ApAppionJobData()
clustq['path'] = pathq
clustq['jobtype'] = jobObject.getJobType()
clustq['name'] = jobObject.getJobName()
remoterundir = jobObject.getOutputDir()
remoterundirq = appiondata.ApPathData(path=os.path.abspath(remoterundir))
clustq['clusterpath'] = remoterundirq
clustq['session'] = apDatabase.getSessionDataFromSessionId(jobObject.getExpId())
clustq['user'] = os.getlogin()
clustq['cluster'] = os.uname()[1]
clustq['clusterjobid'] = job
clustq['status'] = "Q"
clustq.insert()
return retValue
