def setRunDir(self): """ this function only runs if no rundir is defined at the command line """ if self.params['rundir'] is None: if ('sessionname' in self.params and self.params['sessionname'] is not None ): # command line users may use sessionname rather than expId sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname']) self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']]) else: if ('expId' in self.params and self.params['expId']): # expId should always be included from appionwrapper derived appionscript sessiondata = apDatabase.getSessionDataFromSessionId(self.params['expId']) self.params['rundir'] = self.getDefaultBaseAppionDir(sessiondata,[self.processdirname,self.params['runname']]) # The rest should not be needed with appionwrapper format from appionlib import apStack if ( self.params['rundir'] is None and 'reconid' in self.params and self.params['reconid'] is not None ): self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid'], msg=False) if ( self.params['rundir'] is None and 'stackid' in self.params and self.params['stackid'] is not None ): #auto set the run directory stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) path = os.path.abspath(stackdata['path']['path']) path = os.path.dirname(path) path = os.path.dirname(path) self.params['rundir'] = os.path.join(path, self.processdirname, self.params['runname']) self.params['outdir'] = self.params['rundir']
def checkGlobalConflicts(self): """ put in any conflicting parameters """ appionScript.AppionScript.checkGlobalConflicts(self) if self.params['runname'] is None: apDisplay.printError("please enter a runname, example: 'runname=run1'") if self.params['runname'] == 'templates': apDisplay.printError("templates is a reserved runname, please use another runname") if self.params['runname'] == 'models': apDisplay.printError("models is a reserved runname, please use another runname") if self.params['rundir']is not None and self.params['runname'] != os.path.basename(self.params['rundir']): apDisplay.printError("runname and rundir basename are different: " +self.params['runname']+" vs. "+os.path.basename(self.params['rundir'])) if self.params['mrcnames'] and self.params['preset']: apDisplay.printError("preset can not be specified if particular images have been specified") if (self.params['sessionname'] is None and self.params['expid'] is None) and self.params['mrcnames'] is None: apDisplay.printError("please specify an mrc name or session") if self.params['sessionname'] is None and self.params['expid'] is not None: self.params['sessionname'] = apDatabase.getSessionDataFromSessionId(self.params['expid'])['name'] if self.params['sessionname'] is not None and self.params['projectid'] is not None: ### Check that project and images are in sync imgproject = apProject.getProjectIdFromSessionName(self.params['sessionname']) if imgproject and imgproject != self.params['projectid']: apDisplay.printError("project id and session do not correlate")
def undoCTFCorrect(self, newstackroot): # At this point, the stack needs to be remade un-ctf-corrected, and possibly normalized and/or inverted apDisplay.printWarning('Relion needs a stack without ctf correction. A new stack is being made....') # Gather all the makestack parameters totalpart = self.originalStackData.numpart numpart = totalpart if not self.params['last'] else min(self.params['last'],totalpart) stackpathname = os.path.basename( self.originalStackData.path ) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir,'start.hed') presetname = self.originalStackData.preset # binning is combination of the original binning of the stack and the preparation binnning bin = self.originalStackData.bin * self.params['bin'] unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin lowpasstext = setArgText( 'lowpass', ( self.params['lowpass'], self.originalStackData.lowpass ), False) highpasstext = setArgText( 'highpass', ( self.params['highpass'], self.originalStackData.highpass ), True) partlimittext = setArgText('partlimit',(numpart,),False) xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'],), True) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid reversetext = '--reverse' if self.originalStackData.reverse else '' defoctext = '--defocpair' if self.originalStackData.defocpair else '' inverttext = '--no-invert' if not self.invert else '' # Build the makestack2 command cmd = "makestack2.py " cmd += (" --single=%s --fromstackid=%d %s %s %s %s %s %s "% (os.path.basename(newstackimagicfile),selectionid,stackid,lowpasstext, highpasstext,partlimittext,reversetext,defoctext,inverttext)) cmd += (" --normalized %s --boxsize=%d --bin=%d "% (xmipp_normtext,unbinnedboxsize,bin)) cmd += (" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "% (stackpathname,stackid,projectid)) cmd += (" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d "% (presetname,newstackrunname,newstackrundir,sessionname,sessionid)) cmd += (" --no-wait --no-commit --no-continue --jobtype=makestack2 ") # Run the command logfilepath = os.path.join(newstackrundir,'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd,logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Make sure our new stack params reflects the changes made # Use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] self.stack['file'] = newstackroot+'.hed' # Clean up boxfiles = glob.glob("*.box") for boxfile in boxfiles: apFile.removeFile(boxfile)
def checkGlobalConflicts(self): """ put in any conflicting parameters """ appionScript.AppionScript.checkGlobalConflicts(self) if self.params['runname'] is None: apDisplay.printError( "please enter a runname, example: 'runname=run1'") if self.params['runname'] == 'templates': apDisplay.printError( "templates is a reserved runname, please use another runname") if self.params['runname'] == 'models': apDisplay.printError( "models is a reserved runname, please use another runname") if self.params['rundir'] is not None and self.params[ 'runname'] != os.path.basename(self.params['rundir']): apDisplay.printError( "runname and rundir basename are different: " + self.params['runname'] + " vs. " + os.path.basename(self.params['rundir'])) if self.params['mrcnames'] and self.params['preset']: apDisplay.printError( "preset can not be specified if particular images have been specified" ) if (self.params['sessionname'] is None and self.params['expid'] is None ) and self.params['mrcnames'] is None: apDisplay.printError("please specify an mrc name or session") if self.params['sessionname'] is None and self.params[ 'expid'] is not None: self.params[ 'sessionname'] = apDatabase.getSessionDataFromSessionId( self.params['expid'])['name'] if self.params['sessionname'] is not None and self.params[ 'projectid'] is not None: ### Check that project and images are in sync imgproject = apProject.getProjectIdFromSessionName( self.params['sessionname']) if imgproject and imgproject != self.params['projectid']: apDisplay.printError("project id and session do not correlate")
def convertToRefineStack(self): ''' The stack is remaked without ctf correction and inverted and normalized if needed ''' newstackroot = os.path.join(self.params['rundir'],os.path.basename(self.stack['file'])[:-4]) self.stack['phaseflipped'] = False self.stack['format'] = 'relion' #TODO: Where is this used? # If we just want the frealign param file, skip this function if self.params['paramonly'] is True: print 'newstackroot',newstackroot return # If we just need to normalize, run xmipp_normalize if self.normalize and not self.un_ctf_correct: extname,addformat = self.proc2dFormatConversion() outstack = os.path.join(self.params['rundir'], "start.%s" % extname) self.xmippNormStack(self.stack['file'], outstack) self.stack['file'] = outstack # If we don't need to un-ctf-correct, we are done if not self.un_ctf_correct: return # At this point, the stack needs to be remade un-ctf-corrected, and possibly normalized and/or inverted apDisplay.printWarning('Relion needs a stack without ctf correction. A new stack is being made....') # Gather all the makestack parameters totalpart = self.originalStackData.numpart numpart = totalpart if not self.params['last'] else min(self.params['last'],totalpart) stackpathname = os.path.basename( self.originalStackData.path ) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir,'start.hed') presetname = self.originalStackData.preset # binning is combination of the original binning of the stack and the preparation binnning bin = self.originalStackData.bin * self.params['bin'] unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin lowpasstext = setArgText( 'lowpass', ( self.params['lowpass'], self.originalStackData.lowpass ), False) highpasstext = setArgText( 'highpass', ( self.params['highpass'], self.originalStackData.highpass ), True) partlimittext = setArgText('partlimit',(numpart,),False) xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'],), True) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid reversetext = '--reverse' if self.originalStackData.reverse else '' defoctext = '--defocpair' if self.originalStackData.defocpair else '' inverttext = '--no-invert' if not self.invert else '' # Build the makestack2 command cmd = ''' makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s %s --normalized %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2 ''' % (os.path.basename(newstackimagicfile),stackid,lowpasstext,highpasstext,partlimittext,reversetext,defoctext,inverttext,xmipp_normtext,unbinnedboxsize,bin,stackpathname,stackid,projectid,presetname,newstackrunname,newstackrundir,sessionname,sessionid) # Run the command logfilepath = os.path.join(newstackrundir,'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd,logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Convert the imagic stack to an MRC stack format # Relion preferes an mrc stack with .mrcs extension #self.ImagicStackToMrcStack(newstackimagicfile) #os.rename(newstackimagicfile[:-4]+'.mrc',newstackroot+'.mrc') # Make sure our new stack params reflects the changes made # Use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] self.stack['file'] = newstackroot+'.hed' # Clean up rmfiles = glob.glob("*.box") for rmfile in rmfiles: apFile.removeFile(rmfile)
def convertToRefineStack(self): ''' The stack is remaked without ctf correction and without invertion (ccd) ''' newstackroot = os.path.join(self.params['rundir'], os.path.basename(self.stack['file'])[:-4]) if self.params['paramonly'] is True: print 'newstackroot', newstackroot self.setFrealignStack(newstackroot) return # copy existing refined stack if possible existing_refine_stack = apStack.getExistingRefineStack( self.stack['data'], 'frealign', False, self.params['last'], self.params['bin'], lowpass=self.params['lowpass'], highpass=self.params['highpass']) try: if existing_refine_stack: if existing_refine_stack != os.path.join( self.params['rundir'], newstackroot + '.mrc'): shutil.copyfile(existing_refine_stack, newstackroot + '.mrc') new_partorderfile = os.path.join(self.params['rundir'], 'stackpartorder.list') existing_partorderfile = os.path.join( os.path.dirname(existing_refine_stack), 'stackpartorder.list') # particle order list also need to be copied if not os.path.isfile(new_partorderfile) and os.path.isfile( existing_partorderfile): shutil.copyfile(existing_partorderfile, new_partorderfile) self.setFrealignStack(newstackroot) return except: # make one from scratch if there is error apDisplay.printMsg( 'Error copying existing refine stack, will regenerate') if self.no_ctf_correction: self.ImagicStackToFrealignMrcStack(self.stack['file']) self.setFrealignStack(newstackroot) return # stack need to be remade without ctf correction apDisplay.printWarning( 'Frealign needs a stack without ctf correction. A new stack is being made....' ) stackdata = self.stack['data'] madestackdata = self.getOriginalStackMadeByMakeStack() stackid = stackdata.dbid stackruns = apStack.getStackRunsFromStack(madestackdata) # To Do: need to handle combined stack stackrun = stackruns[0] stackpathname = os.path.basename(stackdata['path']['path']) totalpart = apStack.getNumberStackParticlesFromId(stackid) if not self.params['last']: numpart = totalpart else: numpart = min(self.params['last'], totalpart) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir, 'temp.hed') # use first particle image to get presetname oneparticle = apStack.getOneParticleFromStackId(stackid, particlenumber=1) preset = oneparticle['particle']['image']['preset'] if preset: presetname = preset['name'] else: presetname = 'manual' # use first stack run to get parameters paramdata = stackrun['stackParams'] # binning is combination of the original binning of the stack and the preparation binnning bin = paramdata['bin'] * self.params['bin'] unbinnedboxsize = self.stack['boxsize'] * paramdata['bin'] lowpasstext = self.setArgText( 'lowpass', (self.params['lowpass'], paramdata['lowpass']), False) highpasstext = self.setArgText( 'highpass', (self.params['highpass'], paramdata['highpass']), True) partlimittext = self.setArgText('partlimit', (numpart, ), False) #normalization normtext = '' if 'xmipp-norm' in paramdata.keys(): # myami-3.1 or before normtext = self.setArgText('xmipp-normalize', (paramdata['xmipp-norm'], ), True) normtext = '--normalized ' + normtext elif 'normalizemethod' in paramdata.keys(): # myami-3.2 or newer normtext = '--normalize-method=%s' % ( paramdata['normalizemethod'], ) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] # The assumption is that the image is from ice grid and digital camera (black particles on white background if 'reverse' in self.stackrunlogparams.keys(): reversetext = '--reverse' else: reversetext = '' if 'defocpair' in self.stackrunlogparams.keys(): defoctext = '--defocpair' else: defoctext = '' cmd = ''' makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s --no-invert %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2 ''' % (os.path.basename(newstackimagicfile), stackid, lowpasstext, highpasstext, partlimittext, reversetext, defoctext, normtext, unbinnedboxsize, bin, stackpathname, stackid, projectid, presetname, newstackrunname, newstackrundir, sessionname, sessionid) logfilepath = os.path.join(newstackrundir, 'frealignstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd, logfilepath) if returncode > 0: apDisplay.printError('Error in Frealign specific stack making') self.ImagicStackToFrealignMrcStack(newstackimagicfile) os.rename(newstackimagicfile[:-4] + '.mrc', newstackroot + '.mrc') self.setFrealignStack(newstackroot) # use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] #clean up rmfiles = glob.glob("*.box") for rmfile in rmfiles: apFile.removeFile(rmfile)
def runRelionPreprocess(self, newstackroot): ''' 1. Use stackIntoPicks.py to extract the particle locations from the selected stack. 2. Run makestack2.py without ctf correction or normalization using the stackIntoPicks result as the Particles run. 3. Run relion_preprocess with rescale and norm. Outputs .mrcs file. Neil: most of these steps could be done more generally ''' apDisplay.printWarning("Making a new stack from original images") # Build the stackIntoPicks command apDisplay.printMsg('Extracting the particle locations from your selected stack.') newstackrunname = self.params['runname']+"_particles" newstackrundir = self.params['rundir'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid sessionid = int(self.params['expid']) cmd = "stackIntoPicks.py " cmd += (" --stackid=%d --projectid=%d --runname=%s --rundir=%s "% (stackid,projectid,newstackrunname,newstackrundir)) cmd += (" --commit --expId=%d --jobtype=stackintopicks "% (sessionid)) # Run the command logfilepath = os.path.join(newstackrundir,'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd,logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Build the makestack2 command ''' This is the command we want to make a stack from a stackIntoPicks run makestack2.py --single=start.hed --selectionid=130 --invert --boxsize=320 --bin=2 --description="made from stackrun1 using stackintopicks" --runname=stack66 --rundir=/ami/data00/appion/zz07jul25b/stacks/stack66 --commit --preset=en --projectid=303 --session=zz07jul25b --no-rejects --no-wait --continue --expid=8556 --jobtype=makestack2 --ppn=1 --nodes=1 --walltime=240 --jobid=2016 ''' apDisplay.printMsg('Using selected stack particle locations to make a Relion ready stack....') # Get the ID of the stackIntoPicks run we just created #apParticle.getSelectionIdFromName(runname, sessionname) runq = appiondata.ApSelectionRunData() runq['name'] = newstackrunname runq['session'] = leginondata.SessionData.direct_query(self.params['expid']) rundatas = runq.query(results=1) print rundatas if rundatas: selectionid = rundatas[0].dbid else: apDisplay.printError("Error creating Relion ready stack. Could not find stackIntoPicks.py data in database.\n") # Gather all the makestack parameters totalpart = self.originalStackData.numpart numpart = totalpart if not self.params['last'] else min(self.params['last'],totalpart) stackpathname = os.path.basename( self.originalStackData.path ) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir,'start.hed') presetname = self.originalStackData.preset # binning is combination of the original binning of the stack and the preparation binnning bin = self.originalStackData.bin * self.params['bin'] unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin lowpasstext = setArgText( 'lowpass', ( self.params['lowpass'], self.originalStackData.lowpass ), False) highpasstext = setArgText( 'highpass', ( self.params['highpass'], self.originalStackData.highpass ), True) partlimittext = setArgText('partlimit',(numpart,),False) xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'],), True) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid reversetext = '--reverse' if self.originalStackData.reverse else '' defoctext = '--defocpair' if self.originalStackData.defocpair else '' inverttext = '--no-invert' if not self.invert else '' # Build the makestack2 command cmd = "makestack2.py " cmd += (" --single=%s --selectionid=%d %s --boxsize=%d --bin=%d "% (os.path.basename(newstackimagicfile),selectionid,inverttext, unbinnedboxsize,bin)) cmd += (" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "% (stackpathname,stackid,projectid)) cmd += (" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d "% (presetname,newstackrunname,newstackrundir,sessionname,sessionid)) cmd += " --no-wait --no-commit --no-continue --jobtype=makestack2 " # Run the command logfilepath = os.path.join(newstackrundir,'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd,logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Clean up boxfiles = glob.glob("*.box") for boxfile in boxfiles: apFile.removeFile(boxfile) # Make sure our new stack params reflects the changes made # Use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] self.stack['file'] = newstackroot+'.hed' # Run Relion pre-process command ''' Setup the follow relion preprocess command to normalize the new stack: relion_preprocess \ --o particles --norm --bg_radius 60 --white_dust -1 --black_dust -1 \ --operate_on /ami/data00/appion/zz07jul25b/stacks/stack63_no_xmipp_norm/start.hed ''' apDisplay.printMsg('Running Relion preprocessing to normalize your Relion Ready stack....') bg_radius = math.floor((self.stack['boxsize'] / 2) - 1) relioncmd = "relion_preprocess " relioncmd += " --o particles --norm --bg_radius %d "%(bg_radius) relioncmd += " --white_dust -1 --black_dust -1 --operate_on %s "%(newstackimagicfile) apParam.runCmd(relioncmd, package="Relion", verbose=True, showcmd=True) self.stack['file'] = 'particles.mrcs'
def undoCTFCorrect(self, newstackroot): # At this point, the stack needs to be remade un-ctf-corrected, and possibly normalized and/or inverted apDisplay.printWarning( 'Relion needs a stack without ctf correction. A new stack is being made....' ) # Gather all the makestack parameters totalpart = self.originalStackData.numpart numpart = totalpart if not self.params['last'] else min( self.params['last'], totalpart) stackpathname = os.path.basename(self.originalStackData.path) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir, 'start.hed') presetname = self.originalStackData.preset # binning is combination of the original binning of the stack and the preparation binnning bin = self.originalStackData.bin * self.params['bin'] unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin lowpasstext = setArgText( 'lowpass', (self.params['lowpass'], self.originalStackData.lowpass), False) highpasstext = setArgText( 'highpass', (self.params['highpass'], self.originalStackData.highpass), True) partlimittext = setArgText('partlimit', (numpart, ), False) xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'], ), True) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid reversetext = '--reverse' if self.originalStackData.reverse else '' defoctext = '--defocpair' if self.originalStackData.defocpair else '' inverttext = '--no-invert' if not self.invert else '' # Build the makestack2 command cmd = "makestack2.py " cmd += (" --single=%s --fromstackid=%d %s %s %s %s %s %s " % (os.path.basename(newstackimagicfile), selectionid, stackid, lowpasstext, highpasstext, partlimittext, reversetext, defoctext, inverttext)) cmd += (" --normalized %s --boxsize=%d --bin=%d " % (xmipp_normtext, unbinnedboxsize, bin)) cmd += ( " --description='Relion refinestack based on %s(id=%d)' --projectid=%d " % (stackpathname, stackid, projectid)) cmd += ( " --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d " % (presetname, newstackrunname, newstackrundir, sessionname, sessionid)) cmd += (" --no-wait --no-commit --no-continue --jobtype=makestack2 ") # Run the command logfilepath = os.path.join(newstackrundir, 'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd, logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Make sure our new stack params reflects the changes made # Use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] self.stack['file'] = newstackroot + '.hed' # Clean up boxfiles = glob.glob("*.box") for boxfile in boxfiles: apFile.removeFile(boxfile)
def runRelionPreprocess(self, newstackroot): ''' 1. Use stackIntoPicks.py to extract the particle locations from the selected stack. 2. Run makestack2.py without ctf correction or normalization using the stackIntoPicks result as the Particles run. 3. Run relion_preprocess with rescale and norm. Outputs .mrcs file. Neil: most of these steps could be done more generally ''' apDisplay.printWarning("Making a new stack from original images") # Build the stackIntoPicks command apDisplay.printMsg( 'Extracting the particle locations from your selected stack.') newstackrunname = self.params['runname'] + "_particles" newstackrundir = self.params['rundir'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid sessionid = int(self.params['expid']) cmd = "stackIntoPicks.py " cmd += (" --stackid=%d --projectid=%d --runname=%s --rundir=%s " % (stackid, projectid, newstackrunname, newstackrundir)) cmd += (" --commit --expId=%d --jobtype=stackintopicks " % (sessionid)) # Run the command logfilepath = os.path.join(newstackrundir, 'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd, logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Build the makestack2 command ''' This is the command we want to make a stack from a stackIntoPicks run makestack2.py --single=start.hed --selectionid=130 --invert --boxsize=320 --bin=2 --description="made from stackrun1 using stackintopicks" --runname=stack66 --rundir=/ami/data00/appion/zz07jul25b/stacks/stack66 --commit --preset=en --projectid=303 --session=zz07jul25b --no-rejects --no-wait --continue --expid=8556 --jobtype=makestack2 --ppn=1 --nodes=1 --walltime=240 --jobid=2016 ''' apDisplay.printMsg( 'Using selected stack particle locations to make a Relion ready stack....' ) # Get the ID of the stackIntoPicks run we just created #apParticle.getSelectionIdFromName(runname, sessionname) runq = appiondata.ApSelectionRunData() runq['name'] = newstackrunname runq['session'] = leginondata.SessionData.direct_query( self.params['expid']) rundatas = runq.query(results=1) print rundatas if rundatas: selectionid = rundatas[0].dbid else: apDisplay.printError( "Error creating Relion ready stack. Could not find stackIntoPicks.py data in database.\n" ) # Gather all the makestack parameters totalpart = self.originalStackData.numpart numpart = totalpart if not self.params['last'] else min( self.params['last'], totalpart) stackpathname = os.path.basename(self.originalStackData.path) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir, 'start.hed') presetname = self.originalStackData.preset # binning is combination of the original binning of the stack and the preparation binnning bin = self.originalStackData.bin * self.params['bin'] unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin lowpasstext = setArgText( 'lowpass', (self.params['lowpass'], self.originalStackData.lowpass), False) highpasstext = setArgText( 'highpass', (self.params['highpass'], self.originalStackData.highpass), True) partlimittext = setArgText('partlimit', (numpart, ), False) xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'], ), True) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid reversetext = '--reverse' if self.originalStackData.reverse else '' defoctext = '--defocpair' if self.originalStackData.defocpair else '' inverttext = '--no-invert' if not self.invert else '' # Build the makestack2 command cmd = "makestack2.py " cmd += (" --single=%s --selectionid=%d %s --boxsize=%d --bin=%d " % (os.path.basename(newstackimagicfile), selectionid, inverttext, unbinnedboxsize, bin)) cmd += ( " --description='Relion refinestack based on %s(id=%d)' --projectid=%d " % (stackpathname, stackid, projectid)) cmd += ( " --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d " % (presetname, newstackrunname, newstackrundir, sessionname, sessionid)) cmd += " --no-wait --no-commit --no-continue --jobtype=makestack2 " # Run the command logfilepath = os.path.join(newstackrundir, 'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd, logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Clean up boxfiles = glob.glob("*.box") for boxfile in boxfiles: apFile.removeFile(boxfile) # Make sure our new stack params reflects the changes made # Use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] self.stack['file'] = newstackroot + '.hed' # Run Relion pre-process command ''' Setup the follow relion preprocess command to normalize the new stack: relion_preprocess \ --o particles --norm --bg_radius 60 --white_dust -1 --black_dust -1 \ --operate_on /ami/data00/appion/zz07jul25b/stacks/stack63_no_xmipp_norm/start.hed ''' apDisplay.printMsg( 'Running Relion preprocessing to normalize your Relion Ready stack....' ) bg_radius = math.floor((self.stack['boxsize'] / 2) - 1) relioncmd = "relion_preprocess " relioncmd += " --o particles --norm --bg_radius %d " % (bg_radius) relioncmd += " --white_dust -1 --black_dust -1 --operate_on %s " % ( newstackimagicfile) apParam.runCmd(relioncmd, package="Relion", verbose=True, showcmd=True) self.stack['file'] = 'particles.mrcs'
def convertToRefineStack(self): ''' The stack is remaked without ctf correction and without invertion (ccd) ''' newstackroot = os.path.join(self.params['rundir'],os.path.basename(self.stack['file'])[:-4]) if self.params['paramonly'] is True: print 'newstackroot',newstackroot self.setFrealignStack(newstackroot) return # copy existing refined stack if possible existing_refine_stack = apStack.getExistingRefineStack(self.stack['data'],'frealign',False,self.params['last'],self.params['bin'],lowpass=self.params['lowpass'],highpass=self.params['highpass']) if existing_refine_stack: if existing_refine_stack != os.path.join(self.params['rundir'],newstackroot+'.mrc'): shutil.copyfile(existing_refine_stack,newstackroot+'.mrc') new_partorderfile = os.path.join(self.params['rundir'],'stackpartorder.list') existing_partorderfile = os.path.join(os.path.dirname(existing_refine_stack),'stackpartorder.list') # particle order list also need to be copied if not os.path.isfile(new_partorderfile) and os.path.isfile(existing_partorderfile): shutil.copyfile(existing_partorderfile,new_partorderfile) self.setFrealignStack(newstackroot) return if self.no_ctf_correction: self.ImagicStackToFrealignMrcStack(self.stack['file']) self.setFrealignStack(newstackroot) return # stack need to be remade without ctf correction apDisplay.printWarning('Frealign needs a stack without ctf correction. A new stack is being made....') stackdata = self.stack['data'] madestackdata = self.getOriginalStackMadeByMakeStack() stackid = stackdata.dbid stackruns = apStack.getStackRunsFromStack(madestackdata) # To Do: need to handle combined stack stackrun = stackruns[0] stackpathname = os.path.basename(stackdata['path']['path']) totalpart = apStack.getNumberStackParticlesFromId(stackid) if not self.params['last']: numpart = totalpart else: numpart = min(self.params['last'],totalpart) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir,'temp.hed') # use first particle image to get presetname oneparticle = apStack.getOneParticleFromStackId(stackid, particlenumber=1) preset =oneparticle['particle']['image']['preset'] if preset: presetname = preset['name'] else: presetname = 'manual' # use first stack run to get parameters paramdata = stackrun['stackParams'] # binning is combination of the original binning of the stack and the preparation binnning bin = paramdata['bin']*self.params['bin'] unbinnedboxsize = self.stack['boxsize']*paramdata['bin'] lowpasstext = self.setArgText('lowpass',(self.params['lowpass'],paramdata['lowpass']),False) highpasstext = self.setArgText('highpass',(self.params['highpass'],paramdata['highpass']),True) partlimittext = self.setArgText('partlimit',(numpart,),False) xmipp_normtext = self.setArgText('xmipp-normalize',(paramdata['xmipp-norm'],),True) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] # The assumption is that the image is from ice grid and digital camera (black particles on white background if 'reverse' in self.stackrunlogparams.keys(): reversetext = '--reverse' else: reversetext = '' if 'defocpair' in self.stackrunlogparams.keys(): defoctext = '--defocpair' else: defoctext = '' cmd = ''' makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s --no-invert --normalized %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2 ''' % (os.path.basename(newstackimagicfile),stackid,lowpasstext,highpasstext,partlimittext,reversetext,defoctext,xmipp_normtext,unbinnedboxsize,bin,stackpathname,stackid,projectid,presetname,newstackrunname,newstackrundir,sessionname,sessionid) logfilepath = os.path.join(newstackrundir,'frealignstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd,logfilepath) if returncode > 0: apDisplay.printError('Error in Frealign specific stack making') self.ImagicStackToFrealignMrcStack(newstackimagicfile) os.rename(newstackimagicfile[:-4]+'.mrc',newstackroot+'.mrc') self.setFrealignStack(newstackroot) # use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] #clean up rmfiles = glob.glob("*.box") for rmfile in rmfiles: apFile.removeFile(rmfile)
cursor.close() dbConnection.close() except MySQLdb.DatabaseError, e: sys.stderr.write("Warning: Failure determining project database: %s \n" % (e)) #if jobId is not set, assume there is no entry in ApAppionJobData for this run if not jobObject.getJobId(): ### insert a cluster job # TODO: what happens when this runs remotely??? rundir = jobObject.getRundir() pathq = appiondata.ApPathData(path=os.path.abspath(rundir)) clustq = appiondata.ApAppionJobData() clustq['path'] = pathq clustq['jobtype'] = jobObject.getJobType() clustq['name'] = jobObject.getJobName() remoterundir = jobObject.getOutputDir() remoterundirq = appiondata.ApPathData(path=os.path.abspath(remoterundir)) clustq['clusterpath'] = remoterundirq clustq['session'] = apDatabase.getSessionDataFromSessionId(jobObject.getExpId()) clustq['user'] = os.getlogin() clustq['cluster'] = os.uname()[1] clustq['clusterjobid'] = job clustq['status'] = "Q" clustq.insert() return retValue
def convertToRefineStack(self): ''' The stack is remaked without ctf correction and inverted and normalized if needed ''' newstackroot = os.path.join(self.params['rundir'], os.path.basename(self.stack['file'])[:-4]) self.stack['phaseflipped'] = False self.stack['format'] = 'relion' #TODO: Where is this used? # If we just want the frealign param file, skip this function if self.params['paramonly'] is True: print 'newstackroot', newstackroot return # If we just need to normalize, run xmipp_normalize if self.normalize and not self.un_ctf_correct: extname, addformat = self.proc2dFormatConversion() outstack = os.path.join(self.params['rundir'], "start.%s" % extname) self.xmippNormStack(self.stack['file'], outstack) self.stack['file'] = outstack # If we don't need to un-ctf-correct, we are done if not self.un_ctf_correct: return # At this point, the stack needs to be remade un-ctf-corrected, and possibly normalized and/or inverted apDisplay.printWarning( 'Relion needs a stack without ctf correction. A new stack is being made....' ) # Gather all the makestack parameters totalpart = self.originalStackData.numpart numpart = totalpart if not self.params['last'] else min( self.params['last'], totalpart) stackpathname = os.path.basename(self.originalStackData.path) newstackrunname = self.params['runname'] newstackrundir = self.params['rundir'] newstackimagicfile = os.path.join(newstackrundir, 'start.hed') presetname = self.originalStackData.preset # binning is combination of the original binning of the stack and the preparation binnning bin = self.originalStackData.bin * self.params['bin'] unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin lowpasstext = setArgText( 'lowpass', (self.params['lowpass'], self.originalStackData.lowpass), False) highpasstext = setArgText( 'highpass', (self.params['highpass'], self.originalStackData.highpass), True) partlimittext = setArgText('partlimit', (numpart, ), False) xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'], ), True) sessionid = int(self.params['expid']) sessiondata = apDatabase.getSessionDataFromSessionId(sessionid) sessionname = sessiondata['name'] projectid = self.params['projectid'] stackid = self.originalStackData.stackid reversetext = '--reverse' if self.originalStackData.reverse else '' defoctext = '--defocpair' if self.originalStackData.defocpair else '' inverttext = '--no-invert' if not self.invert else '' # Build the makestack2 command cmd = ''' makestack2.py --single=%s --fromstackid=%d %s %s %s %s %s %s --normalized %s --boxsize=%d --bin=%d --description="frealign refinestack based on %s(id=%d)" --projectid=%d --preset=%s --runname=%s --rundir=%s --no-wait --no-commit --no-continue --session=%s --expId=%d --jobtype=makestack2 ''' % (os.path.basename(newstackimagicfile), stackid, lowpasstext, highpasstext, partlimittext, reversetext, defoctext, inverttext, xmipp_normtext, unbinnedboxsize, bin, stackpathname, stackid, projectid, presetname, newstackrunname, newstackrundir, sessionname, sessionid) # Run the command logfilepath = os.path.join(newstackrundir, 'relionstackrun.log') returncode = self.runAppionScriptInSubprocess(cmd, logfilepath) if returncode > 0: apDisplay.printError('Error in Relion specific stack making') # Convert the imagic stack to an MRC stack format # Relion preferes an mrc stack with .mrcs extension #self.ImagicStackToMrcStack(newstackimagicfile) #os.rename(newstackimagicfile[:-4]+'.mrc',newstackroot+'.mrc') # Make sure our new stack params reflects the changes made # Use the same complex equation as in eman clip clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin']) self.stack['boxsize'] = clipsize / self.params['bin'] self.stack['apix'] = self.stack['apix'] * self.params['bin'] self.stack['file'] = newstackroot + '.hed' # Clean up rmfiles = glob.glob("*.box") for rmfile in rmfiles: apFile.removeFile(rmfile)
cursor.close() dbConnection.close() except MySQLdb.DatabaseError, e: sys.stderr.write("Warning: Failure determining project database: %s \n" % (e)) #if jobId is not set, assume there is no entry in ApAppionJobData for this run if not jobObject.getJobId(): ### insert a cluster job # TODO: what happens when this runs remotely??? rundir = jobObject.getRundir() pathq = appiondata.ApPathData(path=os.path.abspath(rundir)) clustq = appiondata.ApAppionJobData() clustq['path'] = pathq clustq['jobtype'] = jobObject.getJobType() clustq['name'] = jobObject.getJobName() remoterundir = jobObject.getOutputDir() remoterundirq = appiondata.ApPathData(path=os.path.abspath(remoterundir)) clustq['clusterpath'] = remoterundirq clustq['session'] = apDatabase.getSessionDataFromSessionId(jobObject.getExpId()) clustq['user'] = os.getlogin() clustq['cluster'] = os.uname()[1] clustq['clusterjobid'] = job clustq['status'] = "Q" clustq.insert() return retValue