def checkConflicts(self):
if self.params['sessionname'] is None:
apDisplay.printError("Enter a session name (e.g. --session=06mar12a)")
if self.params['pdbid'] is None and self.params['pdbfile'] is None:
apDisplay.printError("Specify a PDB id or PDB file")
if self.params['pdbid'] is not None and self.params['pdbfile'] is not None:
apDisplay.printError("Specify only one of PDB id or PDB file")
if self.params['pdbfile'] is not None and not os.path.isfile(self.params['pdbfile']):
apDisplay.printError("PDB file does not exist")
### PDB is case-insensitve, so make all caps for consistency
if self.params['pdbid'] is not None:
self.params['pdbid'] = self.params['pdbid'].upper()
if self.params['lowpass'] is None:
apDisplay.printError("Enter a lowpass value")
if self.params['apix'] is None:
apDisplay.printError("Specify a pixel size")
if self.params['lowpass'] < 1.5*self.params['apix']:
apDisplay.printWarning("Lowpass is less than pixelsize, overriding lp=1.5*apix")
self.params['lowpass'] = 1.5*self.params['apix']
if self.params['symmetry'] is None and self.params['pdbfile'] is not None:
apDisplay.printError("Please specify symmetry group")
elif self.params['symmetry'] is None:
apDisplay.printWarning("No symmetry specified using 'c1'")
self.params['symmetry'] = 'c1'
self.params['symdata'] = apSymmetry.findSymmetry(self.params['symmetry'])
self.sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
def createChimeraVolumeSnapshot(self,
volume,
iteration,
reference_number=1):
''' create chimera snapshot images of volume '''
if self.params['snapfilter']:
resolution = self.params['snapfilter']
else:
newfscfile = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.fsc" \
% (self.params['timestamp'], iteration, reference_number))
try:
resolution = apRecon.getResolutionFromGenericFSCFile(
newfscfile,
self.runparams['boxsize'],
self.runparams['apix'],
filtradius=3,
criterion=self.fscResCriterion)
except:
apDisplay.printWarning(
"Failed to get resolution from generic FSC file: " +
newfscfile)
resolution = 30
apDisplay.printWarning(
"Running Chimera Snapshot with resolution: %d " % resolution)
# TODO: need to work this out.
# symmetry was hard coded to 'c1'. why?
try:
symmetry = self.runparams['symmetry']
except Exception, e:
symmetry = self.runparams['symmetry'].split()[0]
symmetry = apSymmetry.findSymmetry(symmetry)
def checkConflicts(self):
if self.params['sessionname'] is None:
apDisplay.printError(
"Enter a session name (e.g. --session=06jul12a)")
if self.params['projectid'] is not None:
projid = apProject.getProjectIdFromSessionName(
self.params['sessionname'])
if projid != self.params['projectid']:
apDisplay.printError(
"Project ID and Session name do not match")
if self.params['description'] is None:
apDisplay.printError("Enter a description of the initial model")
if self.params['chimeraonly'] is True:
self.params['commit'] = False
if self.params['newbox'] is not None and self.params['newbox'] % 8 != 0:
apDisplay.printWarning("Box size is not a multiple of 8")
### program requires either a model id or density id or filename
checkvalue = int(self.params['file'] is not None)
checkvalue += int(self.params['oldmodelid'] is not None)
checkvalue += int(self.params['densityid'] is not None)
if checkvalue != 1:
apDisplay.printError(
"Either provide a modelid or densityid or file, but not more than one of them"
)
elif self.params[
'oldmodelid'] is not None and self.params['newbox'] is None:
apDisplay.printError(
"Please specify either a new boxsize or scale for your model")
elif self.params['densityid'] is not None:
self.getDensityParams()
elif self.params['oldmodelid'] is not None:
self.getModelParams()
elif self.params['file'] is not None:
if not os.path.isfile(self.params['file']):
apDisplay.printError("could not find file: " +
self.params['file'])
if self.params['file'][-4:] != ".mrc":
apDisplay.printError("uploadModel.py only supports MRC files")
self.params['file'] = os.path.abspath(self.params['file'])
else:
apDisplay.printError(
"Please provide either a --modelid=112 or --file=initmodel.mrc"
)
### required only if now model id is provided
if self.params['newapix'] is None:
apDisplay.printError("Enter the pixel size of the model")
if self.params['symmetry'] is None:
apSymmetry.printSymmetries()
apDisplay.printError("Enter a symmetry ID, e.g. --symm=19")
self.params['symdata'] = apSymmetry.findSymmetry(
self.params['symmetry'])
if self.params['res'] is None:
apDisplay.printError("Enter the resolution of the initial model")
self.params['oldbox'] = apVolume.getModelDimensions(
self.params['file'])
if self.params['newbox'] is None:
self.params['newbox'] = self.params['oldbox']
if self.params['oldapix'] is None:
self.params['oldapix'] = self.params['newapix']
def parseFileForRunParameters(self):
''' PACKAGE-SPECIFIC FILE PARSER: if the parameters were not pickled, parse protocols script to determine ML3D params '''
### parameters can be found in python protocols
xmipp_protocol_ml3d = apXmipp.importProtocolPythonFile("xmipp_protocol_ml3d", self.params['rundir'])
packageparams = {}
packageparams['InSelFile'] = xmipp_protocol_ml3d.InSelFile
packageparams['InitialReference'] = xmipp_protocol_ml3d.InitialReference
packageparams['WorkingDir'] = xmipp_protocol_ml3d.WorkingDir
packageparams['DoDeleteWorkingDir'] = xmipp_protocol_ml3d.DoDeleteWorkingDir
# packageparams['ProjectDir'] = xmipp_protocol_ml3d.ProjectDir
# packageparams['LogDir'] = xmipp_protocol_ml3d.LogDir
packageparams['DoMlf'] = xmipp_protocol_ml3d.DoMlf
packageparams['DoCorrectAmplitudes'] = xmipp_protocol_ml3d.DoCorrectAmplitudes
packageparams['InCtfDatFile'] = xmipp_protocol_ml3d.InCtfDatFile
packageparams['HighResLimit'] = xmipp_protocol_ml3d.HighResLimit
packageparams['ImagesArePhaseFlipped'] = xmipp_protocol_ml3d.ImagesArePhaseFlipped
packageparams['InitialMapIsAmplitudeCorrected'] = xmipp_protocol_ml3d.InitialMapIsAmplitudeCorrected
packageparams['SeedsAreAmplitudeCorrected'] = xmipp_protocol_ml3d.SeedsAreAmplitudeCorrected
packageparams['DoCorrectGreyScale'] = xmipp_protocol_ml3d.DoCorrectGreyScale
packageparams['ProjMatchSampling'] = xmipp_protocol_ml3d.ProjMatchSampling
packageparams['DoLowPassFilterReference'] = xmipp_protocol_ml3d.DoLowPassFilterReference
packageparams['LowPassFilter'] = xmipp_protocol_ml3d.LowPassFilter
packageparams['PixelSize'] = xmipp_protocol_ml3d.PixelSize
packageparams['DoGenerateSeeds'] = xmipp_protocol_ml3d.DoGenerateSeeds
packageparams['NumberOfReferences'] = xmipp_protocol_ml3d.NumberOfReferences
packageparams['DoJustRefine'] = xmipp_protocol_ml3d.DoJustRefine
packageparams['SeedsSelfile'] = xmipp_protocol_ml3d.SeedsSelfile
packageparams['DoML3DClassification'] = xmipp_protocol_ml3d.DoML3DClassification
packageparams['AngularSampling'] = xmipp_protocol_ml3d.AngularSampling
packageparams['NumberOfIterations'] = xmipp_protocol_ml3d.NumberOfIterations
packageparams['Symmetry'] = xmipp_protocol_ml3d.Symmetry
packageparams['DoNorm'] = xmipp_protocol_ml3d.DoNorm
packageparams['DoFourier'] = xmipp_protocol_ml3d.DoFourier
packageparams['RestartIter'] = xmipp_protocol_ml3d.RestartIter
packageparams['ExtraParamsMLrefine3D'] = xmipp_protocol_ml3d.ExtraParamsMLrefine3D
# packageparams['NumberOfThreads'] = xmipp_protocol_ml3d.NumberOfThreads
# packageparams['DoParallel'] = xmipp_protocol_ml3d.DoParallel
# packageparams['NumberOfMpiProcesses'] = xmipp_protocol_ml3d.NumberOfMpiProcesses
# packageparams['SystemFlavour'] = xmipp_protocol_ml3d.SystemFlavour
# packageparams['AnalysisScript'] = xmipp_protocol_ml3d.AnalysisScript
if xmipp_protocol_ml3d.Symmetry[:1] == "i":
sym = "icos"
else:
sym = xmipp_protocol_ml3d.Symmetry
sym = sym.split()[0]
### set global parameters
runparams = {}
runparams['symmetry'] = apSymmetry.findSymmetry(sym)
runparams['apix'] = packageparams['PixelSize']
runparams['angularSamplingRate'] = packageparams['AngularSampling']
runparams['NumberOfReferences'] = packageparams['NumberOfReferences']
runparams['numiter'] = packageparams['NumberOfIterations']
runparams['package_params'] = packageparams
runparams['remoterundir'] = os.path.abspath(xmipp_protocol_ml3d.ProjectDir)
return runparams
def checkConflicts(self):
if self.params['sessionname'] is None:
apDisplay.printError("Enter a session name (e.g. --session=06mar12a)")
if self.params['pdbid'] is None and self.params['pdbfile'] is None:
apDisplay.printError("Specify a PDB id or PDB file")
if self.params['pdbid'] is not None and self.params['pdbfile'] is not None:
apDisplay.printError("Specify only one of PDB id or PDB file")
if self.params['pdbfile'] is not None and not os.path.isfile(self.params['pdbfile']):
apDisplay.printError("PDB file does not exist")
### PDB is case-insensitve, so make all caps for consistency
if self.params['pdbid'] is not None:
self.params['pdbid'] = self.params['pdbid'].upper()
if self.params['lowpass'] is None:
apDisplay.printError("Enter a lowpass value")
if self.params['apix'] is None:
apDisplay.printError("Specify a pixel size")
if self.params['lowpass'] < 1.5*self.params['apix']:
apDisplay.printWarning("Lowpass is less than pixelsize, overriding lp=1.5*apix")
self.params['lowpass'] = 1.5*self.params['apix']
if self.params['symmetry'] is None and self.params['pdbfile'] is not None:
apDisplay.printError("Please specify symmetry group")
elif self.params['symmetry'] is None:
apDisplay.printWarning("No symmetry specified using 'c1'")
self.params['symmetry'] = 'c1'
self.params['symdata'] = apSymmetry.findSymmetry(self.params['symmetry'])
self.sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
def checkConflicts(self):
if self.params['sessionname'] is None:
apDisplay.printError("Enter a session name (e.g. --session=06mar12a)")
if self.params['emdbid'] is None:
apDisplay.printError("specify a emdb id")
if self.params['symmetry'] is None:
#apSymmetry.printSymmetries()
apDisplay.printError("Enter a symmetry group, e.g. --symm=c1")
self.params['symdata'] = apSymmetry.findSymmetry(self.params['symmetry'])
self.sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
def readParamsFromOptimiserFile(self, iternum):
"""
read the recon_it000_optimiser.star file to get iteration parameters,
such as symmetry and phase residual cutoff
"""
iterparams = {'iternum': iternum,}
#recondir = os.path.join(self.params['rundir'], "recon")
recondir = self.params['rundir']
optimiserfile = "recon_it%03d_optimiser.star"%(iternum)
optimiserfilepath = os.path.join(recondir, optimiserfile)
# Create a dictionary of parameters found in the optimiser file
paramdict = {}
### parse optimiser file to populate dictionary
f = starFile.StarFile( optimiserfilepath )
f.read()
dataBlock = f.getDataBlock("data_optimiser_general")
paramdict = dataBlock.getLabelDict()
header = f.getHeader()
# Parse the header for the command line options used to run relion
sheader = header.strip()
bits = sheader.split("--")
for bit in bits:
params = bit.split()
key = params[0]
if len(params) > 1 :
value = params[1]
paramdict[key] = value
else:
paramdict[key] = True
if 'ini_high' not in paramdict:
paramdict['ini_high'] = 0
iterparams['ctf'] = self.convertBool(paramdict['_rlnDoCorrectCtf']) if paramdict['_rlnDoCorrectCtf'] else False
iterparams['ctf_intact_first_peak'] = self.convertBool(paramdict['_rlnDoIgnoreCtfUntilFirstPeak']) if paramdict['_rlnDoIgnoreCtfUntilFirstPeak'] else False
iterparams['ctf_corrected_ref'] = self.convertBool(paramdict['_rlnRefsAreCtfCorrected']) if paramdict['_rlnRefsAreCtfCorrected'] else False
iterparams['offset_step'] = int(paramdict['offset_step'])
iterparams['auto_local_healpix_order'] = int(paramdict['_rlnAutoLocalSearchesHealpixOrder'])
iterparams['healpix_order'] = int(paramdict['healpix_order'])
iterparams['offset_range'] = int(paramdict['offset_range'])
iterparams['ini_high'] = int(paramdict['ini_high'])
iterparams['sym'] = paramdict['sym']
### convert symetry to Appion
symtext = apFrealign.convertFrealignSymToAppionSym(iterparams['sym'])
symmdata = apSymmetry.findSymmetry(symtext)
apDisplay.printMsg("Found symmetry %s with id %s"%(symmdata['eman_name'], symmdata.dbid))
iterparams['symmdata'] = symmdata
return iterparams
def readParamsFromOptimiserFile(self, iternum):
"""
read the recon_it000_optimiser.star file to get iteration parameters,
such as symmetry and phase residual cutoff
"""
iterparams = {'iternum': iternum,}
#recondir = os.path.join(self.params['rundir'], "recon")
recondir = self.params['rundir']
optimiserfile = "recon_it%03d_optimiser.star"%(iternum)
optimiserfilepath = os.path.join(recondir, optimiserfile)
# Create a dictionary of parameters found in the optimiser file
paramdict = {}
### parse optimiser file to populate dictionary
f = starFile.StarFile( optimiserfilepath )
f.read()
dataBlock = f.getDataBlock("data_optimiser_general")
paramdict = dataBlock.getLabelDict()
header = f.getHeader()
# Parse the header for the command line options used to run relion
sheader = header.strip()
bits = sheader.split("--")
for bit in bits:
params = bit.split()
key = params[0]
if len(params) > 1 :
value = params[1]
paramdict[key] = value
else:
paramdict[key] = True
if 'ini_high' not in paramdict:
paramdict['ini_high'] = 0
iterparams['ctf'] = self.convertBool(paramdict['_rlnDoCorrectCtf']) if paramdict['_rlnDoCorrectCtf'] else False
iterparams['ctf_intact_first_peak'] = self.convertBool(paramdict['_rlnDoIgnoreCtfUntilFirstPeak']) if paramdict['_rlnDoIgnoreCtfUntilFirstPeak'] else False
iterparams['ctf_corrected_ref'] = self.convertBool(paramdict['_rlnRefsAreCtfCorrected']) if paramdict['_rlnRefsAreCtfCorrected'] else False
iterparams['offset_step'] = int(paramdict['offset_step'])
iterparams['auto_local_healpix_order'] = int(paramdict['_rlnAutoLocalSearchesHealpixOrder'])
iterparams['healpix_order'] = int(paramdict['healpix_order'])
iterparams['offset_range'] = int(paramdict['offset_range'])
iterparams['ini_high'] = int(paramdict['ini_high'])
iterparams['sym'] = paramdict['sym']
### convert symetry to Appion
symtext = apFrealign.convertFrealignSymToAppionSym(iterparams['sym'])
symmdata = apSymmetry.findSymmetry(symtext)
apDisplay.printMsg("Found symmetry %s with id %s"%(symmdata['eman_name'], symmdata.dbid))
iterparams['symmdata'] = symmdata
return iterparams
def insert3dDensity(self):
apDisplay.printMsg("committing density to database")
symdata = apSymmetry.findSymmetry(self.params['sym'])
if not symdata:
apDisplay.printError("no symmetry associated with this id\n")
self.params['syminfo'] = symdata
modq = appiondata.Ap3dDensityData()
sessiondata = apDatabase.getSessionDataFromSessionName(
self.params['sessionname'])
modq['session'] = sessiondata
modq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
modq['name'] = self.params['name']
modq['resolution'] = self.params['res']
modq['symmetry'] = symdata
modq['pixelsize'] = self.params['apix']
modq['boxsize'] = self.params['box']
modq['description'] = self.params['description']
modq['lowpass'] = self.params['lp']
modq['highpass'] = self.params['hp']
modq['mask'] = self.params['mask']
modq['imask'] = self.params['imask']
if self.params['reconiterid'] is not None:
iterdata = appiondata.ApRefineIterData.direct_query(
self.params['reconiterid'])
if not iterdata:
apDisplay.printError(
"this iteration was not found in the database\n")
modq['refineIter'] = iterdata
if self.params['reconid'] is not None:
iterdata = appiondata.ApRefineIterData.direct_query(
self.params['reconid'])
if not iterdata:
apDisplay.printError(
"this iteration was not found in the database\n")
modq['refineIter'] = iterdata
### if ampfile specified
if self.params['ampfile'] is not None:
(ampdir, ampname) = os.path.split(self.params['ampfile'])
modq['ampPath'] = appiondata.ApPathData(
path=os.path.abspath(ampdir))
modq['ampName'] = ampname
modq['maxfilt'] = self.params['maxfilt']
modq['handflip'] = self.params['yflip']
modq['norm'] = self.params['norm']
modq['invert'] = self.params['invert']
modq['hidden'] = False
filepath = os.path.join(self.params['rundir'], self.params['name'])
modq['md5sum'] = apFile.md5sumfile(filepath)
if self.params['commit'] is True:
modq.insert()
else:
apDisplay.printWarning("not commiting model to database")
def checkConflicts(self):
if self.params["stackid"] is None:
apDisplay.printError("stack id was not defined")
if self.params["modelid"] is None:
apDisplay.printError("model id was not defined")
if self.params["runname"] is None:
apDisplay.printError("missing a reconstruction name")
if self.params["last"] is None:
self.params["last"] = apStack.getNumberStackParticlesFromId(self.params["stackid"])
self.boxsize = apStack.getStackBoxsize(self.params["stackid"], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
if self.params["reconstackid"] is not None:
reconboxsize = apStack.getStackBoxsize(self.params["reconstackid"], msg=False)
reconapix = apStack.getStackPixelSizeFromStackId(self.params["reconstackid"])
refinenumpart = apStack.getNumberStackParticlesFromId(self.params["stackid"])
reconnumpart = apStack.getNumberStackParticlesFromId(self.params["reconstackid"])
if reconboxsize != self.boxsize:
apDisplay.printError("Boxsize do not match for stacks")
if reconapix != self.apix:
apDisplay.printError("Pixelsize do not match for stacks")
if refinenumpart != reconnumpart:
apDisplay.printError("Number of particles do not match for stacks")
maxmask = math.floor(self.apix * (self.boxsize - 10) / 2.0)
if self.params["mask"] is None:
apDisplay.printWarning("mask was not defined, setting to boxsize: %d" % (maxmask))
self.params["mask"] = maxmask
if self.params["mask"] > maxmask:
apDisplay.printWarning("mask was too big, setting to boxsize: %d" % (maxmask))
self.params["mask"] = maxmask
if self.params["noctf"] is True:
apDisplay.printWarning("Using no CTF method")
self.params["wgh"] = -1.0
if self.params["nodes"] > 1 and self.params["cluster"] is False:
apDisplay.printError("cluster mode must be enabled to run on more than 1 node")
if self.params["ppn"] is None:
if self.params["cluster"] is True:
apDisplay.printError("you must define ppn for cluster mode")
self.params["ppn"] = apParam.getNumProcessors()
apDisplay.printMsg("Setting number of processors to %d" % (self.params["ppn"]))
if self.params["rpn"] is None:
self.params["rpn"] = self.params["ppn"]
self.params["nproc"] = self.params["nodes"] * self.params["rpn"]
### get the symmetry data
if self.params["sym"] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params["sym"])
self.params["symm_id"] = self.symmdata.dbid
self.params["symm_name"] = self.symmdata["eman_name"]
apDisplay.printMsg("Selected symmetry %s with id %s" % (self.symmdata["eman_name"], self.symmdata.dbid))
### set cs value
self.params["cs"] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['runname'] is None:
apDisplay.printError("missing a reconstruction name")
if self.params['last'] is None:
self.params['last'] = apStack.getNumberStackParticlesFromId(self.params['stackid'])
self.boxsize = apStack.getStackBoxsize(self.params['stackid'], msg=False)
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
if self.params['reconstackid'] is not None:
reconboxsize = apStack.getStackBoxsize(self.params['reconstackid'], msg=False)
reconapix = apStack.getStackPixelSizeFromStackId(self.params['reconstackid'])
refinenumpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
reconnumpart = apStack.getNumberStackParticlesFromId(self.params['reconstackid'])
if reconboxsize != self.boxsize:
apDisplay.printError("Boxsize do not match for stacks")
if reconapix != self.apix:
apDisplay.printError("Pixelsize do not match for stacks")
if refinenumpart != reconnumpart:
apDisplay.printError("Number of particles do not match for stacks")
maxmask = math.floor(self.apix*(self.boxsize-10)/2.0)
if self.params['mask'] is None:
apDisplay.printWarning("mask was not defined, setting to boxsize: %d"%(maxmask))
self.params['mask'] = maxmask
if self.params['mask'] > maxmask:
apDisplay.printWarning("mask was too big, setting to boxsize: %d"%(maxmask))
self.params['mask'] = maxmask
if self.params['noctf'] is True:
apDisplay.printWarning("Using no CTF method")
self.params['wgh'] = -1.0
if self.params['nodes'] > 1 and self.params['cluster'] is False:
apDisplay.printError("cluster mode must be enabled to run on more than 1 node")
if self.params['ppn'] is None:
if self.params['cluster'] is True:
apDisplay.printError("you must define ppn for cluster mode")
self.params['ppn'] = apParam.getNumProcessors()
apDisplay.printMsg("Setting number of processors to %d"%(self.params['ppn']))
if self.params['rpn'] is None:
self.params['rpn'] = self.params['ppn']
self.params['nproc'] = self.params['nodes']*self.params['rpn']
### get the symmetry data
if self.params['sym'] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params['sym'])
self.params['symm_id'] = self.symmdata.dbid
self.params['symm_name'] = self.symmdata['eman_name']
apDisplay.printMsg("Selected symmetry %s with id %s"%(self.symmdata['eman_name'], self.symmdata.dbid))
### set cs value
self.params['cs'] = apInstrument.getCsValueFromSession(self.getSessionData())
def checkConflicts(self):
### check the method
if self.params['method'] is None:
apDisplay.printError("Please enter the EMAN commonline method")
#if self.params['method'] != 'any' and self.params['numkeep'] is None:
# apDisplay.printError("Please enter the number of particles per view, e.g. --numkeep=120")
if self.params['description'] is None:
apDisplay.printError("Please enter a description")
### get the symmetry data
if self.params['symm'] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params['symm'])
self.params['symm_id'] = self.symmdata.dbid
self.params['symm_name'] = self.symmdata['eman_name']
apDisplay.printMsg(
"Selected symmetry %s with id %s" %
(self.symmdata['eman_name'], self.symmdata.dbid))
if self.params['method'] == 'oct' and self.params['symm_name'] != 'oct':
apDisplay.printError("startoct only works on octahedral symmetry")
if self.params[
'method'] == 'icos' and self.params['symm_name'] != 'icos':
apDisplay.printError(
"starticos only works on icosahedral symmetry")
if self.params[
'method'] == 'csym' and self.params['symm_name'][0] != 'c':
apDisplay.printError("startcsym only works on c symmetries")
if self.params['method'] == 'any' and self.params['symm_name'] != 'c1':
apDisplay.printError("startAny only works on c1 symmetry")
### check for missing and duplicate entries
if self.params['clusterid'] is None:
apDisplay.printError("Please provide --cluster-id")
### get the stack ID from the other IDs
self.clusterstackdata = appiondata.ApClusteringStackData.direct_query(
self.params['clusterid'])
self.alignstackdata = self.clusterstackdata['clusterrun']['alignstack']
self.params['stackid'] = self.alignstackdata['stack'].dbid
### check and make sure we got the stack id
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
### check that we only have one of include and exclude
if self.params['includelist'] is not None and self.params[
'excludelist'] is not None:
apDisplay.printError(
"both --include and --exclude were defined, only one is allowed"
)
def checkConflicts(self):
if self.params['sessionname'] is None:
apDisplay.printError(
"Enter a session name (e.g. --session=06mar12a)")
if self.params['emdbid'] is None:
apDisplay.printError("specify a emdb id")
if self.params['symmetry'] is None:
#apSymmetry.printSymmetries()
apDisplay.printError("Enter a symmetry group, e.g. --symm=c1")
self.params['symdata'] = apSymmetry.findSymmetry(
self.params['symmetry'])
self.sessiondata = apDatabase.getSessionDataFromSessionName(
self.params['sessionname'])
def checkConflicts(self):
if self.params['sessionname'] is None:
apDisplay.printError("Enter a session name (e.g. --session=06jul12a)")
if self.params['projectid'] is not None:
projid = apProject.getProjectIdFromSessionName(self.params['sessionname'])
if projid != self.params['projectid']:
apDisplay.printError("Project ID and Session name do not match")
if self.params['description'] is None:
apDisplay.printError("Enter a description of the initial model")
if self.params['chimeraonly'] is True:
self.params['commit'] = False
if self.params['newbox'] is not None and self.params['newbox'] % 8 != 0:
apDisplay.printWarning("Box size is not a multiple of 8")
### program requires either a model id or density id or filename
checkvalue = int(self.params['file'] is not None)
checkvalue += int(self.params['oldmodelid'] is not None)
checkvalue += int(self.params['densityid'] is not None)
if checkvalue != 1:
apDisplay.printError("Either provide a modelid or densityid or file, but not more than one of them")
elif self.params['oldmodelid'] is not None and self.params['newbox'] is None:
apDisplay.printError("Please specify either a new boxsize or scale for your model")
elif self.params['densityid'] is not None:
self.getDensityParams()
elif self.params['oldmodelid'] is not None:
self.getModelParams()
elif self.params['file'] is not None:
if not os.path.isfile(self.params['file']):
apDisplay.printError("could not find file: "+self.params['file'])
if self.params['file'][-4:] != ".mrc":
apDisplay.printError("uploadModel.py only supports MRC files")
self.params['file'] = os.path.abspath(self.params['file'])
else:
apDisplay.printError("Please provide either a --modelid=112 or --file=initmodel.mrc")
### required only if now model id is provided
if self.params['newapix'] is None:
apDisplay.printError("Enter the pixel size of the model")
if self.params['symmetry'] is None:
apSymmetry.printSymmetries()
apDisplay.printError("Enter a symmetry ID, e.g. --symm=19")
self.params['symdata'] = apSymmetry.findSymmetry(self.params['symmetry'])
if self.params['res'] is None:
apDisplay.printError("Enter the resolution of the initial model")
self.params['oldbox'] = apVolume.getModelDimensions(self.params['file'])
if self.params['newbox'] is None:
self.params['newbox'] = self.params['oldbox']
if self.params['oldapix'] is None:
self.params['oldapix'] = self.params['newapix']
def insert3dDensity(self):
apDisplay.printMsg("committing density to database")
symdata=apSymmetry.findSymmetry(self.params['sym'])
if not symdata:
apDisplay.printError("no symmetry associated with this id\n")
self.params['syminfo'] = symdata
modq=appiondata.Ap3dDensityData()
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
modq['session'] = sessiondata
modq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
modq['name'] = self.params['name']
modq['resolution'] = self.params['res']
modq['symmetry'] = symdata
modq['pixelsize'] = self.params['apix']
modq['boxsize'] = self.params['box']
modq['description'] = self.params['description']
modq['lowpass'] = self.params['lp']
modq['highpass'] = self.params['hp']
modq['mask'] = self.params['mask']
modq['imask'] = self.params['imask']
if self.params['reconiterid'] is not None:
iterdata = appiondata.ApHipIterData.direct_query(self.params['reconiterid'])
if not iterdata:
apDisplay.printError("this iteration was not found in the database\n")
modq['hipIter'] = iterdata
if self.params['reconid'] is not None:
iterdata = appiondata.ApHipIterData.direct_query(self.params['reconid'])
if not iterdata:
apDisplay.printError("this iteration was not found in the database\n")
modq['hipIter'] = iterdata
### if ampfile specified
if self.params['ampfile'] is not None:
(ampdir, ampname) = os.path.split(self.params['ampfile'])
modq['ampPath'] = appiondata.ApPathData(path=os.path.abspath(ampdir))
modq['ampName'] = ampname
modq['maxfilt'] = self.params['maxfilt']
modq['handflip'] = self.params['yflip']
modq['norm'] = self.params['norm']
modq['invert'] = self.params['invert']
modq['hidden'] = False
filepath = os.path.join(self.params['rundir'], self.params['name'])
modq['md5sum'] = apFile.md5sumfile(filepath)
if self.params['commit'] is True:
modq.insert()
else:
apDisplay.printWarning("not commiting model to database")
def checkConflicts(self):
### check the method
if self.params['method'] is None:
apDisplay.printError("Please enter the EMAN commonline method")
#if self.params['method'] != 'any' and self.params['numkeep'] is None:
# apDisplay.printError("Please enter the number of particles per view, e.g. --numkeep=120")
if self.params['description'] is None:
apDisplay.printError("Please enter a description")
### get the symmetry data
if self.params['symm'] is None:
apDisplay.printError("Symmetry was not defined")
else:
self.symmdata = apSymmetry.findSymmetry(self.params['symm'])
self.params['symm_id'] = self.symmdata.dbid
self.params['symm_name'] = self.symmdata['eman_name']
apDisplay.printMsg("Selected symmetry %s with id %s"%(self.symmdata['eman_name'], self.symmdata.dbid))
if self.params['method'] == 'oct' and self.params['symm_name'] != 'oct':
apDisplay.printError("startoct only works on octahedral symmetry")
if self.params['method'] == 'icos' and self.params['symm_name'] != 'icos':
apDisplay.printError("starticos only works on icosahedral symmetry")
if self.params['method'] == 'csym' and self.params['symm_name'][0] != 'c':
apDisplay.printError("startcsym only works on c symmetries")
if self.params['method'] == 'any' and self.params['symm_name'] != 'c1':
apDisplay.printError("startAny only works on c1 symmetry")
### check for missing and duplicate entries
if self.params['clusterid'] is None:
apDisplay.printError("Please provide --cluster-id")
### get the stack ID from the other IDs
self.clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
self.alignstackdata = self.clusterstackdata['clusterrun']['alignstack']
self.params['stackid'] = self.alignstackdata['stack'].dbid
### check and make sure we got the stack id
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
### check that we only have one of include and exclude
if self.params['includelist'] is not None and self.params['excludelist'] is not None:
apDisplay.printError("both --include and --exclude were defined, only one is allowed")
def createChimeraVolumeSnapshot(self, volume, iteration, reference_number=1):
''' create chimera snapshot images of volume '''
if self.params['snapfilter']:
resolution = self.params['snapfilter']
else:
newfscfile = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.fsc" \
% (self.params['timestamp'], iteration, reference_number))
try:
resolution = apRecon.getResolutionFromGenericFSCFile(newfscfile, self.runparams['boxsize'], self.runparams['apix'],filtradius=3, criterion=self.fscResCriterion)
except:
apDisplay.printWarning("Failed to get resolution from generic FSC file: "+newfscfile)
resolution = 30
apDisplay.printWarning("Running Chimera Snapshot with resolution: %d " % resolution)
# TODO: need to work this out.
# symmetry was hard coded to 'c1'. why?
try:
symmetry = self.runparams['symmetry']
except Exception, e:
symmetry = self.runparams['symmetry'].split()[0]
symmetry = apSymmetry.findSymmetry( symmetry )
def parseLogFile(self):
# parse out the refine command from the .emanlog to get the parameters for each iteration
logfile = self.findEmanJobFile()
apDisplay.printMsg("parsing eman log file: "+logfile)
lines=open(logfile,'r')
self.iterationdatas = []
for line in lines:
# if read a refine line, get the parameters
line=line.rstrip()
if re.search("refine \d+ ", line):
emanparams=line.split(' ')
if emanparams[0] is "#":
emanparams.pop(0)
# get rid of first "refine"
emanparams.pop(0)
iteration=self.defineIteration()
iteration['num']=emanparams[0]
iteration['sym']=''
for p in emanparams:
elements=p.strip().split('=')
if elements[0]=='ang':
iteration['ang']=float(elements[1])
elif elements[0]=='mask':
iteration['mask']=int(float(elements[1]))
elif elements[0]=='imask':
iteration['imask']=int(float(elements[1]))
elif elements[0]=='pad':
iteration['pad']=int(float(elements[1]))
elif elements[0]=='sym':
iteration['sym'] = apSymmetry.findSymmetry(elements[1])
elif elements[0]=='maxshift':
iteration['maxshift']=int(float(elements[1]))
elif elements[0]=='hard':
iteration['hard']=int(float(elements[1]))
elif elements[0]=='classkeep':
iteration['classkeep']=float(elements[1].strip())
elif elements[0]=='classiter':
iteration['classiter']=int(float(elements[1]))
elif elements[0]=='filt3d':
iteration['filt3d']=int(float(elements[1]))
elif elements[0]=='shrink':
iteration['shrink']=int(float(elements[1]))
elif elements[0]=='euler2':
iteration['euler2']=int(float(elements[1]))
elif elements[0]=='xfiles':
## trying to extract "xfiles" as entered into emanJobGen.php
values = elements[1]
apix,mass,alito = values.split(',')
iteration['xfiles']=float(mass)
elif elements[0]=='amask1':
iteration['amask1']=float(elements[1])
elif elements[0]=='amask2':
iteration['amask2']=float(elements[1])
elif elements[0]=='amask3':
iteration['amask3']=float(elements[1])
elif elements[0]=='median':
iteration['median']=True
elif elements[0]=='phasecls':
iteration['phasecls']=True
elif elements[0]=='fscls':
iteration['fscls']=True
elif elements[0]=='refine':
iteration['refine']=True
elif elements[0]=='goodbad':
iteration['goodbad']=True
elif elements[0]=='perturb':
iteration['perturb']=True
self.iterationdatas.append(iteration)
if re.search("coran_for_cls.py \d+ ", line):
self.params['package']='EMAN/SpiCoran'
if re.search("msgPassing_subClassification.py \d+ ", line):
self.params['package']='EMAN/MsgP'
apDisplay.printColor("Found "+str(len(self.iterationdatas))+" iterations", "green")
lines.close()
def readParamsFromCombineFile(self, iternum):
"""
read the combine file to get iteration parameters,
such as symmetry and phase residual cutoff
"""
iterparams = {
'iternum': iternum,
}
recondir = os.path.join(self.params['rundir'], "recon")
combinefile = "iter%03d/frealign.iter%03d.proc%03d.sh" % (
iternum, iternum, iternum)
conbinefilepath = os.path.join(recondir, combinefile)
apDisplay.printMsg("Reading combine file %s for parameters" %
(conbinefilepath))
f = open(conbinefilepath, "r")
### locate start
for line in f:
if line.startswith("### START FREALIGN ###"):
break
### parse file
cards = []
for line in f:
cards.append(line.strip())
if line.startswith("EOF"):
break
f.close()
### get lots of info from card #1
data = cards[1].split(",")
#print data
iterparams['cform'] = data[0]
iterparams['iflag'] = int(data[1])
iterparams['fmag'] = self.convertBool(data[2])
iterparams['fdef'] = self.convertBool(data[3])
iterparams['fastig'] = self.convertBool(data[4])
iterparams['fpart'] = self.convertBool(data[5])
iterparams['iewald'] = float(data[6])
iterparams['fbeaut'] = self.convertBool(data[7])
iterparams['fcref'] = self.convertBool(data[8])
iterparams['fbfact'] = self.convertBool(data[9])
iterparams['fmatch'] = self.convertBool(data[10])
iterparams['ifsc'] = self.convertBool(data[11])
iterparams['fstat'] = self.convertBool(data[12])
iterparams['iblow'] = int(data[13])
### get lots of info from card #2
data = cards[2].split(",")
#print data
iterparams['mask'] = float(data[0])
iterparams['imask'] = float(data[1])
iterparams['wgh'] = float(data[3])
iterparams['xstd'] = float(data[4])
iterparams['pbc'] = float(data[5])
iterparams['boff'] = float(data[6])
iterparams['dang'] = float(data[7]) ### only used for recon search
iterparams['itmax'] = int(data[8])
iterparams['ipmax'] = int(data[9])
### get symmetry info from card #5
apDisplay.printMsg("Found Frealign symmetry %s in card 5" %
(cards[5], ))
symtext = apFrealign.convertFrealignSymToAppionSym(cards[5])
print symtext
symmdata = apSymmetry.findSymmetry(symtext)
apDisplay.printMsg("Found symmetry %s with id %s" %
(symmdata['eman_name'], symmdata.dbid))
iterparams['symmdata'] = symmdata
### get threshold info from card #6
data = cards[6].split(",")
#print data
iterparams['target'] = float(data[2])
iterparams['thresh'] = float(data[3])
iterparams['cs'] = float(data[4])
### get limit info from card #7
data = cards[7].split(",")
#print data
iterparams['rrec'] = float(data[0])
iterparams['highpass'] = float(data[1])
iterparams['lowpass'] = float(data[2])
iterparams['rbfact'] = float(data[4])
#print iterparams
return iterparams
def readParamsFromCombineFile(self, iternum):
"""
read the combine file to get iteration parameters,
such as symmetry and phase residual cutoff
"""
iterparams = {'iternum': iternum,}
combinefile = "iter%03d/frealign.iter%03d.combine.sh"%(iternum, iternum)
f = open(combinefile, "r")
### locate start
for line in f:
if line.startswith("### START FREALIGN ###"):
break
### parse file
cards = []
for line in f:
cards.append(line.strip())
if line.startswith("EOF"):
break
f.close()
### get lots of info from card #1
data = cards[1].split(",")
#print data
iterparams['cform'] = data[0]
iterparams['iflag'] = int(data[1])
iterparams['fmag'] = self.cb(data[2])
iterparams['fdef'] = self.cb(data[3])
iterparams['fastig'] = self.cb(data[4])
iterparams['fpart'] = self.cb(data[5])
iterparams['iewald'] = float(data[6])
iterparams['fbeaut'] = self.cb(data[7])
iterparams['fcref'] = self.cb(data[8])
iterparams['fmatch'] = self.cb(data[9])
iterparams['ifsc'] = self.cb(data[10])
iterparams['fstat'] = self.cb(data[11])
iterparams['iblow'] = int(data[12])
### get lots of info from card #2
data = cards[2].split(",")
#print data
iterparams['mask'] = float(data[0])
iterparams['imask'] = float(data[1])
iterparams['wgh'] = float(data[3])
iterparams['xstd'] = float(data[4])
iterparams['pbc'] = float(data[5])
iterparams['boff'] = float(data[6])
iterparams['dang'] = float(data[7]) ### only used for recon search
iterparams['itmax'] = int(data[8])
iterparams['ipmax'] = int(data[9])
### get symmetry info from card #5
symmdata = apSymmetry.findSymmetry(cards[5])
apDisplay.printMsg("Found symmetry %s with id %s"%(symmdata['eman_name'], symmdata.dbid))
iterparams['symmdata'] = symmdata
### get threshold info from card #6
data = cards[6].split(",")
#print data
iterparams['target'] = float(data[2])
iterparams['thresh'] = float(data[3])
iterparams['cs'] = float(data[4])
### get limit info from card #7
data = cards[7].split(",")
#print data
iterparams['rrec'] = float(data[0])
iterparams['highpass'] = float(data[1])
iterparams['lowpass'] = float(data[2])
iterparams['rbfact'] = float(data[4])
#print iterparams
return iterparams
def start(self):
### start the outfile name
fileroot = os.path.splitext(self.params['filename'])[0]
fileroot += "-"+self.timestamp
filename = self.params['file'].split('/')[-1]
filepath = self.params['file'].strip(filename)
self.params['box'] = apVolume.getModelDimensions(self.params['file'])
if self.params['ampfile'] is not None:
### run amplitude correction
spifile = apVolume.MRCtoSPI(self.params['file'], self.params['rundir'])
tmpfile = apAmpCorrect.createAmpcorBatchFile(spifile, self.params)
apAmpCorrect.runAmpcor()
### check if spider was successful
fileroot += ".amp"
if not os.path.isfile(tmpfile) :
apDisplay.printError("amplitude correction failed")
### convert amplitude corrected file back to mrc
fileroot += ".amp"
a = spider.read(tmpfile)
outfile = os.path.join( self.params['rundir'], "ampl-fix.mrc" )
mrc.write(a, outfile)
curfile = outfile
elif self.params['bfactor'] is True:
outfile = os.path.join( self.params['rundir'], "bfactor-fix.mrc" )
outfile = apAmpCorrect.applyBfactor(self.params['file'], fscfile=self.params['fscfile'], apix=self.params['apix'], mass=self.params['mass'], outfile=outfile)
curfile = outfile
elif self.params['lrdw'] is True:
lrdwcmd = "s_diffmap2 %s %s %s %s"%(self.params['filepath'],self.params['filename'],self.params['ampx'],self.params['resol'])
proc = subprocess.Popen(lrdwcmd)
print lrdwcmd
else :
### just run proc3d
curfile = self.params['file']
emancmd = "proc3d "+self.params['file']+" "
emancmd = "proc3d "+curfile+" "
if self.params['median'] is not None:
data = mrc.read(curfile)
data = ndimage.median_filter(data, size=self.params['median'])
mrc.write(data, curfile)
emancmd+="apix=%s " %self.params['apix']
if self.params['lp'] is not None:
fileroot += (".lp%d" % ( int(self.params['lp']), ))
emancmd += "lp=%d " %self.params['lp']
if self.params['yflip'] is True:
fileroot += ".yflip"
emancmd +="yflip "
if self.params['invert'] is True:
fileroot += ".inv"
emancmd +="mult=-1 "
if self.params['viper'] is True:
fileroot += ".vip"
emancmd +="icos5fTo2f "
if self.params['mask'] is not None:
# convert ang to pixels
maskpix=int(self.params['mask']/self.params['apix'])
fileroot += (".m%d" % ( int(self.params['mask']), ))
emancmd += "mask=%d " %maskpix
self.params['mask'] = maskpix
if self.params['imask'] is not None:
# convert ang to pixels
maskpix=int(self.params['imask']/self.params['apix'])
fileroot += (".im%d" % ( int(self.params['imask']), ))
emancmd += "imask=%d " %maskpix
self.params['imask'] = maskpix
if self.params['norm'] is True:
fileroot += ".norm"
emancmd += "norm=0,1 "
### add output filename to emancmd string
fileroot += ".mrc"
self.params['name'] = fileroot
outfile = os.path.join(self.params['rundir'], fileroot)
emancmd = re.sub(" apix=",(" %s apix=" % outfile), emancmd)
apEMAN.executeEmanCmd(emancmd)
if self.params['description'] is None:
self.params['description'] = "Volume from recon with amplitude adjustment"
### clean up files created during amp correction
if self.params['ampfile'] is not None:
apFile.removeFile(spifile)
apFile.removeFile(tmpfile)
if self.params['commit'] is True:
symdata = apSymmetry.findSymmetry(self.params['sym'])
print symdata
symmetry = symdata['eman_name']
self.params['reconid'] = self.params['reconiterid']
self.insert3dDensity()
### render chimera images of model
apChimera.filterAndChimera(outfile, res=self.params['res'], apix=self.params['apix'], box=self.params['box'],
chimtype='snapshot', mass=self.params['mass'], contour=self.params['contour'],
zoom=self.params['zoom'], sym=symmetry)
def start(self):
self.params['outputstack'] = os.path.join(self.params['rundir'],
self.params['stackname'])
particles, self.params['refineiter'] = getParticleInfo(
self.params['reconid'], self.params['iter'])
stackdata = particles[0]['particle']['stack']
stack = os.path.join(stackdata['path']['path'], stackdata['name'])
classes, cstats = determineClasses(particles)
rejectlst = []
if self.params['sigma'] is not None:
cutoff = cstats[
'meanquality'] + self.params['sigma'] * cstats['stdquality']
apDisplay.printMsg("Cutoff = " + str(cutoff))
rejectlst = self.removePtclsByQualityFactor(
particles, rejectlst, cutoff)
if self.params['avgjump'] is not None:
rejectlst = self.removePtclsByJumps(particles, rejectlst)
if self.params['rejectlst']:
rejectlst = removePtclsByLst(rejectlst, self.params)
classkeys = classes.keys()
classkeys.sort()
classnum = 0
totalptcls = 0
keepfile = open('keep.lst', 'w')
keepfile.write('#LST\n')
reject = open('reject.lst', 'w')
reject.write('#LST\n')
apDisplay.printMsg("Processing classes")
#loop through classes
for key in classkeys:
# file to hold particles of this class
clsfile = open('clstmp.lst', 'w')
clsfile.write('#LST\n')
classnum += 1
if classnum % 10 == 1:
apDisplay.printMsg(
str(classnum) + " of " + (str(len(classkeys))))
images = EMAN.EMData()
#loop through particles in class
nptcls = 0
for ptcl in classes[key]['particles']:
if ptcl['mirror']:
mirror = 1
else:
mirror = 0
rot = ptcl['euler3']
rot = rot * math.pi / 180
if ptcl['particle']['particleNumber'] not in rejectlst:
l = '%d\t%s\t%f,\t%f,%f,%f,%d\n' % (
ptcl['particle']['particleNumber'] - 1, stack,
ptcl['quality_factor'], rot, ptcl['shiftx'],
ptcl['shifty'], mirror)
keepfile.write(l)
clsfile.write(l)
totalptcls += 1
nptcls += 1
else:
reject.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' %
(ptcl['particle']['particleNumber'] - 1,
stack, ptcl['quality_factor'], rot,
ptcl['shiftx'], ptcl['shifty'], mirror))
#if ptcl['quality_factor']>cstats['meanquality']+3*cstats['stdquality']:
# high.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1,
# stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror))
clsfile.close()
if nptcls < 1:
continue
if self.params['skipavg'] is False:
makeClassAverages('clstmp.lst', self.params['outputstack'],
classes[key], self.params)
if self.params['eotest'] is True:
self.makeEvenOddClasses('clstmp.lst', classes[key])
apDisplay.printMsg("\n")
reject.close()
keepfile.close()
os.remove('clstmp.lst')
# make 3d density file if specified:
if self.params['make3d'] is not None:
self.params['make3d'] = os.path.basename(self.params['make3d'])
outfile = os.path.join(self.params['rundir'],
self.params['make3d'])
apEMAN.make3d(self.params['stackname'],
outfile,
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d %s %s mask=%d norm" %
(outfile, outfile, self.params['mask']))
if self.params['eotest'] is True:
apEMAN.make3d(self.params['oddstack'],
"odd.mrc",
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.make3d(self.params['evenstack'],
"even.mrc",
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d odd.mrc even.mrc fsc=fsc.eotest")
if os.path.exists(outfile):
# run rmeasure
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
box = apVolume.getModelDimensions(outfile)
apDisplay.printMsg('inserting density into database')
symdata = apSymmetry.findSymmetry(self.params['sym'])
if not symdata:
apDisplay.printError('no symmetry associated with this model')
modq = appiondata.Ap3dDensityData()
modq['session'] = apStack.getSessionDataFromStackId(
self.params['stackid'])
modq['name'] = self.params['make3d']
modq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
modq['boxsize'] = box
modq['mask'] = self.params['mask']
modq['pixelsize'] = apix
fscres = apRecon.getResolutionFromFSCFile('fsc.eotest',
box,
apix,
msg=True)
modq['resolution'] = fscres
modq['rmeasure'] = apRecon.runRMeasure(apix, outfile)
modq['md5sum'] = apFile.md5sumfile(outfile)
modq['maxjump'] = self.params['avgjump']
modq['sigma'] = self.params['sigma']
modq['hard'] = self.params['hard']
modq['symmetry'] = symdata
modq['refineIter'] = self.params['refineiter']
if self.params['commit'] is True:
modq.insert()
apChimera.filterAndChimera(outfile,
res=fscres,
apix=apix,
box=box,
chimtype='snapshot',
zoom=self.params['zoom'],
sym=self.params['sym'],
mass=self.params['mass'])
else:
apDisplay.printError(
'no 3d volume was generated - check the class averages:')
apDisplay.printError(self.params['stackname'])
apDisplay.printError(
'hard may be set too high, or avg euler jump set too low for the # of particles'
)
stackstr = str(stackdata.dbid)
reconstr = str(self.params['reconid'])
apDisplay.printColor(
"Make a new stack with only non-jumpers:\n" +
"subStack.py --projectid=" + str(self.params['projectid']) +
" -s " + stackstr + " \\\n " + " -k " +
os.path.join(self.params['rundir'], "keep.lst") + " \\\n " +
" -d 'recon " + reconstr + " sitters' -n sitters" + reconstr +
" -C ", "purple")
iterationParamsq['refineRun'] = self.refinerunq iterationParamsq['iteration'] = iteration iterationParamsq['resolution'] = resq if not self.params['euleronly']: iterationParamsq['rMeasure'] = self.getRMeasureData(iteration, reference_number) else: iterationParamsq['rMeasure'] = None iterationParamsq['mask'] = apRecon.getComponentFromVector(self.runparams['mask'], iteration-1) iterationParamsq['imask'] = apRecon.getComponentFromVector(self.runparams['imask'], iteration-1) iterationParamsq['alignmentInnerRadius'] = apRecon.getComponentFromVector(self.runparams['alignmentInnerRadius'], iteration-1) iterationParamsq['alignmentOuterRadius'] = apRecon.getComponentFromVector(self.runparams['alignmentOuterRadius'], iteration-1) try: iterationParamsq['symmetry'] = self.runparams['symmetry'] except Exception, e: symmetry = self.runparams['symmetry'].split()[0] iterationParamsq['symmetry'] = apSymmetry.findSymmetry( symmetry ) iterationParamsq['exemplar'] = False iterationParamsq['volumeDensity'] = "recon_%s_it%.3d_vol%.3d.mrc" % (self.params['timestamp'], iteration, reference_number) projections_and_avgs = "proj-avgs_%s_it%.3d_vol%.3d.img" \ % (self.params['timestamp'], iteration, reference_number) if os.path.exists(os.path.join(self.resultspath, projections_and_avgs)): iterationParamsq['refineClassAverages'] = projections_and_avgs refined_projections_and_avgs = "refined_proj-avgs_%s_it%.3d_vol%.3d.img" \ % (self.params['timestamp'], iteration, reference_number) if os.path.exists(os.path.join(self.resultspath, refined_projections_and_avgs)): iterationParamsq['postRefineClassAverages'] = refined_projections_and_avgs varianceAvgs = "variance_avgs_%s_it%.3d_vol%.3d.img" \ % (self.params['timestamp'], iteration, reference_number) if os.path.exists(os.path.join(self.resultspath, varianceAvgs)): iterationParamsq['classVariance'] = varianceAvgs
def parseFileForRunParameters(self):
''' PACKAGE-SPECIFIC FILE PARSER: if the parameters were not pickled, parse protocols script to determine projection-matching params '''
### parse out the refine command from the .emanlog to get the parameters for each iteration
#jobfile = self.findEmanJobFile()
jobfile = self.findEmanCommandFile()
apDisplay.printMsg("parsing eman log file: " + jobfile)
jobf = open(jobfile, 'r')
lines = jobf.readlines()
jobf.close()
itercount = 0
packageparams = self.setDefaultPackageParameters()
for i in range(len(lines)):
### if read a refine line, get the parameters
line = lines[i].rstrip()
if re.search("refine \d+ ", line):
itercount += 1
emanparams = line.split(' ')
if emanparams[0] is "#":
emanparams.pop(0)
### get rid of first "refine"
emanparams.pop(0)
packageparams['num'] += emanparams[0]
packageparams['sym'] = ''
for p in emanparams:
elements = p.strip().split('=')
if elements[0] == 'ang':
packageparams['ang'] += str(elements[1]) + " "
elif elements[0] == 'mask':
packageparams['mask'] += str(elements[1]) + " "
elif elements[0] == 'imask':
packageparams['imask'] += str(elements[1]) + " "
elif elements[0] == 'pad':
packageparams['pad'] += str(elements[1]) + " "
elif elements[0] == 'sym':
packageparams['sym'] += str(elements[1]) + " "
elif elements[0] == 'maxshift':
packageparams['maxshift'] += str(elements[1]) + " "
elif elements[0] == 'hard':
packageparams['hard'] += str(elements[1]) + " "
elif elements[0] == 'classkeep':
packageparams['classkeep'] += str(
elements[1].strip()) + " "
elif elements[0] == 'classiter':
packageparams['classiter'] += str(elements[1]) + " "
elif elements[0] == 'filt3d':
packageparams['filt3d'] += str(elements[1]) + " "
elif elements[0] == 'shrink':
packageparams['shrink'] += str(elements[1]) + " "
elif elements[0] == 'euler2':
packageparams['euler2'] += str(elements[1]) + " "
elif elements[0] == 'xfiles':
### trying to extract "xfiles" as entered into emanJobGen.php
values = elements[1]
apix, mass, alito = values.split(',')
packageparams['xfiles'] += str(mass) + " "
elif elements[0] == 'amask1':
packageparams['amask1'] += str(elements[1]) + " "
elif elements[0] == 'amask2':
packageparams['amask2'] += str(elements[1]) + " "
elif elements[0] == 'amask3':
packageparams['amask3'] += str(elements[1]) + " "
elif elements[0] == 'median':
packageparams['median'] += str(True) + " "
elif elements[0] == 'phasecls':
packageparams['phasecls'] += str(True) + " "
elif elements[0] == 'fscls':
packageparams['fscls'] += str(True) + " "
elif elements[0] == 'refine':
packageparams['refine'] += str(True) + " "
elif elements[0] == 'goodbad':
packageparams['goodbad'] += str(True) + " "
elif elements[0] == 'perturb':
packageparams['perturb'] += str(True) + " "
### Coran / MsgP performed 6 lines after standard refinement
if re.search("coran_for_cls.py", lines[i + 6]):
packageparams['package'] += 'EMAN/SpiCoran '
apDisplay.printMsg(
"correspondence analysis was performed on iteration %d to refine averages"
% itercount)
elif re.search("msgPassing_subClassification.py",
lines[i + 6]):
packageparams['package'] += 'EMAN/MsgP '
apDisplay.printMsg(
"Message-passing was performed on iteration %d to refine averages"
% itercount)
msgpassparams = lines[i + 6].split()
for p in msgpassparams:
elements = p.split('=')
if elements[0] == 'corCutOff':
packageparams['msgpasskeep'] += str(
elements[1]) + " "
elif elements[0] == 'minNumOfPtcls':
packageparams['msgpassminp'] += str(
elements[1]) + " "
else:
packageparams['package'] += 'EMAN '
apDisplay.printColor("Found %d iterations" % itercount, "green")
### set global parameters
runparams = {}
runparams['numiter'] = itercount
runparams['mask'] = packageparams['mask']
runparams['imask'] = packageparams['imask']
runparams['symmetry'] = apSymmetry.findSymmetry(packageparams['sym'])
runparams['angularSamplingRate'] = packageparams['ang']
runparams['package_params'] = packageparams
return runparams
iterationParamsq['rMeasure'] = None
iterationParamsq['mask'] = apRecon.getComponentFromVector(
self.runparams['mask'], iteration - 1)
iterationParamsq['imask'] = apRecon.getComponentFromVector(
self.runparams['imask'], iteration - 1)
iterationParamsq[
'alignmentInnerRadius'] = apRecon.getComponentFromVector(
self.runparams['alignmentInnerRadius'], iteration - 1)
iterationParamsq[
'alignmentOuterRadius'] = apRecon.getComponentFromVector(
self.runparams['alignmentOuterRadius'], iteration - 1)
try:
iterationParamsq['symmetry'] = self.runparams['symmetry']
except Exception, e:
symmetry = self.runparams['symmetry'].split()[0]
iterationParamsq['symmetry'] = apSymmetry.findSymmetry(symmetry)
iterationParamsq['exemplar'] = False
iterationParamsq['volumeDensity'] = "recon_%s_it%.3d_vol%.3d.mrc" % (
self.params['timestamp'], iteration, reference_number)
projections_and_avgs = "proj-avgs_%s_it%.3d_vol%.3d.img" \
% (self.params['timestamp'], iteration, reference_number)
if os.path.exists(os.path.join(self.resultspath,
projections_and_avgs)):
iterationParamsq['refineClassAverages'] = projections_and_avgs
refined_projections_and_avgs = "refined_proj-avgs_%s_it%.3d_vol%.3d.img" \
% (self.params['timestamp'], iteration, reference_number)
if os.path.exists(
os.path.join(self.resultspath, refined_projections_and_avgs)):
iterationParamsq[
'postRefineClassAverages'] = refined_projections_and_avgs
varianceAvgs = "variance_avgs_%s_it%.3d_vol%.3d.img" \
iterationParamsq["rMeasure"] = self.getRMeasureData(iteration, reference_number)
else:
iterationParamsq["rMeasure"] = None
iterationParamsq["mask"] = apRecon.getComponentFromVector(self.runparams["mask"], iteration - 1)
iterationParamsq["imask"] = apRecon.getComponentFromVector(self.runparams["imask"], iteration - 1)
iterationParamsq["alignmentInnerRadius"] = apRecon.getComponentFromVector(
self.runparams["alignmentInnerRadius"], iteration - 1
)
iterationParamsq["alignmentOuterRadius"] = apRecon.getComponentFromVector(
self.runparams["alignmentOuterRadius"], iteration - 1
)
try:
iterationParamsq["symmetry"] = self.runparams["symmetry"]
except Exception, e:
symmetry = self.runparams["symmetry"].split()[0]
iterationParamsq["symmetry"] = apSymmetry.findSymmetry(symmetry)
iterationParamsq["exemplar"] = False
iterationParamsq["volumeDensity"] = "recon_%s_it%.3d_vol%.3d.mrc" % (
self.params["timestamp"],
iteration,
reference_number,
)
projections_and_avgs = "proj-avgs_%s_it%.3d_vol%.3d.img" % (
self.params["timestamp"],
iteration,
reference_number,
)
if os.path.exists(os.path.join(self.resultspath, projections_and_avgs)):
iterationParamsq["refineClassAverages"] = projections_and_avgs
refined_projections_and_avgs = "refined_proj-avgs_%s_it%.3d_vol%.3d.img" % (
self.params["timestamp"],
def parseFileForRunParameters(self):
''' PACKAGE-SPECIFIC FILE PARSER: if the parameters were not pickled, parse protocols script to determine projection-matching params '''
### parse out the refine command from the .emanlog to get the parameters for each iteration
#jobfile = self.findEmanJobFile()
jobfile = self.findEmanCommandFile()
apDisplay.printMsg("parsing eman log file: "+jobfile)
jobf = open(jobfile,'r')
lines = jobf.readlines()
jobf.close()
itercount = 0
packageparams = self.setDefaultPackageParameters()
for i in range(len(lines)):
### if read a refine line, get the parameters
line = lines[i].rstrip()
if re.search("refine \d+ ", line):
itercount += 1
emanparams=line.split(' ')
if emanparams[0] is "#":
emanparams.pop(0)
### get rid of first "refine"
emanparams.pop(0)
packageparams['num'] += emanparams[0]
packageparams['sym']=''
for p in emanparams:
elements = p.strip().split('=')
if elements[0]=='ang':
packageparams['ang'] += str(elements[1])+" "
elif elements[0]=='mask':
packageparams['mask'] += str(elements[1])+" "
elif elements[0]=='imask':
packageparams['imask'] += str(elements[1])+" "
elif elements[0]=='pad':
packageparams['pad'] += str(elements[1])+" "
elif elements[0]=='sym':
packageparams['sym'] += str(elements[1])+" "
elif elements[0]=='maxshift':
packageparams['maxshift'] += str(elements[1])+" "
elif elements[0]=='hard':
packageparams['hard'] += str(elements[1])+" "
elif elements[0]=='classkeep':
packageparams['classkeep'] += str(elements[1].strip())+" "
elif elements[0]=='classiter':
packageparams['classiter'] += str(elements[1])+" "
elif elements[0]=='filt3d':
packageparams['filt3d'] += str(elements[1])+" "
elif elements[0]=='shrink':
packageparams['shrink'] += str(elements[1])+" "
elif elements[0]=='euler2':
packageparams['euler2'] += str(elements[1])+" "
elif elements[0]=='xfiles':
### trying to extract "xfiles" as entered into emanJobGen.php
values = elements[1]
apix,mass,alito = values.split(',')
packageparams['xfiles'] += str(mass)+" "
elif elements[0]=='amask1':
packageparams['amask1'] += str(elements[1])+" "
elif elements[0]=='amask2':
packageparams['amask2'] += str(elements[1])+" "
elif elements[0]=='amask3':
packageparams['amask3'] += str(elements[1])+" "
elif elements[0]=='median':
packageparams['median'] += str(True)+" "
elif elements[0]=='phasecls':
packageparams['phasecls'] += str(True)+" "
elif elements[0]=='fscls':
packageparams['fscls'] += str(True)+" "
elif elements[0]=='refine':
packageparams['refine'] += str(True)+" "
elif elements[0]=='goodbad':
packageparams['goodbad'] += str(True)+" "
elif elements[0]=='perturb':
packageparams['perturb'] += str(True)+" "
### Coran / MsgP performed 6 lines after standard refinement
if re.search("coran_for_cls.py", lines[i+6]):
packageparams['package'] += 'EMAN/SpiCoran '
apDisplay.printMsg("correspondence analysis was performed on iteration %d to refine averages" % itercount)
elif re.search("msgPassing_subClassification.py", lines[i+6]):
packageparams['package'] += 'EMAN/MsgP '
apDisplay.printMsg("Message-passing was performed on iteration %d to refine averages" % itercount)
msgpassparams = lines[i+6].split()
for p in msgpassparams:
elements = p.split('=')
if elements[0]=='corCutOff':
packageparams['msgpasskeep'] += str(elements[1])+" "
elif elements[0]=='minNumOfPtcls':
packageparams['msgpassminp'] += str(elements[1])+" "
else:
packageparams['package'] += 'EMAN '
apDisplay.printColor("Found %d iterations" % itercount, "green")
### set global parameters
runparams = {}
runparams['numiter'] = itercount
runparams['mask'] = packageparams['mask']
runparams['imask'] = packageparams['imask']
runparams['symmetry'] = apSymmetry.findSymmetry(packageparams['sym'])
runparams['angularSamplingRate'] = packageparams['ang']
runparams['package_params'] = packageparams
return runparams
def parseFileForRunParameters(self):
''' PACKAGE-SPECIFIC FILE PARSER: if the parameters were not pickled, parse protocols script to determine projection-matching params '''
### parameters can be found in python protocols
xmipp_protocol_projmatch = apXmipp.importProtocolPythonFile(
"xmipp_protocol_projmatch", self.params['rundir'])
packageparams = {}
packageparams[
'NumberofIterations'] = xmipp_protocol_projmatch.NumberofIterations
packageparams['MaskFileName'] = xmipp_protocol_projmatch.MaskFileName
packageparams['MaskRadius'] = xmipp_protocol_projmatch.MaskRadius
packageparams['InnerRadius'] = xmipp_protocol_projmatch.InnerRadius
packageparams['OuterRadius'] = xmipp_protocol_projmatch.OuterRadius
packageparams['SymmetryGroup'] = xmipp_protocol_projmatch.SymmetryGroup
packageparams[
'FourierMaxFrequencyOfInterest'] = xmipp_protocol_projmatch.FourierMaxFrequencyOfInterest
packageparams['SelFileName'] = xmipp_protocol_projmatch.SelFileName
packageparams['DocFileName'] = xmipp_protocol_projmatch.DocFileName
packageparams[
'ReferenceFileName'] = xmipp_protocol_projmatch.ReferenceFileName
packageparams['WorkingDir'] = xmipp_protocol_projmatch.WorkingDir
# packageparams['DoDeleteWorkingDir'] = xmipp_protocol_projmatch.DoDeleteWorkingDir
# packageparams['ContinueAtIteration'] = xmipp_protocol_projmatch.ContinueAtIteration
packageparams['CleanUpFiles'] = xmipp_protocol_projmatch.CleanUpFiles
# packageparams['ProjectDir'] = xmipp_protocol_projmatch.ProjectDir
# packageparams['LogDir'] = xmipp_protocol_projmatch.LogDir
packageparams[
'DoCtfCorrection'] = xmipp_protocol_projmatch.DoCtfCorrection
packageparams['CTFDatName'] = xmipp_protocol_projmatch.CTFDatName
packageparams[
'DoAutoCtfGroup'] = xmipp_protocol_projmatch.DoAutoCtfGroup
packageparams[
'CtfGroupMaxDiff'] = xmipp_protocol_projmatch.CtfGroupMaxDiff
packageparams[
'CtfGroupMaxResol'] = xmipp_protocol_projmatch.CtfGroupMaxResol
# packageparams['SplitDefocusDocFile'] = xmipp_protocol_projmatch.SplitDefocusDocFile
packageparams['PaddingFactor'] = xmipp_protocol_projmatch.PaddingFactor
packageparams[
'WienerConstant'] = xmipp_protocol_projmatch.WienerConstant
packageparams[
'DataArePhaseFlipped'] = xmipp_protocol_projmatch.DataArePhaseFlipped
packageparams[
'ReferenceIsCtfCorrected'] = xmipp_protocol_projmatch.ReferenceIsCtfCorrected
packageparams['DoMask'] = xmipp_protocol_projmatch.DoMask
packageparams[
'DoSphericalMask'] = xmipp_protocol_projmatch.DoSphericalMask
# packageparams['DoProjectionMatching'] = xmipp_protocol_projmatch.DoProjectionMatching
# packageparams['DisplayProjectionMatching'] = xmipp_protocol_projmatch.DisplayProjectionMatching
# packageparams['AvailableMemory'] = xmipp_protocol_projmatch.AvailableMemory
packageparams[
'AngSamplingRateDeg'] = xmipp_protocol_projmatch.AngSamplingRateDeg
packageparams[
'MaxChangeInAngles'] = xmipp_protocol_projmatch.MaxChangeInAngles
packageparams[
'PerturbProjectionDirections'] = xmipp_protocol_projmatch.PerturbProjectionDirections
packageparams[
'MaxChangeOffset'] = xmipp_protocol_projmatch.MaxChangeOffset
packageparams['Search5DShift'] = xmipp_protocol_projmatch.Search5DShift
packageparams['Search5DStep'] = xmipp_protocol_projmatch.Search5DStep
packageparams[
'DoRetricSearchbyTiltAngle'] = xmipp_protocol_projmatch.DoRetricSearchbyTiltAngle
packageparams['Tilt0'] = xmipp_protocol_projmatch.Tilt0
packageparams['TiltF'] = xmipp_protocol_projmatch.TiltF
packageparams[
'SymmetryGroupNeighbourhood'] = xmipp_protocol_projmatch.SymmetryGroupNeighbourhood
packageparams['OnlyWinner'] = xmipp_protocol_projmatch.OnlyWinner
packageparams[
'MinimumCrossCorrelation'] = xmipp_protocol_projmatch.MinimumCrossCorrelation
packageparams[
'DiscardPercentage'] = xmipp_protocol_projmatch.DiscardPercentage
packageparams[
'ProjMatchingExtra'] = xmipp_protocol_projmatch.ProjMatchingExtra
packageparams['DoAlign2D'] = xmipp_protocol_projmatch.DoAlign2D
packageparams['Align2DIterNr'] = xmipp_protocol_projmatch.Align2DIterNr
packageparams[
'Align2dMaxChangeOffset'] = xmipp_protocol_projmatch.Align2dMaxChangeOffset
packageparams[
'Align2dMaxChangeRot'] = xmipp_protocol_projmatch.Align2dMaxChangeRot
# packageparams['DoReconstruction'] = xmipp_protocol_projmatch.DoReconstruction
# packageparams['DisplayReconstruction'] = xmipp_protocol_projmatch.DisplayReconstruction
packageparams[
'ReconstructionMethod'] = xmipp_protocol_projmatch.ReconstructionMethod
packageparams['ARTLambda'] = xmipp_protocol_projmatch.ARTLambda
packageparams[
'ARTReconstructionExtraCommand'] = xmipp_protocol_projmatch.ARTReconstructionExtraCommand
packageparams[
'WBPReconstructionExtraCommand'] = xmipp_protocol_projmatch.WBPReconstructionExtraCommand
packageparams[
'FourierReconstructionExtraCommand'] = xmipp_protocol_projmatch.FourierReconstructionExtraCommand
# packageparams['DoSplitReferenceImages'] = xmipp_protocol_projmatch.DoSplitReferenceImages
packageparams['ResolSam'] = xmipp_protocol_projmatch.ResolSam
# packageparams['DisplayResolution'] = xmipp_protocol_projmatch.DisplayResolution
packageparams[
'ConstantToAddToFiltration'] = xmipp_protocol_projmatch.ConstantToAddToFiltration
# packageparams['NumberOfThreads'] = xmipp_protocol_projmatch.NumberOfThreads
# packageparams['DoParallel'] = xmipp_protocol_projmatch.DoParallel
# packageparams['NumberOfMpiProcesses'] = xmipp_protocol_projmatch.NumberOfMpiProcesses
# packageparams['MpiJobSize'] = xmipp_protocol_projmatch.MpiJobSize
# packageparams['SystemFlavour'] = xmipp_protocol_projmatch.SystemFlavour
# packageparams['AnalysisScript'] = xmipp_protocol_projmatch.AnalysisScript
if xmipp_protocol_projmatch.SymmetryGroup[:1] == "i":
sym = "icos"
else:
sym = xmipp_protocol_projmatch.SymmetryGroup
sym = sym.split()[0]
### set global parameters
runparams = {}
runparams['numiter'] = packageparams['NumberofIterations']
# Mask should be in pixels of original stack. Xmipp returns pixels in terms of preped stack.
# so orig_pix = preped_pix * (orig_boxsize / preped_boxsize).
# This is needed in case the prepedatack was binned.
boxscale = self.runparams['original_boxsize'] / self.runparams[
'boxsize']
runparams['mask'] = packageparams['MaskRadius'] * boxscale
runparams[
'alignmentInnerRadius'] = packageparams['InnerRadius'] * boxscale
runparams[
'alignmentOuterRadius'] = packageparams['OuterRadius'] * boxscale
runparams['symmetry'] = apSymmetry.findSymmetry(sym)
runparams['angularSamplingRate'] = packageparams['AngSamplingRateDeg']
runparams['apix'] = packageparams['ResolSam']
runparams['package_params'] = packageparams
runparams['remoterundir'] = os.path.abspath(
xmipp_protocol_projmatch.ProjectDir)
runparams['rundir'] = self.params['rundir']
return runparams
def parseFileForRunParameters(self):
''' PACKAGE-SPECIFIC FILE PARSER: if the parameters were not pickled, parse protocols script to determine projection-matching params '''
### parameters can be found in python protocols
xmipp_protocol_projmatch = apXmipp.importProtocolPythonFile("xmipp_protocol_projmatch", self.params['rundir'])
packageparams = {}
packageparams['NumberofIterations'] = xmipp_protocol_projmatch.NumberofIterations
packageparams['MaskFileName'] = xmipp_protocol_projmatch.MaskFileName
packageparams['MaskRadius'] = xmipp_protocol_projmatch.MaskRadius
packageparams['InnerRadius'] = xmipp_protocol_projmatch.InnerRadius
packageparams['OuterRadius'] = xmipp_protocol_projmatch.OuterRadius
packageparams['SymmetryGroup'] = xmipp_protocol_projmatch.SymmetryGroup
packageparams['FourierMaxFrequencyOfInterest'] = xmipp_protocol_projmatch.FourierMaxFrequencyOfInterest
packageparams['SelFileName'] = xmipp_protocol_projmatch.SelFileName
packageparams['DocFileName'] = xmipp_protocol_projmatch.DocFileName
packageparams['ReferenceFileName'] = xmipp_protocol_projmatch.ReferenceFileName
packageparams['WorkingDir'] = xmipp_protocol_projmatch.WorkingDir
# packageparams['DoDeleteWorkingDir'] = xmipp_protocol_projmatch.DoDeleteWorkingDir
# packageparams['ContinueAtIteration'] = xmipp_protocol_projmatch.ContinueAtIteration
packageparams['CleanUpFiles'] = xmipp_protocol_projmatch.CleanUpFiles
# packageparams['ProjectDir'] = xmipp_protocol_projmatch.ProjectDir
# packageparams['LogDir'] = xmipp_protocol_projmatch.LogDir
packageparams['DoCtfCorrection'] = xmipp_protocol_projmatch.DoCtfCorrection
packageparams['CTFDatName'] = xmipp_protocol_projmatch.CTFDatName
packageparams['DoAutoCtfGroup'] = xmipp_protocol_projmatch.DoAutoCtfGroup
packageparams['CtfGroupMaxDiff'] = xmipp_protocol_projmatch.CtfGroupMaxDiff
packageparams['CtfGroupMaxResol'] = xmipp_protocol_projmatch.CtfGroupMaxResol
# packageparams['SplitDefocusDocFile'] = xmipp_protocol_projmatch.SplitDefocusDocFile
packageparams['PaddingFactor'] = xmipp_protocol_projmatch.PaddingFactor
packageparams['WienerConstant'] = xmipp_protocol_projmatch.WienerConstant
packageparams['DataArePhaseFlipped'] = xmipp_protocol_projmatch.DataArePhaseFlipped
packageparams['ReferenceIsCtfCorrected'] = xmipp_protocol_projmatch.ReferenceIsCtfCorrected
packageparams['DoMask'] = xmipp_protocol_projmatch.DoMask
packageparams['DoSphericalMask'] = xmipp_protocol_projmatch.DoSphericalMask
# packageparams['DoProjectionMatching'] = xmipp_protocol_projmatch.DoProjectionMatching
# packageparams['DisplayProjectionMatching'] = xmipp_protocol_projmatch.DisplayProjectionMatching
# packageparams['AvailableMemory'] = xmipp_protocol_projmatch.AvailableMemory
packageparams['AngSamplingRateDeg'] = xmipp_protocol_projmatch.AngSamplingRateDeg
packageparams['MaxChangeInAngles'] = xmipp_protocol_projmatch.MaxChangeInAngles
packageparams['PerturbProjectionDirections'] = xmipp_protocol_projmatch.PerturbProjectionDirections
packageparams['MaxChangeOffset'] = xmipp_protocol_projmatch.MaxChangeOffset
packageparams['Search5DShift'] = xmipp_protocol_projmatch.Search5DShift
packageparams['Search5DStep'] = xmipp_protocol_projmatch.Search5DStep
packageparams['DoRetricSearchbyTiltAngle'] = xmipp_protocol_projmatch.DoRetricSearchbyTiltAngle
packageparams['Tilt0'] = xmipp_protocol_projmatch.Tilt0
packageparams['TiltF'] = xmipp_protocol_projmatch.TiltF
packageparams['SymmetryGroupNeighbourhood'] = xmipp_protocol_projmatch.SymmetryGroupNeighbourhood
packageparams['OnlyWinner'] = xmipp_protocol_projmatch.OnlyWinner
packageparams['MinimumCrossCorrelation'] = xmipp_protocol_projmatch.MinimumCrossCorrelation
packageparams['DiscardPercentage'] = xmipp_protocol_projmatch.DiscardPercentage
packageparams['ProjMatchingExtra'] = xmipp_protocol_projmatch.ProjMatchingExtra
packageparams['DoAlign2D'] = xmipp_protocol_projmatch.DoAlign2D
packageparams['Align2DIterNr'] = xmipp_protocol_projmatch.Align2DIterNr
packageparams['Align2dMaxChangeOffset'] = xmipp_protocol_projmatch.Align2dMaxChangeOffset
packageparams['Align2dMaxChangeRot'] = xmipp_protocol_projmatch.Align2dMaxChangeRot
# packageparams['DoReconstruction'] = xmipp_protocol_projmatch.DoReconstruction
# packageparams['DisplayReconstruction'] = xmipp_protocol_projmatch.DisplayReconstruction
packageparams['ReconstructionMethod'] = xmipp_protocol_projmatch.ReconstructionMethod
packageparams['ARTLambda'] = xmipp_protocol_projmatch.ARTLambda
packageparams['ARTReconstructionExtraCommand'] = xmipp_protocol_projmatch.ARTReconstructionExtraCommand
packageparams['WBPReconstructionExtraCommand'] = xmipp_protocol_projmatch.WBPReconstructionExtraCommand
packageparams['FourierReconstructionExtraCommand'] = xmipp_protocol_projmatch.FourierReconstructionExtraCommand
# packageparams['DoSplitReferenceImages'] = xmipp_protocol_projmatch.DoSplitReferenceImages
packageparams['ResolSam'] = xmipp_protocol_projmatch.ResolSam
# packageparams['DisplayResolution'] = xmipp_protocol_projmatch.DisplayResolution
packageparams['ConstantToAddToFiltration'] = xmipp_protocol_projmatch.ConstantToAddToFiltration
# packageparams['NumberOfThreads'] = xmipp_protocol_projmatch.NumberOfThreads
# packageparams['DoParallel'] = xmipp_protocol_projmatch.DoParallel
# packageparams['NumberOfMpiProcesses'] = xmipp_protocol_projmatch.NumberOfMpiProcesses
# packageparams['MpiJobSize'] = xmipp_protocol_projmatch.MpiJobSize
# packageparams['SystemFlavour'] = xmipp_protocol_projmatch.SystemFlavour
# packageparams['AnalysisScript'] = xmipp_protocol_projmatch.AnalysisScript
if xmipp_protocol_projmatch.SymmetryGroup[:1] == "i":
sym = "icos"
else:
sym = xmipp_protocol_projmatch.SymmetryGroup
sym = sym.split()[0]
### set global parameters
runparams = {}
runparams['numiter'] = packageparams['NumberofIterations']
# Mask should be in pixels of original stack. Xmipp returns pixels in terms of preped stack.
# so orig_pix = preped_pix * (orig_boxsize / preped_boxsize).
# This is needed in case the prepedatack was binned.
boxscale = self.runparams['original_boxsize'] / self.runparams['boxsize']
runparams['mask'] = packageparams['MaskRadius'] * boxscale
runparams['alignmentInnerRadius'] = packageparams['InnerRadius'] * boxscale
runparams['alignmentOuterRadius'] = packageparams['OuterRadius'] * boxscale
runparams['symmetry'] = apSymmetry.findSymmetry(sym)
runparams['angularSamplingRate'] = packageparams['AngSamplingRateDeg']
runparams['apix'] = packageparams['ResolSam']
runparams['package_params'] = packageparams
runparams['remoterundir'] = os.path.abspath(xmipp_protocol_projmatch.ProjectDir)
runparams['rundir'] = self.params['rundir']
return runparams
def start(self):
self.params["outputstack"] = os.path.join(self.params["rundir"], self.params["stackname"])
particles, self.params["refineiter"] = getParticleInfo(self.params["reconid"], self.params["iter"])
stackdata = particles[0]["particle"]["stack"]
stack = os.path.join(stackdata["path"]["path"], stackdata["name"])
classes, cstats = determineClasses(particles)
rejectlst = []
if self.params["sigma"] is not None:
cutoff = cstats["meanquality"] + self.params["sigma"] * cstats["stdquality"]
apDisplay.printMsg("Cutoff = " + str(cutoff))
rejectlst = self.removePtclsByQualityFactor(particles, rejectlst, cutoff)
if self.params["avgjump"] is not None:
rejectlst = self.removePtclsByJumps(particles, rejectlst)
if self.params["rejectlst"]:
rejectlst = removePtclsByLst(rejectlst, self.params)
classkeys = classes.keys()
classkeys.sort()
classnum = 0
totalptcls = 0
keepfile = open("keep.lst", "w")
keepfile.write("#LST\n")
reject = open("reject.lst", "w")
reject.write("#LST\n")
apDisplay.printMsg("Processing classes")
# loop through classes
for key in classkeys:
# file to hold particles of this class
clsfile = open("clstmp.lst", "w")
clsfile.write("#LST\n")
classnum += 1
if classnum % 10 == 1:
apDisplay.printMsg(str(classnum) + " of " + (str(len(classkeys))))
images = EMAN.EMData()
# loop through particles in class
nptcls = 0
for ptcl in classes[key]["particles"]:
if ptcl["mirror"]:
mirror = 1
else:
mirror = 0
rot = ptcl["euler3"]
rot = rot * math.pi / 180
if ptcl["particle"]["particleNumber"] not in rejectlst:
l = "%d\t%s\t%f,\t%f,%f,%f,%d\n" % (
ptcl["particle"]["particleNumber"] - 1,
stack,
ptcl["quality_factor"],
rot,
ptcl["shiftx"],
ptcl["shifty"],
mirror,
)
keepfile.write(l)
clsfile.write(l)
totalptcls += 1
nptcls += 1
else:
reject.write(
"%d\t%s\t%f,\t%f,%f,%f,%d\n"
% (
ptcl["particle"]["particleNumber"] - 1,
stack,
ptcl["quality_factor"],
rot,
ptcl["shiftx"],
ptcl["shifty"],
mirror,
)
)
# if ptcl['quality_factor']>cstats['meanquality']+3*cstats['stdquality']:
# high.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1,
# stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror))
clsfile.close()
if nptcls < 1:
continue
if self.params["skipavg"] is False:
makeClassAverages("clstmp.lst", self.params["outputstack"], classes[key], self.params)
if self.params["eotest"] is True:
self.makeEvenOddClasses("clstmp.lst", classes[key])
apDisplay.printMsg("\n")
reject.close()
keepfile.close()
os.remove("clstmp.lst")
# make 3d density file if specified:
if self.params["make3d"] is not None:
self.params["make3d"] = os.path.basename(self.params["make3d"])
outfile = os.path.join(self.params["rundir"], self.params["make3d"])
apEMAN.make3d(
self.params["stackname"],
outfile,
sym=self.params["sym"],
mode=self.params["mode"],
hard=self.params["hard"],
)
apEMAN.executeEmanCmd("proc3d %s %s mask=%d norm" % (outfile, outfile, self.params["mask"]))
if self.params["eotest"] is True:
apEMAN.make3d(
self.params["oddstack"],
"odd.mrc",
sym=self.params["sym"],
mode=self.params["mode"],
hard=self.params["hard"],
)
apEMAN.make3d(
self.params["evenstack"],
"even.mrc",
sym=self.params["sym"],
mode=self.params["mode"],
hard=self.params["hard"],
)
apEMAN.executeEmanCmd("proc3d odd.mrc even.mrc fsc=fsc.eotest")
if os.path.exists(outfile):
# run rmeasure
apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
box = apVolume.getModelDimensions(outfile)
apDisplay.printMsg("inserting density into database")
symdata = apSymmetry.findSymmetry(self.params["sym"])
if not symdata:
apDisplay.printError("no symmetry associated with this model")
modq = appiondata.Ap3dDensityData()
modq["session"] = self.params["sessionname"]
modq["name"] = self.params["make3d"]
modq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
modq["boxsize"] = box
modq["mask"] = self.params["mask"]
modq["pixelsize"] = apix
fscres = apRecon.getResolutionFromFSCFile("fsc.eotest", box, apix, msg=True)
modq["resolution"] = fscres
modq["rmeasure"] = apRecon.runRMeasure(apix, outfile)
modq["md5sum"] = apFile.md5sumfile(outfile)
modq["maxjump"] = self.params["avgjump"]
modq["sigma"] = self.params["sigma"]
modq["hard"] = self.params["hard"]
modq["symmetry"] = symdata
modq["refineIter"] = self.params["refineiter"]
if self.params["commit"] is True:
modq.insert()
apChimera.filterAndChimera(
outfile,
res=fscres,
apix=apix,
box=box,
chimtype="snapshot",
zoom=self.params["zoom"],
sym=self.params["sym"],
mass=self.params["mass"],
)
else:
apDisplay.printError("no 3d volume was generated - check the class averages:")
apDisplay.printError(self.params["stackname"])
apDisplay.printError("hard may be set too high, or avg euler jump set too low for the # of particles")
stackstr = str(stackdata.dbid)
reconstr = str(self.params["reconid"])
apDisplay.printColor(
"Make a new stack with only non-jumpers:\n"
+ "subStack.py --projectid="
+ str(self.params["projectid"])
+ " -s "
+ stackstr
+ " \\\n "
+ " -k "
+ os.path.join(self.params["rundir"], "keep.lst")
+ " \\\n "
+ " -d 'recon "
+ reconstr
+ " sitters' -n sitters"
+ reconstr
+ " -C ",
"purple",
)
def readParamsFromCombineFile(self, iternum):
"""
read the combine file to get iteration parameters,
such as symmetry and phase residual cutoff
"""
iterparams = {"iternum": iternum}
combinefile = "iter%03d/frealign.iter%03d.combine.sh" % (iternum, iternum)
f = open(combinefile, "r")
### locate start
for line in f:
if line.startswith("### START FREALIGN ###"):
break
### parse file
cards = []
for line in f:
cards.append(line.strip())
if line.startswith("EOF"):
break
f.close()
### get lots of info from card #1
data = cards[1].split(",")
# print data
iterparams["cform"] = data[0]
iterparams["iflag"] = int(data[1])
iterparams["fmag"] = self.cb(data[2])
iterparams["fdef"] = self.cb(data[3])
iterparams["fastig"] = self.cb(data[4])
iterparams["fpart"] = self.cb(data[5])
iterparams["iewald"] = float(data[6])
iterparams["fbeaut"] = self.cb(data[7])
iterparams["fcref"] = self.cb(data[8])
iterparams["fmatch"] = self.cb(data[9])
iterparams["ifsc"] = self.cb(data[10])
iterparams["fstat"] = self.cb(data[11])
iterparams["iblow"] = int(data[12])
### get lots of info from card #2
data = cards[2].split(",")
# print data
iterparams["mask"] = float(data[0])
iterparams["imask"] = float(data[1])
iterparams["wgh"] = float(data[3])
iterparams["xstd"] = float(data[4])
iterparams["pbc"] = float(data[5])
iterparams["boff"] = float(data[6])
iterparams["dang"] = float(data[7]) ### only used for recon search
iterparams["itmax"] = int(data[8])
iterparams["ipmax"] = int(data[9])
### get symmetry info from card #5
symmdata = apSymmetry.findSymmetry(cards[5])
apDisplay.printMsg("Found symmetry %s with id %s" % (symmdata["eman_name"], symmdata.dbid))
iterparams["symmdata"] = symmdata
### get threshold info from card #6
data = cards[6].split(",")
# print data
iterparams["target"] = float(data[2])
iterparams["thresh"] = float(data[3])
iterparams["cs"] = float(data[4])
### get limit info from card #7
data = cards[7].split(",")
# print data
iterparams["rrec"] = float(data[0])
iterparams["highpass"] = float(data[1])
iterparams["lowpass"] = float(data[2])
iterparams["rbfact"] = float(data[4])
# print iterparams
return iterparams
def parseLogFile(self):
# parse out the refine command from the .emanlog to get the parameters for each iteration
logfile = self.findEmanJobFile()
apDisplay.printMsg("parsing eman log file: " + logfile)
lines = open(logfile, 'r')
self.iterationdatas = []
for line in lines:
# if read a refine line, get the parameters
line = line.rstrip()
if re.search("refine \d+ ", line):
emanparams = line.split(' ')
if emanparams[0] is "#":
emanparams.pop(0)
# get rid of first "refine"
emanparams.pop(0)
iteration = self.defineIteration()
iteration['num'] = emanparams[0]
iteration['sym'] = ''
for p in emanparams:
elements = p.strip().split('=')
if elements[0] == 'ang':
iteration['ang'] = float(elements[1])
elif elements[0] == 'mask':
iteration['mask'] = int(float(elements[1]))
elif elements[0] == 'imask':
iteration['imask'] = int(float(elements[1]))
elif elements[0] == 'pad':
iteration['pad'] = int(float(elements[1]))
elif elements[0] == 'sym':
iteration['sym'] = apSymmetry.findSymmetry(elements[1])
elif elements[0] == 'maxshift':
iteration['maxshift'] = int(float(elements[1]))
elif elements[0] == 'hard':
iteration['hard'] = int(float(elements[1]))
elif elements[0] == 'classkeep':
iteration['classkeep'] = float(elements[1].strip())
elif elements[0] == 'classiter':
iteration['classiter'] = int(float(elements[1]))
elif elements[0] == 'filt3d':
iteration['filt3d'] = int(float(elements[1]))
elif elements[0] == 'shrink':
iteration['shrink'] = int(float(elements[1]))
elif elements[0] == 'euler2':
iteration['euler2'] = int(float(elements[1]))
elif elements[0] == 'xfiles':
## trying to extract "xfiles" as entered into emanJobGen.php
values = elements[1]
apix, mass, alito = values.split(',')
iteration['xfiles'] = float(mass)
elif elements[0] == 'amask1':
iteration['amask1'] = float(elements[1])
elif elements[0] == 'amask2':
iteration['amask2'] = float(elements[1])
elif elements[0] == 'amask3':
iteration['amask3'] = float(elements[1])
elif elements[0] == 'median':
iteration['median'] = True
elif elements[0] == 'phasecls':
iteration['phasecls'] = True
elif elements[0] == 'fscls':
iteration['fscls'] = True
elif elements[0] == 'refine':
iteration['refine'] = True
elif elements[0] == 'goodbad':
iteration['goodbad'] = True
elif elements[0] == 'perturb':
iteration['perturb'] = True
self.iterationdatas.append(iteration)
if re.search("coran_for_cls.py \d+ ", line):
self.params['package'] = 'EMAN/SpiCoran'
if re.search("msgPassing_subClassification.py \d+ ", line):
self.params['package'] = 'EMAN/MsgP'
apDisplay.printColor(
"Found " + str(len(self.iterationdatas)) + " iterations", "green")
lines.close()
