def prepareRecon(self):
processdir = self.params['rundir']
stackdir = self.params['tiltseriesdir']
thickness = int(self.params['thickness'])
# Make Sample Tomogram for etomo manual positioning and exit
aligndir = self.alignerdata['alignrun']['path']['path']
templatedir = os.path.join(os.path.dirname(apImod.__file__), 'data')
yspacing_fraction = 0.66
apImod.sampleRecon(stackdir, processdir, aligndir, self.seriesname, 10,
0.66, thickness, self.excludelist)
stackpath = os.path.join(stackdir, self.seriesname + ".st")
yspacing_pixel = apFile.getMrcFileShape(
stackpath)[1] * yspacing_fraction * 0.5
has_rotation = False
if self.alignerdata['protomo']:
if self.alignerdata['refine_cycle']['cycle'] > 0:
has_rotation = True
apImod.makeFilesForETomoSampleRecon(processdir, stackdir, aligndir,
templatedir, self.seriesname,
thickness, self.pixelsize,
yspacing_pixel, has_rotation)
paramfile = os.path.join(processdir,
'%s_sample.params' % (self.params['runname']))
apParam.dumpParameters(self.params, paramfile)
return
def prepareRecon(self):
processdir = self.params['rundir']
stackdir = self.params['tiltseriesdir']
thickness = int(self.params['thickness'])
# Make Sample Tomogram for etomo manual positioning and exit
aligndir = self.alignerdata['alignrun']['path']['path']
templatedir = os.path.join(os.path.dirname(apImod.__file__),'data')
yspacing_fraction = 0.66
apImod.sampleRecon(stackdir, processdir, aligndir, self.seriesname, 10, 0.66, thickness, self.excludelist)
stackpath = os.path.join(stackdir, self.seriesname+".st")
yspacing_pixel = apFile.getMrcFileShape(stackpath)[1] * yspacing_fraction * 0.5
has_rotation = False
if self.alignerdata['protomo']:
if self.alignerdata['refine_cycle']['cycle'] > 0:
has_rotation = True
apImod.makeFilesForETomoSampleRecon(processdir, stackdir,aligndir, templatedir, self.seriesname, thickness, self.pixelsize,yspacing_pixel,has_rotation)
paramfile = os.path.join(processdir,'%s_sample.params' % (self.params['runname']))
apParam.dumpParameters(self.params, paramfile)
return
def saveRunParamsToFile(self, methodParams):
### Write out the run parameters, both generic and method specific, for posterior uploading
self.runparams = {} ### these are generic params that includes a dictionary entry for package-specific params
# self.runparams['symmetry'] = apSymmetry.getSymmetryDataFromName(self.params['symmetry'])
#sym = apSymmetry.getSymmetryDataFromID(self.params['symmetry'])
#sym2 = self.convertSymmetryNameForPackage(sym)
self.runparams['symmetry'] = methodParams["sym"]
self.runparams['numiter'] = 1
self.runparams['mask'] = methodParams["particle_diameter"][0] / 2
self.runparams['reconstruction_package'] = "Relion"
self.runparams['remoterundir'] = self.params['remoterundir']
self.runparams['reconstruction_working_dir'] = methodParams["o"]
self.runparams['package_params'] = methodParams
self.picklefile = os.path.join(self.params['remoterundir'], "relion_"+self.timestamp+"-params.pickle")
apParam.dumpParameters(self.runparams, self.picklefile)
### finished setup of input files, now run xmipp_protocols_ml3d.py from jobfile
apDisplay.printMsg("finished setting up input files, now ready to run protocol")
def saveRunParamsToFile(self, methodParams):
### Write out the run parameters, both generic and method specific, for posterior uploading
self.runparams = {} ### these are generic params that includes a dictionary entry for package-specific params
# self.runparams['symmetry'] = apSymmetry.getSymmetryDataFromName(self.params['symmetry'])
#sym = apSymmetry.getSymmetryDataFromID(self.params['symmetry'])
#sym2 = self.convertSymmetryNameForPackage(sym)
self.runparams['symmetry'] = methodParams["sym"]
self.runparams['numiter'] = 1
self.runparams['mask'] = methodParams["particle_diameter"][0] / 2
self.runparams['reconstruction_package'] = "Relion"
self.runparams['remoterundir'] = self.params['remoterundir']
self.runparams['reconstruction_working_dir'] = methodParams["o"]
self.runparams['package_params'] = methodParams
self.picklefile = os.path.join(self.params['remoterundir'], "relion_"+self.timestamp+"-params.pickle")
apParam.dumpParameters(self.runparams, self.picklefile)
### finished setup of input files, now run xmipp_protocols_ml3d.py from jobfile
apDisplay.printMsg("finished setting up input files, now ready to run protocol")
def start(self):
### simple is written in Fortran, which cannot take inputs of certain length, therefore one needs
### to change to the directory to minimize the filename length, in particular for the stack
os.chdir(self.params['rundir'])
### stack needs to be centered
if self.params['no_center'] is False:
if os.path.isfile(os.path.join(self.params['rundir'], "ali.hed")):
apFile.removeStack(
os.path.join(self.params['rundir'], "ali.hed"))
centstack = os.path.join(self.params['rundir'], "ali.hed")
centcmd = "cenalignint %s > cenalignint.log" % (self.stack['file'])
apParam.runCmd(centcmd, "EMAN")
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".spi")
if os.path.isfile(self.params['localstack']):
apFile.removeFile(self.params['localstack'])
if self.params['no_center'] is False:
proccmd = "proc2d " + centstack + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
else:
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
self.boxsize, self.boxsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
proccmd += " spiderswap"
# if self.params['highpass'] is not None and self.params['highpass'] > 1:
# proccmd += " hp="+str(self.params['highpass'])
# if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
# proccmd += " lp="+str(self.params['lowpass'])
apParam.runCmd(proccmd, "EMAN", verbose=True)
# if self.params['numpart'] != int(spider.getSpiderHeader(self.params['localstack'])[-2]):
# apDisplay.printError("Missing particles in stack")
### setup Simple command
aligntime = time.time()
simpleopts = (
"" + " stk=%s" % os.path.basename(self.params['localstack']) +
" box=%d" % self.boxsize + " nptcls=%d" % self.params['numpart'] +
" smpd=%.3f" % self.apix + " ring2=%d" % self.params['ring2'] +
" ncls=%d" % self.params['ncls'] +
" minp=%d" % self.params['minp'] +
" nvars=%d" % self.params['nvars'] +
" nthr=%d" % self.params['nproc'])
if self.params['no_kmeans'] is True:
simpleopts += " kmeans=off"
if self.params['nran'] is not None:
simpleopts += "nran=%d" % self.params['nran']
### SIMPLE 2D clustering
apDisplay.printColor(
"Using " + str(self.params['nproc']) + " processors!", "green")
simpleexe = apParam.getExecPath("cluster", die=True)
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd,
package="SIMPLE",
verbose=True,
showcmd=True,
logfile="cluster.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment & Classification time: " +
apDisplay.timeString(self.params['runtime']))
### SIMPLE spider to Fourier format
clsavgs = "cavgstk.spi"
if not os.path.isfile(os.path.join(self.params['rundir'], clsavgs)):
apDisplay.printError(
"class averages were not created! try rerunning with centering, more particles, or less ppc"
)
try:
nptcls = spider.getSpiderHeader(clsavgs)[-2]
except:
nptcls = self.params['ncls']
apDisplay.printWarning(
"class average file may not have been created! Please check existence of file cavgstk.spi"
)
projfile = "projs"
projext = ".fim"
simpleexe = apParam.getExecPath("spi_to_fim", die=True)
simpleopts = ("" + " stk=%s" % clsavgs + " box=%d" % self.boxsize +
" nptcls=%d" % nptcls + " smpd=%.3f" % self.apix +
" outbdy=%s" % projfile +
" msk=%d" % self.params['mask'])
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd,
package="SIMPLE",
verbose=True,
showcmd=True,
logfile="spi_to_fim.std")
### SIMPLE origami, ab initio 3D reconstruction
refinetime = time.time()
simpleexe = apParam.getExecPath("origami", die=True)
simpleopts = (
"" + " fstk=%s" % projfile + projext +
" froms=%d" % self.params['froms'] +
" tos=%d" % self.params['tos'] + " lp=%d" % self.params['lp'] +
" hp=%d" % self.params['hp'] +
" maxits=%d" % self.params['maxits'] +
" msk=%d" % self.params['mask'] + " mw=%d" % self.params['mw'] +
" frac=%.3f" % self.params['frac'] +
" amsklp=%d" % self.params['amsklp'] +
" edge=%d" % self.params['edge'] + " trs=%d" % self.params['trs'] +
" nthr=%d" % self.params['nproc'])
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd,
package="SIMPLE",
verbose=True,
showcmd=True,
logfile="origami.std")
refinetime = time.time() - refinetime
apDisplay.printMsg("Origami reconstruction time: " +
apDisplay.timeString(refinetime))
# '''
### minor post-processing
self.clearIntermediateFiles()
apParam.dumpParameters(
self.params,
"simple-" + self.params['timestamp'] + "-params.pickle")
### upload results
self.runparams = apParam.readRunParameters("simple-" +
self.params['timestamp'] +
"-params.pickle")
### create average of aligned and clustered stacks, convert to IMAGIC
alignedStackSpi = "inplalgnstk.spi"
alignedStack = "inplalgnstk.hed"
if os.path.isfile(alignedStack):
apFile.removeStack(alignedStack)
emancmd = "proc2d %s %s flip" % (alignedStackSpi, alignedStack)
apParam.runCmd(emancmd, "EMAN")
clusterStackSpi = "cavgstk.spi"
clusterStack = "cavgstk.hed"
if os.path.isfile(clusterStack):
apFile.removeStack(clusterStack)
emancmd = "proc2d %s %s flip" % (clusterStackSpi, clusterStack)
apParam.runCmd(emancmd, "EMAN")
# apStack.averageStack(alignedStack)
### parse alignment and classification results
if self.params['no_center'] is False:
self.alignD = self.getAlignParameters(centparams="cenalignint.log")
else:
self.alignD = self.getAlignParameters()
if self.params['no_kmeans'] is False:
self.classD = self.getClassification("kmeans.spi", clusterStack)
else:
self.classD = self.getClassification("hcl.spi", clusterStack)
### upload to database
self.insertSIMPLEAlignParamsIntoDatabase()
self.insertAlignStackRunIntoDatabase(alignedStack, clusterStack)
self.calcResolution(alignedStack)
self.insertAlignParticlesIntoDatabase()
self.insertClusterRunIntoDatabase()
self.insertClusterStackIntoDatabase(clusterStack, len(self.classD))
self.insertSIMPLEOrigamiParamsIntoDatabase()
def setupXmippML3DProtocol(self):
''' sets up ML3D protocol parameters, pickles to file, etc.'''
protocolname = 'xmipp_protocol_ml3d'
### Locate protocol_ml3d
protocol_ml3d = apXmipp.locateXmippProtocol(protocolname)
### setup protocol parameters
protocolPrm = {}
### check for n input model dependencies
protocolPrm["InitialReference"] = self.params['modelnames'][0]
if len(self.params['modelnames']) > 1:
selfile = "reference_volumes.sel"
sf = open(selfile, "w")
for model in self.params['modelnames']:
sf.write("%s\t1\n" % os.path.join(self.params['rundir'], model))
sf.close()
protocolPrm["SeedsSelfile"] = os.path.join(self.params['rundir'], "reference_volumes.sel")
protocolPrm["DoGenerateSeeds"] = False
else:
protocolPrm["SeedsSelfile"] = ""
protocolPrm["DoGenerateSeeds"] = True
protocolPrm["InSelFile"] = "partlist.sel" ### this maybe should not be hardcoded
protocolPrm["WorkingDir"] = "ml3d"
protocolPrm["DoDeleteWorkingDir"] = False
protocolPrm["ProjectDir"] = self.params['recondir']
protocolPrm["LogDir"] = "Logs"
protocolPrm["DoMlf"] = self.params['DoMlf']
protocolPrm["DoCorrectAmplitudes"] = False
protocolPrm["InCtfDatFile"] = "all_images.ctfdat"
protocolPrm["HighResLimit"] = self.params['HighResLimit']
protocolPrm["ImagesArePhaseFlipped"] = self.params['ImagesArePhaseFlipped']
protocolPrm["InitialMapIsAmplitudeCorrected"] = False
protocolPrm["SeedsAreAmplitudeCorrected"] = False
protocolPrm["DoCorrectGreyScale"] = self.params['DoCorrectGreyScale']
protocolPrm["ProjMatchSampling"] = self.params['ProjMatchSampling']
if (self.params['LowPassFilter']>1) is True:
protocolPrm["DoLowPassFilterReference"] = True
protocolPrm["LowPassFilter"] = self.params['LowPassFilter']
else:
protocolPrm["DoLowPassFilterReference"] = False
protocolPrm["LowPassFilter"] = self.params['LowPassFilter']
protocolPrm["PixelSize"] = self.params['apix']
protocolPrm["NumberOfReferences"] = self.params['NumberOfReferences']
protocolPrm["DoJustRefine"] = False
protocolPrm["DoML3DClassification"] = True
protocolPrm["AngularSampling"] = self.params['AngularSampling']
protocolPrm["NumberOfIterations"] = self.params['numiter']
protocolPrm["Symmetry"] = self.params['symmetry'][0]
protocolPrm["DoNorm"] = False
protocolPrm["DoFourier"] = False
protocolPrm["RestartIter"] = 0
protocolPrm["ExtraParamsMLrefine3D"] = ""
protocolPrm["NumberOfThreads"] = 1
protocolPrm["DoParallel"] = self.params['nproc']>1
protocolPrm["NumberOfMpiProcesses"] = self.params['nproc']
protocolPrm["SystemFlavour"] = ""
protocolPrm["AnalysisScript"] = "visualize_ml3d.py"
### write out python protocol into run directory
protocolfile = os.path.join(self.params['remoterundir'],"%s.py" % protocolname)
apXmipp.particularizeProtocol(protocol_ml3d, protocolPrm, protocolfile)
os.chmod(os.path.join(self.params['rundir'], protocolfile), 0775)
### Write the parameters for posterior uploading, both generic and specific
self.runparams = {}
self.runparams['reconstruction_package'] = "xmipp_ml3d"
self.runparams['remoterundir'] = self.params['remoterundir']
# self.runparams['reconstruction_working_dir'] = protocolPrm["WorkingDir"]
self.runparams['reconstruction_working_dir'] = protocolPrm['WorkingDir']+"/RunML3D"
self.runparams['numiter'] = protocolPrm['NumberOfIterations']
self.runparams['NumberOfReferences'] = protocolPrm['NumberOfReferences']
self.runparams['symmetry'] = protocolPrm["Symmetry"]
self.runparams['package_params'] = protocolPrm
paramfile = os.path.join(self.params['remoterundir'], "xmipp_ml3d_"+self.timestamp+"-params.pickle")
apParam.dumpParameters(self.runparams, paramfile)
### finished setup of input files, now run xmipp_protocols_ml3d.py from jobfile
apDisplay.printMsg("finished setting up input files, now running xmipp_protocols_ml3d.py")
return protocolfile, protocolPrm
def start(self):
### simple is written in Fortran, which cannot take inputs of certain length, therefore one needs
### to change to the directory to minimize the filename length, in particular for the stack
os.chdir(self.params['rundir'])
### stack needs to be centered
if self.params['no_center'] is False:
if os.path.isfile(os.path.join(self.params['rundir'], "ali.hed")):
apFile.removeStack(os.path.join(self.params['rundir'], "ali.hed"))
centstack = os.path.join(self.params['rundir'], "ali.hed")
centcmd = "cenalignint %s > cenalignint.log" % (self.stack['file'])
apParam.runCmd(centcmd, "EMAN")
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".spi")
if os.path.isfile(self.params['localstack']):
apFile.removeFile(self.params['localstack'])
if self.params['no_center'] is False:
proccmd = "proc2d "+centstack+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
else:
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], self.boxsize, self.boxsize)
proccmd += " last="+str(self.params['numpart']-1)
proccmd += " spiderswap"
# if self.params['highpass'] is not None and self.params['highpass'] > 1:
# proccmd += " hp="+str(self.params['highpass'])
# if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
# proccmd += " lp="+str(self.params['lowpass'])
apParam.runCmd(proccmd, "EMAN", verbose=True)
# if self.params['numpart'] != int(spider.getSpiderHeader(self.params['localstack'])[-2]):
# apDisplay.printError("Missing particles in stack")
### setup Simple command
aligntime = time.time()
simpleopts = (""
+" stk=%s" % os.path.basename(self.params['localstack'])
+" box=%d" % self.boxsize
+" nptcls=%d" % self.params['numpart']
+" smpd=%.3f" % self.apix
+" ring2=%d" % self.params['ring2']
+" ncls=%d" % self.params['ncls']
+" minp=%d" % self.params['minp']
+" nvars=%d" % self.params['nvars']
+" nthr=%d" % self.params['nproc']
)
if self.params['no_kmeans'] is True:
simpleopts += " kmeans=off"
if self.params['nran'] is not None:
simpleopts += "nran=%d" % self.params['nran']
### SIMPLE 2D clustering
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
simpleexe = apParam.getExecPath("cluster", die=True)
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="cluster.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment & Classification time: "+apDisplay.timeString(self.params['runtime']))
### SIMPLE spider to Fourier format
clsavgs = "cavgstk.spi"
if not os.path.isfile(os.path.join(self.params['rundir'], clsavgs)):
apDisplay.printError("class averages were not created! try rerunning with centering, more particles, or less ppc")
try:
nptcls = spider.getSpiderHeader(clsavgs)[-2]
except:
nptcls = self.params['ncls']
apDisplay.printWarning("class average file may not have been created! Please check existence of file cavgstk.spi")
projfile = "projs"
projext = ".fim"
simpleexe = apParam.getExecPath("spi_to_fim", die=True)
simpleopts = (""
+" stk=%s" % clsavgs
+" box=%d" % self.boxsize
+" nptcls=%d" % nptcls
+" smpd=%.3f" % self.apix
+" outbdy=%s" % projfile
+" msk=%d" % self.params['mask']
)
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="spi_to_fim.std")
### SIMPLE origami, ab initio 3D reconstruction
refinetime = time.time()
simpleexe = apParam.getExecPath("origami", die=True)
simpleopts = (""
+" fstk=%s" % projfile+projext
+" froms=%d" % self.params['froms']
+" tos=%d" % self.params['tos']
+" lp=%d" % self.params['lp']
+" hp=%d" % self.params['hp']
+" maxits=%d" % self.params['maxits']
+" msk=%d" % self.params['mask']
+" mw=%d" % self.params['mw']
+" frac=%.3f" % self.params['frac']
+" amsklp=%d" % self.params['amsklp']
+" edge=%d" % self.params['edge']
+" trs=%d" % self.params['trs']
+" nthr=%d" % self.params['nproc']
)
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="origami.std")
refinetime = time.time() - refinetime
apDisplay.printMsg("Origami reconstruction time: "+apDisplay.timeString(refinetime))
# '''
### minor post-processing
self.clearIntermediateFiles()
apParam.dumpParameters(self.params, "simple-"+self.params['timestamp']+"-params.pickle")
### upload results
self.runparams = apParam.readRunParameters("simple-"+self.params['timestamp']+"-params.pickle")
### create average of aligned and clustered stacks, convert to IMAGIC
alignedStackSpi = "inplalgnstk.spi"
alignedStack = "inplalgnstk.hed"
if os.path.isfile(alignedStack):
apFile.removeStack(alignedStack)
emancmd = "proc2d %s %s flip" % (alignedStackSpi, alignedStack)
apParam.runCmd(emancmd, "EMAN")
clusterStackSpi = "cavgstk.spi"
clusterStack = "cavgstk.hed"
if os.path.isfile(clusterStack):
apFile.removeStack(clusterStack)
emancmd = "proc2d %s %s flip" % (clusterStackSpi, clusterStack)
apParam.runCmd(emancmd, "EMAN")
# apStack.averageStack(alignedStack)
### parse alignment and classification results
if self.params['no_center'] is False:
self.alignD = self.getAlignParameters(centparams="cenalignint.log")
else:
self.alignD = self.getAlignParameters()
if self.params['no_kmeans'] is False:
self.classD = self.getClassification("kmeans.spi", clusterStack)
else:
self.classD = self.getClassification("hcl.spi", clusterStack)
### upload to database
self.insertSIMPLEAlignParamsIntoDatabase()
self.insertAlignStackRunIntoDatabase(alignedStack, clusterStack)
self.calcResolution(alignedStack)
self.insertAlignParticlesIntoDatabase()
self.insertClusterRunIntoDatabase()
self.insertClusterStackIntoDatabase(clusterStack, len(self.classD))
self.insertSIMPLEOrigamiParamsIntoDatabase()
def setupXmippProtocol(self):
protocolname = 'xmipp_protocol_projmatch'
# Locate protocol_projmatch
protocol_projmatch = apXmipp.locateXmippProtocol(protocolname)
#make threads and mpi processes compatible with the xmipprequirement
self.params['alwaysone'] = 1
protocolPrm = {}
protocolPrm["SelFileName"] = "partlist.sel"
protocolPrm["DocFileName"] = ""
protocolPrm["ReferenceFileName"] = self.params['modelnames'][0]
protocolPrm["WorkingDir"] = "ProjMatch"
protocolPrm["DoDeleteWorkingDir"] = True
protocolPrm["NumberofIterations"] = self.params['enditer']
protocolPrm["ContinueAtIteration"] = self.params['startiter']
protocolPrm["CleanUpFiles"] = False
protocolPrm["ProjectDir"] = self.params['recondir']
protocolPrm["LogDir"] = "Logs"
protocolPrm["DoCtfCorrection"] = False
protocolPrm["CTFDatName"] = ""
protocolPrm["DoAutoCtfGroup"] = False
protocolPrm["CtfGroupMaxDiff"] = 0.5
protocolPrm["CtfGroupMaxResol"] = 15
protocolPrm["SplitDefocusDocFile"] = ""
protocolPrm["PaddingFactor"] = 2
protocolPrm["WienerConstant"] = -1
protocolPrm["DataArePhaseFlipped"] = True
protocolPrm["ReferenceIsCtfCorrected"] = True
protocolPrm["DoMask"] = self.params['maskvol'] > 0
protocolPrm["DoSphericalMask"] = self.params['outerMaskRadius'] > 0
# split is needed because refineJob base class defines outMaskRadius as iteration parameter but xmipp can
# no accept such assignment
protocolPrm["MaskRadius"] = self.convertAngstromToPixel(
self.params['outerMaskRadius'].split()[0])
protocolPrm["MaskFileName"] = self.params['maskvol']
protocolPrm["DoProjectionMatching"] = True
protocolPrm["DisplayProjectionMatching"] = False
protocolPrm["InnerRadius"] = self.convertAngstromToPixel(
self.params['innerAlignRadius'])
protocolPrm["OuterRadius"] = self.convertAngstromToPixel(
self.params['outerAlignRadius'])
protocolPrm["AvailableMemory"] = self.calcRefineMem()
# protocolPrm["AngSamplingRateDeg"] = self.params['AngularSteps']
protocolPrm["AngSamplingRateDeg"] = self.params['angSampRate']
protocolPrm["MaxChangeInAngles"] = self.params['maxAngularChange']
protocolPrm["PerturbProjectionDirections"] = False
protocolPrm["MaxChangeOffset"] = self.params['maxChangeOffset']
protocolPrm["Search5DShift"] = self.params['search5DShift']
protocolPrm["Search5DStep"] = self.params['search5DStep']
protocolPrm["DoRetricSearchbyTiltAngle"] = False
protocolPrm["Tilt0"] = 0
protocolPrm["TiltF"] = 0
protocolPrm["SymmetryGroup"] = self.params['symmetry']
protocolPrm["SymmetryGroupNeighbourhood"] = ''
protocolPrm["OnlyWinner"] = False
protocolPrm["MinimumCrossCorrelation"] = '-1'
protocolPrm["DiscardPercentage"] = self.params['percentDiscard']
protocolPrm["ProjMatchingExtra"] = ''
protocolPrm["DoAlign2D"] = '0'
protocolPrm["Align2DIterNr"] = 4
protocolPrm["Align2dMaxChangeOffset"] = '1000'
protocolPrm["Align2dMaxChangeRot"] = '1000'
if not self.params['refineonly']:
protocolPrm["DoReconstruction"] = True
else:
protocolPrm["DoReconstruction"] = False
protocolPrm["DisplayReconstruction"] = False
protocolPrm["ARTLambda"] = self.params['ARTLambda']
protocolPrm["ARTReconstructionExtraCommand"] = ''
protocolPrm["FourierMaxFrequencyOfInterest"] = self.params[
'fouriermaxfrequencyofinterest']
protocolPrm["WBPReconstructionExtraCommand"] = ''
protocolPrm["FourierReconstructionExtraCommand"] = ''
if not self.params['refineonly']:
protocolPrm["ReconstructionMethod"] = self.params['reconMethod']
protocolPrm["DoComputeResolution"] = self.params[
'docomputeresolution']
protocolPrm["DoSplitReferenceImages"] = True
protocolPrm["DoLowPassFilter"] = self.params['dolowpassfilter']
protocolPrm["UseFscForFilter"] = self.params['usefscforfilter']
else:
#Xmipp does not allow False SplitReferenceImage except the exact syntex of 'fourier' in ReconstructionMethod
protocolPrm["ReconstructionMethod"] = 'fourier'
#These should never be used if reconstruction is not done
protocolPrm["DoComputeResolution"] = False
protocolPrm["DoSplitReferenceImages"] = False
protocolPrm["DoLowPassFilter"] = False
protocolPrm["UseFscForFilter"] = False
protocolPrm["ResolSam"] = self.params['apix']
protocolPrm["DisplayResolution"] = False
if self.params['usefscforfilter'] is False and self.params[
'filterResolution']:
protocolPrm["ConstantToAddToFiltration"] = str(
self.params['apix'] / self.params['filterResolution'])
else:
protocolPrm["ConstantToAddToFiltration"] = str(
self.params['filterConstant'])
print protocolPrm["ConstantToAddToFiltration"]
protocolPrm["NumberOfThreads"] = self.params['alwaysone']
protocolPrm["DoParallel"] = self.params['nproc'] > 1
protocolPrm["NumberOfMpiProcesses"] = self.params['nproc']
protocolPrm["MpiJobSize"] = '10'
protocolPrm["SystemFlavour"] = ''
protocolPrm["AnalysisScript"] = 'visualize_projmatch.py'
### write out python protocol into run directory
protocolfile = os.path.join(self.params['remoterundir'],
"%s.py" % protocolname)
if self.params['refineonly']:
tempprotocolfile = os.path.join(self.params['remoterundir'],
"%s.temp" % protocolname)
apXmipp.fixRefineOnlyProtocol(protocol_projmatch, tempprotocolfile)
apXmipp.particularizeProtocol(tempprotocolfile, protocolPrm,
protocolfile)
os.remove(tempprotocolfile)
else:
apXmipp.particularizeProtocol(protocol_projmatch, protocolPrm,
protocolfile)
os.chmod(os.path.join(self.params['rundir'], protocolfile), 0775)
### Write the parameters for posterior uploading, both generic and specific
self.runparams = {
} ### these are generic params that includes a dictionary entry for package-specific params
# self.runparams['symmetry'] = apSymmetry.getSymmetryDataFromName(self.params['symmetry'])
#sym = apSymmetry.getSymmetryDataFromID(self.params['symmetry'])
#sym2 = self.convertSymmetryNameForPackage(sym)
sym = protocolPrm["SymmetryGroup"][0]
if sym in ('i', 'I'):
self.runparams['symmetry'] = "Icos (2 3 5) Viper/3DEM"
else:
self.runparams['symmetry'] = protocolPrm["SymmetryGroup"]
self.runparams['numiter'] = protocolPrm['NumberofIterations']
self.runparams['mask'] = protocolPrm["MaskRadius"]
self.runparams['imask'] = None
self.runparams['alignmentInnerRadius'] = protocolPrm["InnerRadius"]
self.runparams['alignmentOuterRadius'] = protocolPrm["OuterRadius"]
self.runparams['reconstruction_package'] = "Xmipp"
self.runparams['remoterundir'] = self.params['remoterundir']
self.runparams['reconstruction_working_dir'] = protocolPrm[
"WorkingDir"]
self.runparams['package_params'] = protocolPrm
self.picklefile = os.path.join(
self.params['remoterundir'],
"xmipp_projection_matching_" + self.timestamp + "-params.pickle")
apParam.dumpParameters(self.runparams, self.picklefile)
### finished setup of input files, now run xmipp_protocols_ml3d.py from jobfile
apDisplay.printMsg(
"finished setting up input files, now ready to run protocol")
return protocolfile, protocolPrm
def setupXmippProtocol(self):
protocolname = 'xmipp_protocol_projmatch'
# Locate protocol_projmatch
protocol_projmatch=apXmipp.locateXmippProtocol(protocolname)
#make threads and mpi processes compatible with the xmipprequirement
self.params['alwaysone']=1
protocolPrm={}
protocolPrm["SelFileName"] = "partlist.sel"
protocolPrm["DocFileName"] = ""
protocolPrm["ReferenceFileName"] = self.params['modelnames'][0]
protocolPrm["WorkingDir"] = "ProjMatch"
protocolPrm["DoDeleteWorkingDir"] = True
protocolPrm["NumberofIterations"] = self.params['enditer']
protocolPrm["ContinueAtIteration"] = self.params['startiter']
protocolPrm["CleanUpFiles"] = False
protocolPrm["ProjectDir"] = self.params['recondir']
protocolPrm["LogDir"] = "Logs"
protocolPrm["DoCtfCorrection"] = False
protocolPrm["CTFDatName"] = ""
protocolPrm["DoAutoCtfGroup"] = False
protocolPrm["CtfGroupMaxDiff"] = 0.5
protocolPrm["CtfGroupMaxResol"] = 15
protocolPrm["SplitDefocusDocFile"] = ""
protocolPrm["PaddingFactor"] = 2
protocolPrm["WienerConstant"] = -1
protocolPrm["DataArePhaseFlipped"] = True
protocolPrm["ReferenceIsCtfCorrected"] = True
protocolPrm["DoMask"] = self.params['maskvol']>0
protocolPrm["DoSphericalMask"] = self.params['outerMaskRadius']>0
# split is needed because refineJob base class defines outMaskRadius as iteration parameter but xmipp can
# no accept such assignment
protocolPrm["MaskRadius"] = self.convertAngstromToPixel( self.params['outerMaskRadius'].split()[0] )
protocolPrm["MaskFileName"] = self.params['maskvol']
protocolPrm["DoProjectionMatching"] = True
protocolPrm["DisplayProjectionMatching"] = False
protocolPrm["InnerRadius"] = self.convertAngstromToPixel( self.params['innerAlignRadius'] )
protocolPrm["OuterRadius"] = self.convertAngstromToPixel( self.params['outerAlignRadius'] )
protocolPrm["AvailableMemory"] = self.calcRefineMem()
# protocolPrm["AngSamplingRateDeg"] = self.params['AngularSteps']
protocolPrm["AngSamplingRateDeg"] = self.params['angSampRate']
protocolPrm["MaxChangeInAngles"] = self.params['maxAngularChange']
protocolPrm["PerturbProjectionDirections"] = False
protocolPrm["MaxChangeOffset"] = self.params['maxChangeOffset']
protocolPrm["Search5DShift"] = self.params['search5DShift']
protocolPrm["Search5DStep"] = self.params['search5DStep']
protocolPrm["DoRetricSearchbyTiltAngle"] = False
protocolPrm["Tilt0"] = 0
protocolPrm["TiltF"] = 0
protocolPrm["SymmetryGroup"] = self.params['symmetry']
protocolPrm["SymmetryGroupNeighbourhood"] = ''
protocolPrm["OnlyWinner"] = False
protocolPrm["MinimumCrossCorrelation"] = '-1'
protocolPrm["DiscardPercentage"] = self.params['percentDiscard']
protocolPrm["ProjMatchingExtra"] = ''
protocolPrm["DoAlign2D"] = '0'
protocolPrm["Align2DIterNr"] = 4
protocolPrm["Align2dMaxChangeOffset"] = '1000'
protocolPrm["Align2dMaxChangeRot"] = '1000'
if not self.params['refineonly']:
protocolPrm["DoReconstruction"] = True
else:
protocolPrm["DoReconstruction"] = False
protocolPrm["DisplayReconstruction"] = False
protocolPrm["ARTLambda"] = self.params['ARTLambda']
protocolPrm["ARTReconstructionExtraCommand"]= ''
protocolPrm["FourierMaxFrequencyOfInterest"]= self.params['fouriermaxfrequencyofinterest']
protocolPrm["WBPReconstructionExtraCommand"]=''
protocolPrm["FourierReconstructionExtraCommand"]=''
if not self.params['refineonly']:
protocolPrm["ReconstructionMethod"] = self.params['reconMethod']
protocolPrm["DoComputeResolution"] = self.params['docomputeresolution']
protocolPrm["DoSplitReferenceImages"] = True
protocolPrm["DoLowPassFilter"] = self.params['dolowpassfilter']
protocolPrm["UseFscForFilter"] = self.params['usefscforfilter']
else:
#Xmipp does not allow False SplitReferenceImage except the exact syntex of 'fourier' in ReconstructionMethod
protocolPrm["ReconstructionMethod"] = 'fourier'
#These should never be used if reconstruction is not done
protocolPrm["DoComputeResolution"] = False
protocolPrm["DoSplitReferenceImages"] = False
protocolPrm["DoLowPassFilter"] = False
protocolPrm["UseFscForFilter"] = False
protocolPrm["ResolSam"] = self.params['apix']
protocolPrm["DisplayResolution"] = False
if self.params['usefscforfilter'] is False and self.params['filterResolution']:
protocolPrm["ConstantToAddToFiltration"] = str(self.params['apix'] / self.params['filterResolution'])
else:
protocolPrm["ConstantToAddToFiltration"] = str(self.params['filterConstant'])
print protocolPrm["ConstantToAddToFiltration"]
protocolPrm["NumberOfThreads"] = self.params['alwaysone']
protocolPrm["DoParallel"] = self.params['nproc']>1
protocolPrm["NumberOfMpiProcesses"] = self.params['nproc']
protocolPrm["MpiJobSize"] = '10'
protocolPrm["SystemFlavour"] = ''
protocolPrm["AnalysisScript"] = 'visualize_projmatch.py'
### write out python protocol into run directory
protocolfile = os.path.join(self.params['remoterundir'],"%s.py" % protocolname)
if self.params['refineonly']:
tempprotocolfile = os.path.join(self.params['remoterundir'],"%s.temp" % protocolname)
apXmipp.fixRefineOnlyProtocol(protocol_projmatch,tempprotocolfile)
apXmipp.particularizeProtocol(tempprotocolfile, protocolPrm, protocolfile)
os.remove(tempprotocolfile)
else:
apXmipp.particularizeProtocol(protocol_projmatch, protocolPrm, protocolfile)
os.chmod(os.path.join(self.params['rundir'], protocolfile), 0775)
### Write the parameters for posterior uploading, both generic and specific
self.runparams = {} ### these are generic params that includes a dictionary entry for package-specific params
# self.runparams['symmetry'] = apSymmetry.getSymmetryDataFromName(self.params['symmetry'])
#sym = apSymmetry.getSymmetryDataFromID(self.params['symmetry'])
#sym2 = self.convertSymmetryNameForPackage(sym)
sym = protocolPrm["SymmetryGroup"][0]
if sym in ('i','I'):
self.runparams['symmetry'] = "Icos (2 3 5) Viper/3DEM"
else:
self.runparams['symmetry'] = protocolPrm["SymmetryGroup"]
self.runparams['numiter'] = protocolPrm['NumberofIterations']
self.runparams['mask'] = protocolPrm["MaskRadius"]
self.runparams['imask'] = None
self.runparams['alignmentInnerRadius'] = protocolPrm["InnerRadius"]
self.runparams['alignmentOuterRadius'] = protocolPrm["OuterRadius"]
self.runparams['reconstruction_package'] = "Xmipp"
self.runparams['remoterundir'] = self.params['remoterundir']
self.runparams['reconstruction_working_dir'] = protocolPrm["WorkingDir"]
self.runparams['package_params'] = protocolPrm
self.picklefile = os.path.join(self.params['remoterundir'], "xmipp_projection_matching_"+self.timestamp+"-params.pickle")
apParam.dumpParameters(self.runparams, self.picklefile)
### finished setup of input files, now run xmipp_protocols_ml3d.py from jobfile
apDisplay.printMsg("finished setting up input files, now ready to run protocol")
return protocolfile, protocolPrm
def start(self):
self.addToLog('.... Setting up new ISAC job ....')
self.addToLog('.... Making command for stack pre-processing ....')
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### send file to remotehost
tasks = {}
sfhed = self.stack['file'][:-4] + ".hed"
sfimg = self.stack['file'][:-4] + ".img"
tasks = self.addToTasks(
tasks,
"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s"
% (sfhed, self.params['localhost'], self.params['remoterundir'],
"start1.hed"))
tasks = self.addToTasks(
tasks,
"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s"
% (sfimg, self.params['localhost'], self.params['remoterundir'],
"start1.img"))
### write Sparx jobfile: process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
# apParam.runCmd(proccmd, "EMAN", verbose=True)
self.addSimpleCommand('cd %s' % self.params['rundir'])
self.addSimpleCommand(proccmd)
sparxcmd = "sxcpy.py %s %s_1.hdf" % (self.params['localstack'],
self.params['localstack'][:-4])
# apParam.runCmd(sparxcmd, "SPARX", verbose=True)
self.addSimpleCommand(sparxcmd)
self.addSimpleCommand("")
### write Sparx jobfile: run ISAC
for i in range(self.params['generations']):
sparxopts = " %s_%d.hdf" % (os.path.join(
self.params['localstack'][:-4]), (i + 1))
if self.params['ir'] is not None:
sparxopts += " --ir %d" % int(
float(
apRecon.getComponentFromVector(
self.params['ir'], i, splitter=":")))
if self.params['ou'] is not None:
sparxopts += " --ou %d" % int(
float(
apRecon.getComponentFromVector(
self.params['ou'], i, splitter=":")))
if self.params['rs'] is not None:
sparxopts += " --rs %d" % int(
float(
apRecon.getComponentFromVector(
self.params['rs'], i, splitter=":")))
if self.params['ts'] is not None:
sparxopts += " --ts %.1f" % int(
float(
apRecon.getComponentFromVector(
self.params['ts'], i, splitter=":")))
if self.params['xr'] is not None:
sparxopts += " --xr %.1f" % int(
float(
apRecon.getComponentFromVector(
self.params['xr'], i, splitter=":")))
if self.params['yr'] is not None:
sparxopts += " --yr %.1f" % int(
float(
apRecon.getComponentFromVector(
self.params['yr'], i, splitter=":")))
if self.params['maxit'] is not None:
sparxopts += " --maxit %d" % int(
float(
apRecon.getComponentFromVector(
self.params['maxit'], i, splitter=":")))
if self.params['FL'] is not None:
sparxopts += " --FL %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['FL'], i, splitter=":")))
if self.params['FH'] is not None:
sparxopts += " --FH %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['FH'], i, splitter=":")))
if self.params['FF'] is not None:
sparxopts += " --FF %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['FF'], i, splitter=":")))
if self.params['init_iter'] is not None:
sparxopts += " --init_iter %d" % int(
float(
apRecon.getComponentFromVector(
self.params['init_iter'], i, splitter=":")))
if self.params['main_iter'] is not None:
sparxopts += " --main_iter %d" % int(
float(
apRecon.getComponentFromVector(
self.params['main_iter'], i, splitter=":")))
if self.params['iter_reali'] is not None:
sparxopts += " --iter_reali %d" % int(
float(
apRecon.getComponentFromVector(
self.params['iter_reali'], i, splitter=":")))
if self.params['match_first'] is not None:
sparxopts += " --match_first %d" % int(
float(
apRecon.getComponentFromVector(
self.params['match_first'], i, splitter=":")))
if self.params['max_round'] is not None:
sparxopts += " --max_round %d" % int(
float(
apRecon.getComponentFromVector(
self.params['max_round'], i, splitter=":")))
if self.params['match_second'] is not None:
sparxopts += " --match_second %d" % int(
float(
apRecon.getComponentFromVector(
self.params['match_second'], i, splitter=":")))
if self.params['stab_ali'] is not None:
sparxopts += " --stab_ali %d" % int(
float(
apRecon.getComponentFromVector(
self.params['stab_ali'], i, splitter=":")))
if self.params['thld_err'] is not None:
sparxopts += " --thld_err %.2f" % int(
float(
apRecon.getComponentFromVector(
self.params['thld_err'], i, splitter=":")))
if self.params['indep_run'] is not None:
sparxopts += " --indep_run %d" % int(
float(
apRecon.getComponentFromVector(
self.params['indep_run'], i, splitter=":")))
if self.params['thld_grp'] is not None:
sparxopts += " --thld_grp %d" % int(
float(
apRecon.getComponentFromVector(
self.params['thld_grp'], i, splitter=":")))
if self.params['img_per_grp'] is not None:
sparxopts += " --img_per_grp %d" % int(
apRecon.getComponentFromVector(self.params['img_per_grp'],
i))
sparxopts += " --generation %d" % (i + 1)
sparxexe = apParam.getExecPath("sxisac.py", die=True)
mpiruncmd = self.mpirun + " -np " + str(
self.params['nproc']) + " " + sparxexe + " " + sparxopts
bn = os.path.basename(self.params['localstack'])[:-4]
e2cmd = "e2proc2d.py %s_%d.hdf %s_%d.hdf --list=\"generation_%d_unaccounted.txt\"" % \
(bn, i+1, bn, i+2, i+1)
self.addSimpleCommand(mpiruncmd)
self.addSimpleCommand(e2cmd)
# print self.tasks
# print self.command_list
# self.writeCommandListToFile()
apParam.dumpParameters(
self.params, "isac-" + self.params['timestamp'] + "-params.pickle")
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -codes "+str(self.params['numrefs'])
+" -iter "+str(self.params['maxiter'])
+" -o "+os.path.join(self.params['rundir'], "part"+self.params['timestamp'])
)
if self.params['fast']:
xmippopts += " -fast "
if self.params['correlation']:
xmippopts += " -useCorrelation "
if self.params['classical']:
xmippopts += " -classicalMultiref "
if self.params['align']:
xmippopts += " -alignImages "
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### minor post-processing
self.createReferenceStack()
self.parseOutput()
self.clearIntermediateFiles()
# self.readyUploadFlag()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### setup Xmipp command
aligntime = time.time()
xmippopts = (" -i "+os.path.join(self.params['rundir'], self.params['localstack'])
+" --nref "+str(self.params['numrefs'])
+" --iter "+str(self.params['maxiter'])
+" --odir "+str(self.params['rundir'])
+" --oroot "+ "part"+str(self.params['timestamp'])
+" --classifyAllImages"
)
if self.params['correlation']:
xmippopts += " --distance correlation"
if self.params['classical']:
xmippopts += " --classicalMultiref"
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath(self.execFile, die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp 3", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### post-processing
# Create a stack for the class averages at each level
Nlevels=glob.glob("level_*")
for level in Nlevels:
digits = level.split("_")[1]
apParam.runCmd("xmipp_image_convert -i "+level+"/part"+self.params['timestamp']+"*xmd -o part"
+self.params['timestamp']+"_level_"+digits+"_.hed", package="Xmipp 3", verbose=True)
if self.params['align']:
apParam.runCmd("xmipp_transform_geometry -i images.xmd -o %s_aligned.stk --apply_transform" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apParam.runCmd("xmipp_image_convert -i %s_aligned.xmd -o alignedStack.hed" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apFile.removeFile("%s_aligned.xmd" % self.params['timestamp'])
apFile.removeFile("%s_aligned.stk" % self.params['timestamp'])
self.parseOutput()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
self.clearIntermediateFiles()
def setupXmippML3DProtocol(self):
''' sets up ML3D protocol parameters, pickles to file, etc.'''
protocolname = 'xmipp_protocol_ml3d'
### Locate protocol_ml3d
protocol_ml3d = apXmipp.locateXmippProtocol(protocolname)
### setup protocol parameters
protocolPrm = {}
### check for n input model dependencies
protocolPrm["InitialReference"] = self.params['modelnames'][0]
if len(self.params['modelnames']) > 1:
selfile = "reference_volumes.sel"
sf = open(selfile, "w")
for model in self.params['modelnames']:
sf.write("%s\t1\n" %
os.path.join(self.params['rundir'], model))
sf.close()
protocolPrm["SeedsSelfile"] = os.path.join(
self.params['rundir'], "reference_volumes.sel")
protocolPrm["DoGenerateSeeds"] = False
else:
protocolPrm["SeedsSelfile"] = ""
protocolPrm["DoGenerateSeeds"] = True
protocolPrm[
"InSelFile"] = "partlist.sel" ### this maybe should not be hardcoded
protocolPrm["WorkingDir"] = "ml3d"
protocolPrm["DoDeleteWorkingDir"] = False
protocolPrm["ProjectDir"] = self.params['recondir']
protocolPrm["LogDir"] = "Logs"
protocolPrm["DoMlf"] = self.params['DoMlf']
protocolPrm["DoCorrectAmplitudes"] = False
protocolPrm["InCtfDatFile"] = "all_images.ctfdat"
protocolPrm["HighResLimit"] = self.params['HighResLimit']
protocolPrm["ImagesArePhaseFlipped"] = self.params[
'ImagesArePhaseFlipped']
protocolPrm["InitialMapIsAmplitudeCorrected"] = False
protocolPrm["SeedsAreAmplitudeCorrected"] = False
protocolPrm["DoCorrectGreyScale"] = self.params['DoCorrectGreyScale']
protocolPrm["ProjMatchSampling"] = self.params['ProjMatchSampling']
if (self.params['LowPassFilter'] > 1) is True:
protocolPrm["DoLowPassFilterReference"] = True
protocolPrm["LowPassFilter"] = self.params['LowPassFilter']
else:
protocolPrm["DoLowPassFilterReference"] = False
protocolPrm["LowPassFilter"] = self.params['LowPassFilter']
protocolPrm["PixelSize"] = self.params['apix']
protocolPrm["NumberOfReferences"] = self.params['NumberOfReferences']
protocolPrm["DoJustRefine"] = False
protocolPrm["DoML3DClassification"] = True
protocolPrm["AngularSampling"] = self.params['AngularSampling']
protocolPrm["NumberOfIterations"] = self.params['numiter']
protocolPrm["Symmetry"] = self.params['symmetry'][0]
protocolPrm["DoNorm"] = False
protocolPrm["DoFourier"] = False
protocolPrm["RestartIter"] = 0
protocolPrm["ExtraParamsMLrefine3D"] = ""
protocolPrm["NumberOfThreads"] = 1
protocolPrm["DoParallel"] = self.params['nproc'] > 1
protocolPrm["NumberOfMpiProcesses"] = self.params['nproc']
protocolPrm["SystemFlavour"] = ""
protocolPrm["AnalysisScript"] = "visualize_ml3d.py"
### write out python protocol into run directory
protocolfile = os.path.join(self.params['remoterundir'],
"%s.py" % protocolname)
apXmipp.particularizeProtocol(protocol_ml3d, protocolPrm, protocolfile)
os.chmod(os.path.join(self.params['rundir'], protocolfile), 0775)
### Write the parameters for posterior uploading, both generic and specific
self.runparams = {}
self.runparams['reconstruction_package'] = "xmipp_ml3d"
self.runparams['remoterundir'] = self.params['remoterundir']
# self.runparams['reconstruction_working_dir'] = protocolPrm["WorkingDir"]
self.runparams['reconstruction_working_dir'] = protocolPrm[
'WorkingDir'] + "/RunML3D"
self.runparams['numiter'] = protocolPrm['NumberOfIterations']
self.runparams['NumberOfReferences'] = protocolPrm[
'NumberOfReferences']
self.runparams['symmetry'] = protocolPrm["Symmetry"]
self.runparams['package_params'] = protocolPrm
paramfile = os.path.join(
self.params['remoterundir'],
"xmipp_ml3d_" + self.timestamp + "-params.pickle")
apParam.dumpParameters(self.runparams, paramfile)
### finished setup of input files, now run xmipp_protocols_ml3d.py from jobfile
apDisplay.printMsg(
"finished setting up input files, now running xmipp_protocols_ml3d.py"
)
return protocolfile, protocolPrm
def start(self):
self.addToLog('.... Setting up new ISAC job ....')
self.addToLog('.... Making command for stack pre-processing ....')
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
### send file to remotehost
tasks = {}
sfhed = self.stack['file'][:-4]+".hed"
sfimg = self.stack['file'][:-4]+".img"
tasks = self.addToTasks(tasks,"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s" % (sfhed,self.params['localhost'],self.params['remoterundir'],"start1.hed"))
tasks = self.addToTasks(tasks,"rsync -e 'ssh -o StrictHostKeyChecking=no' -rotouv --partial %s %s:%s/%s" % (sfimg,self.params['localhost'],self.params['remoterundir'],"start1.img"))
### write Sparx jobfile: process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize)*self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
proccmd += " shrink=%d clip=%d,%d "%(self.params['bin'],clipsize,clipsize)
proccmd += " last="+str(self.params['numpart']-1)
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp="+str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp="+str(self.params['lowpass'])
# apParam.runCmd(proccmd, "EMAN", verbose=True)
self.addSimpleCommand('cd %s' % self.params['rundir'])
self.addSimpleCommand(proccmd)
sparxcmd = "sxcpy.py %s %s_1.hdf" % (self.params['localstack'], self.params['localstack'][:-4])
# apParam.runCmd(sparxcmd, "SPARX", verbose=True)
self.addSimpleCommand(sparxcmd)
self.addSimpleCommand("")
### write Sparx jobfile: run ISAC
for i in range(self.params['generations']):
sparxopts = " %s_%d.hdf" % (os.path.join(self.params['localstack'][:-4]), (i+1))
if self.params['ir'] is not None:
sparxopts += " --ir %d" % int(float(apRecon.getComponentFromVector(self.params['ir'], i, splitter=":")))
if self.params['ou'] is not None:
sparxopts += " --ou %d" % int(float(apRecon.getComponentFromVector(self.params['ou'], i, splitter=":")))
if self.params['rs'] is not None:
sparxopts += " --rs %d" % int(float(apRecon.getComponentFromVector(self.params['rs'], i, splitter=":")))
if self.params['ts'] is not None:
sparxopts += " --ts %.1f" % int(float(apRecon.getComponentFromVector(self.params['ts'], i, splitter=":")))
if self.params['xr'] is not None:
sparxopts += " --xr %.1f" % int(float(apRecon.getComponentFromVector(self.params['xr'], i, splitter=":")))
if self.params['yr'] is not None:
sparxopts += " --yr %.1f" % int(float(apRecon.getComponentFromVector(self.params['yr'], i, splitter=":")))
if self.params['maxit'] is not None:
sparxopts += " --maxit %d" % int(float(apRecon.getComponentFromVector(self.params['maxit'], i, splitter=":")))
if self.params['FL'] is not None:
sparxopts += " --FL %.2f" % int(float(apRecon.getComponentFromVector(self.params['FL'], i, splitter=":")))
if self.params['FH'] is not None:
sparxopts += " --FH %.2f" % int(float(apRecon.getComponentFromVector(self.params['FH'], i, splitter=":")))
if self.params['FF'] is not None:
sparxopts += " --FF %.2f" % int(float(apRecon.getComponentFromVector(self.params['FF'], i, splitter=":")))
if self.params['init_iter'] is not None:
sparxopts += " --init_iter %d" % int(float(apRecon.getComponentFromVector(self.params['init_iter'], i, splitter=":")))
if self.params['main_iter'] is not None:
sparxopts += " --main_iter %d" % int(float(apRecon.getComponentFromVector(self.params['main_iter'], i, splitter=":")))
if self.params['iter_reali'] is not None:
sparxopts += " --iter_reali %d" % int(float(apRecon.getComponentFromVector(self.params['iter_reali'], i, splitter=":")))
if self.params['match_first'] is not None:
sparxopts += " --match_first %d" % int(float(apRecon.getComponentFromVector(self.params['match_first'], i, splitter=":")))
if self.params['max_round'] is not None:
sparxopts += " --max_round %d" % int(float(apRecon.getComponentFromVector(self.params['max_round'], i, splitter=":")))
if self.params['match_second'] is not None:
sparxopts += " --match_second %d" % int(float(apRecon.getComponentFromVector(self.params['match_second'], i, splitter=":")))
if self.params['stab_ali'] is not None:
sparxopts += " --stab_ali %d" % int(float(apRecon.getComponentFromVector(self.params['stab_ali'], i, splitter=":")))
if self.params['thld_err'] is not None:
sparxopts += " --thld_err %.2f" % int(float(apRecon.getComponentFromVector(self.params['thld_err'], i, splitter=":")))
if self.params['indep_run'] is not None:
sparxopts += " --indep_run %d" % int(float(apRecon.getComponentFromVector(self.params['indep_run'], i, splitter=":")))
if self.params['thld_grp'] is not None:
sparxopts += " --thld_grp %d" % int(float(apRecon.getComponentFromVector(self.params['thld_grp'], i, splitter=":")))
if self.params['img_per_grp'] is not None:
sparxopts += " --img_per_grp %d" % int(apRecon.getComponentFromVector(self.params['img_per_grp'], i))
sparxopts += " --generation %d" % (i+1)
sparxexe = apParam.getExecPath("sxisac.py", die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+sparxexe+" "+sparxopts
bn = os.path.basename(self.params['localstack'])[:-4]
e2cmd = "e2proc2d.py %s_%d.hdf %s_%d.hdf --list=\"generation_%d_unaccounted.txt\"" % \
(bn, i+1, bn, i+2, i+1)
self.addSimpleCommand(mpiruncmd)
self.addSimpleCommand(e2cmd)
# print self.tasks
# print self.command_list
# self.writeCommandListToFile()
apParam.dumpParameters(self.params, "isac-"+self.params['timestamp']+"-params.pickle")
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apParam.runCmd(proccmd, "EMAN", verbose=True)
if self.params['numpart'] != apFile.numImagesInStack(
self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -codes " + str(self.params['numrefs']) + " -iter " +
str(self.params['maxiter']) + " -o " + os.path.join(
self.params['rundir'], "part" + self.params['timestamp']))
if self.params['fast']:
xmippopts += " -fast "
if self.params['correlation']:
xmippopts += " -useCorrelation "
if self.params['classical']:
xmippopts += " -classicalMultiref "
if self.params['align']:
xmippopts += " -alignImages "
### use multi-processor command
apDisplay.printColor(
"Using " + str(self.params['nproc']) + " processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True)
mpiruncmd = self.mpirun + " -np " + str(
self.params['nproc']) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd,
package="Xmipp",
verbose=True,
showcmd=True,
logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(self.params['runtime']))
### minor post-processing
self.createReferenceStack()
self.parseOutput()
self.clearIntermediateFiles()
# self.readyUploadFlag()
apParam.dumpParameters(
self.params, "cl2d-" + self.params['timestamp'] + "-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-" +
self.params['timestamp'] +
"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(
self.runparams['stackid']) * self.runparams['bin']
self.Nlevels = len(
glob.glob("part" + self.params['timestamp'] + "_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part" + self.params[
'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels - 1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part" + self.params['timestamp'] +
"_level_%02d_.hed" % level, classnum + 1,
partdict[classnum], len(partdict))