def start(self):
### database entry parameters
package_table = 'ApXmippML3DRefineIterData|xmippML3DParams'
### set ml3d path
self.ml3dpath = os.path.abspath(os.path.join(self.params['rundir'], "recon", self.runparams['package_params']['WorkingDir'], "RunML3D"))
### check for variable root directories between file systems
if os.path.split(self.runparams['remoterundir'])[1] == "recon":
self.runparams['remoterundir'] = os.path.split(self.runparams['remoterundir'])[0]
apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(self.params['rundir'], self.runparams['remoterundir'], self.params['rundir'])
### determine which iterations to upload
lastiter = self.findLastCompletedIteration()
uploadIterations = self.verifyUploadIterations(lastiter)
### create ml3d_lib.doc file somewhat of a workaround, but necessary to make projections
total_num_2d_classes = self.createModifiedLibFile()
### upload each iteration
for iteration in uploadIterations:
### set package parameters, as they will appear in database entries
package_database_object = self.instantiateML3DParamsData(iteration)
for j in range(self.runparams['NumberOfReferences']):
### calculate FSC for each iteration using split selfile (selfile requires root directory change)
self.calculateFSCforIteration(iteration, j+1)
### create a stack of class averages and reprojections (optional)
self.compute_stack_of_class_averages_and_reprojections(iteration, j+1)
### create a text file with particle information
self.createParticleDataFile(iteration, j+1, total_num_2d_classes)
### create mrc file of map for iteration and reference number
oldvol = os.path.join(self.ml3dpath, "ml3d_it%.6d_vol%.6d.vol" % (iteration, j+1))
newvol = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.mrc" % (self.params['timestamp'], iteration, j+1))
mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol, self.runparams['apix'])
apParam.runCmd(mrccmd, "EMAN")
### make chimera snapshot of volume
self.createChimeraVolumeSnapshot(newvol, iteration, j+1)
### instantiate database objects
self.insertRefinementRunData(iteration, j+1)
self.insertRefinementIterationData(iteration, package_table, package_database_object, j+1)
### calculate Euler jumps
if self.runparams['numiter'] > 1:
self.calculateEulerJumpsAndGoodBadParticles(uploadIterations)
### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists
### e.g. {1: [5, 4, 3, 2, 1], 2: [6, 5, 4, 3, 2, 1]} means 5 iters completed for refine 1 & 6 iters completed for refine 2
complete_refinements = self.verifyNumberOfCompletedRefinements(multiModelRefinementRun=True)
if self.params['cleanup_files'] is True:
self.cleanupFiles(complete_refinements)
def start(self):
### database entry parameters
package_table = 'ApXmippRefineIterData|xmippParams'
### set projection-matching path
self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], "recon", self.runparams['package_params']['WorkingDir']))
# self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], self.runparams['package_params']['WorkingDir']))
### check for variable root directories between file systems
apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(self.params['rundir'], self.runparams['remoterundir'], self.runparams['rundir'])
### determine which iterations to upload
lastiter = self.findLastCompletedIteration()
uploadIterations = self.verifyUploadIterations(lastiter)
### upload each iteration
for iteration in uploadIterations:
apDisplay.printColor("uploading iteration %d" % iteration, "cyan")
### set package parameters, as they will appear in database entries
package_database_object = self.instantiateProjMatchParamsData(iteration)
### move FSC file to results directory
oldfscfile = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_resolution.fsc" % iteration)
newfscfile = os.path.join(self.resultspath, "recon_%s_it%.3d_vol001.fsc" % (self.params['timestamp'],iteration))
if os.path.exists(oldfscfile):
shutil.copyfile(oldfscfile, newfscfile)
### create a stack of class averages and reprojections (optional)
self.compute_stack_of_class_averages_and_reprojections(iteration)
### create a text file with particle information
self.createParticleDataFile(iteration)
if not self.params['euleronly']:
### create mrc file of map for iteration and reference number
oldvol = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_reconstruction.vol" % iteration)
newvol = os.path.join(self.resultspath, "recon_%s_it%.3d_vol001.mrc" % (self.params['timestamp'], iteration))
mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol, self.runparams['apix'])
apParam.runCmd(mrccmd, "EMAN")
### make chimera snapshot of volume
self.createChimeraVolumeSnapshot(newvol, iteration)
### instantiate database objects
self.insertRefinementRunData(iteration)
self.insertRefinementIterationData(iteration, package_table, package_database_object)
### calculate Euler jumps
if self.runparams['numiter'] > 1:
self.calculateEulerJumpsAndGoodBadParticles(uploadIterations)
### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists
### e.g. {1: [5, 4, 3, 2, 1]} means 5 iters completed for refine 1
complete_refinements = self.verifyNumberOfCompletedRefinements(multiModelRefinementRun=False)
if self.params['cleanup_files'] is True:
self.cleanupFiles(complete_refinements)
def makePreIterationScript(self):
tasks = {}
self.addToLog('....Setting up Xmipp ML3D Protocol....')
protocolfile, protocolPrm = self.setupXmippML3DProtocol()
### check for variable root directories between file systems
apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(self.params['remoterundir'], self.params['rundir'], self.params['remoterundir'])
self.addToLog('....Start running Xmipp Protocol....')
tasks = self.addToTasks(tasks,'python %s' % protocolfile,self.calcRefineMem(),self.params['nproc'])
protocol_pyname = os.path.basename(protocolfile)
protocolname = protocol_pyname.split('.')[0]
tasklogfilename = protocolname+'_'+protocolPrm['WorkingDir']+'.log'
tasklogfile = os.path.join(self.params['recondir'],protocolPrm['LogDir'],tasklogfilename)
tasks = self.logTaskStatus(tasks,'protocol_run',tasklogfile)
tasks = self.addToTasks(tasks,'cp %s %s' % (protocolfile,self.params['recondir']))
self.addJobCommands(tasks)
def makePreIterationScript(self):
tasks = {}
self.addToLog('....Setting up Xmipp ML3D Protocol....')
protocolfile, protocolPrm = self.setupXmippML3DProtocol()
### check for variable root directories between file systems
apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(
self.params['remoterundir'], self.params['rundir'],
self.params['remoterundir'])
self.addToLog('....Start running Xmipp Protocol....')
tasks = self.addToTasks(tasks, 'python %s' % protocolfile,
self.calcRefineMem(), self.params['nproc'])
protocol_pyname = os.path.basename(protocolfile)
protocolname = protocol_pyname.split('.')[0]
tasklogfilename = protocolname + '_' + protocolPrm[
'WorkingDir'] + '.log'
tasklogfile = os.path.join(self.params['recondir'],
protocolPrm['LogDir'], tasklogfilename)
tasks = self.logTaskStatus(tasks, 'protocol_run', tasklogfile)
tasks = self.addToTasks(
tasks, 'cp %s %s' % (protocolfile, self.params['recondir']))
self.addJobCommands(tasks)
def start(self):
### database entry parameters
package_table = 'ApXmippRefineIterData|xmippParams'
### set projection-matching path
self.projmatchpath = os.path.abspath(
os.path.join(self.params['rundir'], "recon",
self.runparams['package_params']['WorkingDir']))
# self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], self.runparams['package_params']['WorkingDir']))
### check for variable root directories between file systems
apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(
self.params['rundir'], self.runparams['remoterundir'],
self.runparams['rundir'])
### determine which iterations to upload
lastiter = self.findLastCompletedIteration()
uploadIterations = self.verifyUploadIterations(lastiter)
### upload each iteration
for iteration in uploadIterations:
apDisplay.printColor("uploading iteration %d" % iteration, "cyan")
### set package parameters, as they will appear in database entries
package_database_object = self.instantiateProjMatchParamsData(
iteration)
### move FSC file to results directory
oldfscfile = os.path.join(self.projmatchpath,
"Iter_%d" % iteration,
"Iter_%d_resolution.fsc" % iteration)
newfscfile = os.path.join(
self.resultspath, "recon_%s_it%.3d_vol001.fsc" %
(self.params['timestamp'], iteration))
if os.path.exists(oldfscfile):
shutil.copyfile(oldfscfile, newfscfile)
### create a stack of class averages and reprojections (optional)
self.compute_stack_of_class_averages_and_reprojections(iteration)
### create a text file with particle information
self.createParticleDataFile(iteration)
if not self.params['euleronly']:
### create mrc file of map for iteration and reference number
oldvol = os.path.join(self.projmatchpath,
"Iter_%d" % iteration,
"Iter_%d_reconstruction.vol" % iteration)
newvol = os.path.join(
self.resultspath, "recon_%s_it%.3d_vol001.mrc" %
(self.params['timestamp'], iteration))
mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol,
self.runparams['apix'])
apParam.runCmd(mrccmd, "EMAN")
### make chimera snapshot of volume
self.createChimeraVolumeSnapshot(newvol, iteration)
### instantiate database objects
self.insertRefinementRunData(iteration)
self.insertRefinementIterationData(iteration, package_table,
package_database_object)
### calculate Euler jumps
if self.runparams['numiter'] > 1:
self.calculateEulerJumpsAndGoodBadParticles(uploadIterations)
### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists
### e.g. {1: [5, 4, 3, 2, 1]} means 5 iters completed for refine 1
complete_refinements = self.verifyNumberOfCompletedRefinements(
multiModelRefinementRun=False)
if self.params['cleanup_files'] is True:
self.cleanupFiles(complete_refinements)