def start(self): ### database entry parameters package_table = 'ApXmippML3DRefineIterData|xmippML3DParams' ### set ml3d path self.ml3dpath = os.path.abspath(os.path.join(self.params['rundir'], "recon", self.runparams['package_params']['WorkingDir'], "RunML3D")) ### check for variable root directories between file systems if os.path.split(self.runparams['remoterundir'])[1] == "recon": self.runparams['remoterundir'] = os.path.split(self.runparams['remoterundir'])[0] apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(self.params['rundir'], self.runparams['remoterundir'], self.params['rundir']) ### determine which iterations to upload lastiter = self.findLastCompletedIteration() uploadIterations = self.verifyUploadIterations(lastiter) ### create ml3d_lib.doc file somewhat of a workaround, but necessary to make projections total_num_2d_classes = self.createModifiedLibFile() ### upload each iteration for iteration in uploadIterations: ### set package parameters, as they will appear in database entries package_database_object = self.instantiateML3DParamsData(iteration) for j in range(self.runparams['NumberOfReferences']): ### calculate FSC for each iteration using split selfile (selfile requires root directory change) self.calculateFSCforIteration(iteration, j+1) ### create a stack of class averages and reprojections (optional) self.compute_stack_of_class_averages_and_reprojections(iteration, j+1) ### create a text file with particle information self.createParticleDataFile(iteration, j+1, total_num_2d_classes) ### create mrc file of map for iteration and reference number oldvol = os.path.join(self.ml3dpath, "ml3d_it%.6d_vol%.6d.vol" % (iteration, j+1)) newvol = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.mrc" % (self.params['timestamp'], iteration, j+1)) mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol, self.runparams['apix']) apParam.runCmd(mrccmd, "EMAN") ### make chimera snapshot of volume self.createChimeraVolumeSnapshot(newvol, iteration, j+1) ### instantiate database objects self.insertRefinementRunData(iteration, j+1) self.insertRefinementIterationData(iteration, package_table, package_database_object, j+1) ### calculate Euler jumps if self.runparams['numiter'] > 1: self.calculateEulerJumpsAndGoodBadParticles(uploadIterations) ### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists ### e.g. {1: [5, 4, 3, 2, 1], 2: [6, 5, 4, 3, 2, 1]} means 5 iters completed for refine 1 & 6 iters completed for refine 2 complete_refinements = self.verifyNumberOfCompletedRefinements(multiModelRefinementRun=True) if self.params['cleanup_files'] is True: self.cleanupFiles(complete_refinements)
def start(self): ### database entry parameters package_table = 'ApXmippRefineIterData|xmippParams' ### set projection-matching path self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], "recon", self.runparams['package_params']['WorkingDir'])) # self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], self.runparams['package_params']['WorkingDir'])) ### check for variable root directories between file systems apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(self.params['rundir'], self.runparams['remoterundir'], self.runparams['rundir']) ### determine which iterations to upload lastiter = self.findLastCompletedIteration() uploadIterations = self.verifyUploadIterations(lastiter) ### upload each iteration for iteration in uploadIterations: apDisplay.printColor("uploading iteration %d" % iteration, "cyan") ### set package parameters, as they will appear in database entries package_database_object = self.instantiateProjMatchParamsData(iteration) ### move FSC file to results directory oldfscfile = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_resolution.fsc" % iteration) newfscfile = os.path.join(self.resultspath, "recon_%s_it%.3d_vol001.fsc" % (self.params['timestamp'],iteration)) if os.path.exists(oldfscfile): shutil.copyfile(oldfscfile, newfscfile) ### create a stack of class averages and reprojections (optional) self.compute_stack_of_class_averages_and_reprojections(iteration) ### create a text file with particle information self.createParticleDataFile(iteration) if not self.params['euleronly']: ### create mrc file of map for iteration and reference number oldvol = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_reconstruction.vol" % iteration) newvol = os.path.join(self.resultspath, "recon_%s_it%.3d_vol001.mrc" % (self.params['timestamp'], iteration)) mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol, self.runparams['apix']) apParam.runCmd(mrccmd, "EMAN") ### make chimera snapshot of volume self.createChimeraVolumeSnapshot(newvol, iteration) ### instantiate database objects self.insertRefinementRunData(iteration) self.insertRefinementIterationData(iteration, package_table, package_database_object) ### calculate Euler jumps if self.runparams['numiter'] > 1: self.calculateEulerJumpsAndGoodBadParticles(uploadIterations) ### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists ### e.g. {1: [5, 4, 3, 2, 1]} means 5 iters completed for refine 1 complete_refinements = self.verifyNumberOfCompletedRefinements(multiModelRefinementRun=False) if self.params['cleanup_files'] is True: self.cleanupFiles(complete_refinements)
def makePreIterationScript(self): tasks = {} self.addToLog('....Setting up Xmipp ML3D Protocol....') protocolfile, protocolPrm = self.setupXmippML3DProtocol() ### check for variable root directories between file systems apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(self.params['remoterundir'], self.params['rundir'], self.params['remoterundir']) self.addToLog('....Start running Xmipp Protocol....') tasks = self.addToTasks(tasks,'python %s' % protocolfile,self.calcRefineMem(),self.params['nproc']) protocol_pyname = os.path.basename(protocolfile) protocolname = protocol_pyname.split('.')[0] tasklogfilename = protocolname+'_'+protocolPrm['WorkingDir']+'.log' tasklogfile = os.path.join(self.params['recondir'],protocolPrm['LogDir'],tasklogfilename) tasks = self.logTaskStatus(tasks,'protocol_run',tasklogfile) tasks = self.addToTasks(tasks,'cp %s %s' % (protocolfile,self.params['recondir'])) self.addJobCommands(tasks)
def makePreIterationScript(self): tasks = {} self.addToLog('....Setting up Xmipp ML3D Protocol....') protocolfile, protocolPrm = self.setupXmippML3DProtocol() ### check for variable root directories between file systems apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree( self.params['remoterundir'], self.params['rundir'], self.params['remoterundir']) self.addToLog('....Start running Xmipp Protocol....') tasks = self.addToTasks(tasks, 'python %s' % protocolfile, self.calcRefineMem(), self.params['nproc']) protocol_pyname = os.path.basename(protocolfile) protocolname = protocol_pyname.split('.')[0] tasklogfilename = protocolname + '_' + protocolPrm[ 'WorkingDir'] + '.log' tasklogfile = os.path.join(self.params['recondir'], protocolPrm['LogDir'], tasklogfilename) tasks = self.logTaskStatus(tasks, 'protocol_run', tasklogfile) tasks = self.addToTasks( tasks, 'cp %s %s' % (protocolfile, self.params['recondir'])) self.addJobCommands(tasks)
def start(self): ### database entry parameters package_table = 'ApXmippRefineIterData|xmippParams' ### set projection-matching path self.projmatchpath = os.path.abspath( os.path.join(self.params['rundir'], "recon", self.runparams['package_params']['WorkingDir'])) # self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], self.runparams['package_params']['WorkingDir'])) ### check for variable root directories between file systems apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree( self.params['rundir'], self.runparams['remoterundir'], self.runparams['rundir']) ### determine which iterations to upload lastiter = self.findLastCompletedIteration() uploadIterations = self.verifyUploadIterations(lastiter) ### upload each iteration for iteration in uploadIterations: apDisplay.printColor("uploading iteration %d" % iteration, "cyan") ### set package parameters, as they will appear in database entries package_database_object = self.instantiateProjMatchParamsData( iteration) ### move FSC file to results directory oldfscfile = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_resolution.fsc" % iteration) newfscfile = os.path.join( self.resultspath, "recon_%s_it%.3d_vol001.fsc" % (self.params['timestamp'], iteration)) if os.path.exists(oldfscfile): shutil.copyfile(oldfscfile, newfscfile) ### create a stack of class averages and reprojections (optional) self.compute_stack_of_class_averages_and_reprojections(iteration) ### create a text file with particle information self.createParticleDataFile(iteration) if not self.params['euleronly']: ### create mrc file of map for iteration and reference number oldvol = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_reconstruction.vol" % iteration) newvol = os.path.join( self.resultspath, "recon_%s_it%.3d_vol001.mrc" % (self.params['timestamp'], iteration)) mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol, self.runparams['apix']) apParam.runCmd(mrccmd, "EMAN") ### make chimera snapshot of volume self.createChimeraVolumeSnapshot(newvol, iteration) ### instantiate database objects self.insertRefinementRunData(iteration) self.insertRefinementIterationData(iteration, package_table, package_database_object) ### calculate Euler jumps if self.runparams['numiter'] > 1: self.calculateEulerJumpsAndGoodBadParticles(uploadIterations) ### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists ### e.g. {1: [5, 4, 3, 2, 1]} means 5 iters completed for refine 1 complete_refinements = self.verifyNumberOfCompletedRefinements( multiModelRefinementRun=False) if self.params['cleanup_files'] is True: self.cleanupFiles(complete_refinements)