def start(self): """ this is the main component of the script where all the processing is done """ ### initialize some variables self.runq = None self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) apDisplay.printMsg("Pixel size: %.5f"%(self.apix)) self.boxsize = apStack.getStackBoxsize(self.params['stackid']) apDisplay.printMsg("Box size: %d"%(self.boxsize)) self.checkResults() self.stackmapping = apRecon.partnum2defid(self.params['stackid']) self.numiter = self.getNumberOfIterations() for i in range(self.numiter): iternum = i+1 apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green") self.uploadIteration(iternum) reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir']) if reconrunid: apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid']) apRecon.setGoodBadParticlesFromReconId(reconrunid) else: apDisplay.printWarning("Could not find recon run id")
def calculateEulerJumpsAndGoodBadParticles(self, uploadIterations): ''' calculate euler jumps for entire recon, currently based on EMAN ZXZ convention ''' if self.params['commit'] is True: reconrunid = self.refinerunq.dbid ### make table entries of good-bad particles apRecon.setGoodBadParticlesFromReconId(reconrunid) ### verify completed refinements and iterations refinecomplete = self.verifyNumberOfCompletedRefinements(multiModelRefinementRun=self.multiModelRefinementRun) if self.multiModelRefinementRun is True: ### Euler jumpers calculated from ApMultiModelRefineRunData in multi-model case, looping over values in all single-model refinements multimodelrunid = self.multimodelq.dbid if len(uploadIterations) > 1: apDisplay.printMsg("calculating euler jumpers for multi-model refinement="+str(multimodelrunid)) eulerjump = apEulerJump.ApEulerJump() ### TECHNICALLY, IT DOESN'T MAKE SENSE TO PASS THE RECONRUNID, SINCE PARTICLES CAN BE JUMPING, ### BUT AS FAR AS I CAN TELL, IT DOESN'T MAKE A DIFFERENCE IN THE RESULTING QUERY, SINCE THE VARIABLE ### IS ONLY USED TO QUERY FOR STACK PARTICLES, WHICH ARE IDENTICAL IN MULTI-MODEL AND SINGLE-MODEL REFINEMENT ### CASES. THEREFORE, THE LAST RECONRUNID IS PASSES ... * DMITRY eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.runparams['stackid'], multimodelrunid=multimodelrunid) else: ### Euler jumpers calculated from ApRefineRunData in single-model case if len(uploadIterations) > 1 or \ (refinecomplete.itervalues().next()[-1] == self.runparams['numiter'] and len(refinecomplete.itervalues().next())>1): apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.runparams['stackid']) return
def calculateEulerJumpsAndGoodBadParticles(self, uploadIterations): ''' calculate euler jumps for entire recon, currently based on EMAN ZXZ convention ''' if self.params['commit'] is True: reconrunid = self.refinerunq.dbid ### make table entries of good-bad particles apRecon.setGoodBadParticlesFromReconId(reconrunid) ### verify completed refinements and iterations refinecomplete = self.verifyNumberOfCompletedRefinements( multiModelRefinementRun=self.multiModelRefinementRun) if self.multiModelRefinementRun is True: ### Euler jumpers calculated from ApMultiModelRefineRunData in multi-model case, looping over values in all single-model refinements multimodelrunid = self.multimodelq.dbid if len(uploadIterations) > 1: apDisplay.printMsg( "calculating euler jumpers for multi-model refinement=" + str(multimodelrunid)) eulerjump = apEulerJump.ApEulerJump() ### TECHNICALLY, IT DOESN'T MAKE SENSE TO PASS THE RECONRUNID, SINCE PARTICLES CAN BE JUMPING, ### BUT AS FAR AS I CAN TELL, IT DOESN'T MAKE A DIFFERENCE IN THE RESULTING QUERY, SINCE THE VARIABLE ### IS ONLY USED TO QUERY FOR STACK PARTICLES, WHICH ARE IDENTICAL IN MULTI-MODEL AND SINGLE-MODEL REFINEMENT ### CASES. THEREFORE, THE LAST RECONRUNID IS PASSES ... * DMITRY eulerjump.calculateEulerJumpsForEntireRecon( reconrunid, self.runparams['stackid'], multimodelrunid=multimodelrunid) else: ### Euler jumpers calculated from ApRefineRunData in single-model case if len(uploadIterations) > 1 or \ (refinecomplete.itervalues().next()[-1] == self.runparams['numiter'] and len(refinecomplete.itervalues().next())>1): apDisplay.printMsg("calculating euler jumpers for recon=" + str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon( reconrunid, self.runparams['stackid']) return
selectq = " SELECT * FROM `install` WHERE `key`='version'" values = projectdb.returnCustomSQL(selectq) if values: projectdb.updateColumn("install", "value", "'2.0'", "install.key = 'version'",timestamp=False) else: insertq = "INSERT INTO `install` (`key`, `value`) VALUES ('version', '2.0')" projectdb.executeCustomSQL(insertq) if __name__ == "__main__": projectids = getProjectIds() projectdb = dbupgrade.DBUpgradeTools('projectdata', drop=True) for projectid in projectids: appiondbname = apProject.getAppionDBFromProjectId(projectid, die=False) if appiondbname is None: continue elif not projectdb.databaseExists(appiondbname): print "\033[31merror database %s does not exist\033[0m"%(appiondbname) time.sleep(1) continue if apProject.setDBfromProjectId(projectid, die=False): reconrunids = getReconRunsIds() for reconrunid in reconrunids: apRecon.setGoodBadParticlesFromReconId(reconrunid) # Tomography upgrades createExcludedColumnByFakeInsert() allalignrundata = getTomoAlignmentRuns() for alignrun in allalignrundata: insertTiltsInAlign(alignrun) upgradeProjectDB(projectdb,backup=False)
def start(self): if self.params['rundir'] is None or not os.path.isdir( self.params['rundir']): apDisplay.printError("upload directory does not exist: " + str(self.params['rundir'])) ### create temp directory for extracting data self.params['tmpdir'] = os.path.join(self.params['rundir'], "temp") apParam.createDirectory(self.params['tmpdir'], warning=True) ### make sure that the stack & model IDs exist in database emanJobFile = self.findEmanJobFile() self.params['stack'] = apStack.getOnlyStackData(self.params['stackid']) self.stackmapping = apRecon.partnum2defid(self.params['stackid']) self.params['model'] = appiondata.ApInitialModelData.direct_query( self.params['modelid']) self.params['boxsize'] = apStack.getStackBoxsize( self.params['stackid']) ### parse out the refinement parameters from the log file self.parseLogFile() ### parse out the message passing subclassification parameters from the job/log file if self.params['package'] == 'EMAN/MsgP': self.parseMsgPassingParams() ### convert class average files from old to new format self.convertClassAvgFiles() ### get a list of the files in the directory self.listFiles() ### create a refinementRun entry in the database self.insertRefinementRun() if self.params['euleronly'] is False: ### insert the Iteration info for iteration in self.iterationdatas: ### if only uploading one iteration, skip to that one if self.params['oneiter'] and int( iteration['num']) != self.params['oneiter']: continue ### if beginning at later iteration, skip to that one if self.params['startiter'] and int( iteration['num']) < self.params['startiter']: continue ### if beginning at later iteration, skip to that one if self.params['enditer'] and int( iteration['num']) > self.params['enditer']: continue apDisplay.printColor( "\nUploading iteration " + str(iteration['num']) + " of " + str(len(self.iterationdatas)) + "\n", "green") for i in range(75): sys.stderr.write("#") sys.stderr.write("\n") self.insertIteration(iteration) ### calculate euler jumps if self.params['commit'] is True: reconrunid = self.params['refineRun'].dbid stackid = self.params['stack'].dbid if self.params['oneiter'] is None and len(self.iterationdatas) > 1: apDisplay.printMsg("calculating euler jumpers for recon=" + str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon( reconrunid, stackid) ### coran keep plot if self.params['package'] == 'EMAN/SpiCoran': apCoranPlot.makeCoranKeepPlot(reconrunid) apRecon.setGoodBadParticlesFromReconId(reconrunid)
def start(self): if self.params['rundir'] is None or not os.path.isdir(self.params['rundir']): apDisplay.printError("upload directory does not exist: "+str(self.params['rundir'])) ### create temp directory for extracting data self.params['tmpdir'] = os.path.join(self.params['rundir'], "temp") apParam.createDirectory(self.params['tmpdir'], warning=True) ### make sure that the stack & model IDs exist in database emanJobFile = self.findEmanJobFile() self.params['stack'] = apStack.getOnlyStackData(self.params['stackid']) self.stackmapping = apRecon.partnum2defid(self.params['stackid']) self.params['model'] = appiondata.ApInitialModelData.direct_query(self.params['modelid']) self.params['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) ### parse out the refinement parameters from the log file self.parseLogFile() ### parse out the message passing subclassification parameters from the job/log file if self.params['package'] == 'EMAN/MsgP': self.parseMsgPassingParams() ### convert class average files from old to new format self.convertClassAvgFiles() ### get a list of the files in the directory self.listFiles() ### create a refinementRun entry in the database self.insertRefinementRun() if self.params['euleronly'] is False: ### insert the Iteration info for iteration in self.iterationdatas: ### if only uploading one iteration, skip to that one if self.params['oneiter'] and int(iteration['num']) != self.params['oneiter']: continue ### if beginning at later iteration, skip to that one if self.params['startiter'] and int(iteration['num']) < self.params['startiter']: continue ### if beginning at later iteration, skip to that one if self.params['enditer'] and int(iteration['num']) > self.params['enditer']: continue apDisplay.printColor("\nUploading iteration "+str(iteration['num'])+" of " +str(len(self.iterationdatas))+"\n", "green") for i in range(75): sys.stderr.write("#") sys.stderr.write("\n") self.insertIteration(iteration) ### calculate euler jumps if self.params['commit'] is True: reconrunid = self.params['refineRun'].dbid stackid = self.params['stack'].dbid if self.params['oneiter'] is None and len(self.iterationdatas) > 1: apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, stackid) ### coran keep plot if self.params['package']=='EMAN/SpiCoran': apCoranPlot.makeCoranKeepPlot(reconrunid) apRecon.setGoodBadParticlesFromReconId(reconrunid)
"'2.0'", "install.key = 'version'", timestamp=False) else: insertq = "INSERT INTO `install` (`key`, `value`) VALUES ('version', '2.0')" projectdb.executeCustomSQL(insertq) if __name__ == "__main__": projectids = getProjectIds() projectdb = dbupgrade.DBUpgradeTools('projectdata', drop=True) for projectid in projectids: appiondbname = apProject.getAppionDBFromProjectId(projectid, die=False) if appiondbname is None: continue elif not projectdb.databaseExists(appiondbname): print "\033[31merror database %s does not exist\033[0m" % ( appiondbname) time.sleep(1) continue if apProject.setDBfromProjectId(projectid, die=False): reconrunids = getReconRunsIds() for reconrunid in reconrunids: apRecon.setGoodBadParticlesFromReconId(reconrunid) # Tomography upgrades createExcludedColumnByFakeInsert() allalignrundata = getTomoAlignmentRuns() for alignrun in allalignrundata: insertTiltsInAlign(alignrun) upgradeProjectDB(projectdb, backup=False)