Example #1
0
	def start(self):
		"""
		this is the main component of the script
		where all the processing is done
		"""
		### initialize some variables
		self.runq = None
		self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
		apDisplay.printMsg("Pixel size: %.5f"%(self.apix))
		self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
		apDisplay.printMsg("Box size: %d"%(self.boxsize))

		self.checkResults()
		self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
		self.numiter = self.getNumberOfIterations()
		for i in range(self.numiter):
			iternum = i+1
			apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green")
			self.uploadIteration(iternum)

		reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir'])
		if reconrunid:
			apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
			eulerjump = apEulerJump.ApEulerJump()
			eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid'])
			apRecon.setGoodBadParticlesFromReconId(reconrunid)
		else:
			apDisplay.printWarning("Could not find recon run id")
	def start(self):
		"""
		this is the main component of the script
		where all the processing is done
		"""
		### initialize some variables
		self.runq = None
		self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
		apDisplay.printMsg("Pixel size: %.5f"%(self.apix))
		self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
		apDisplay.printMsg("Box size: %d"%(self.boxsize))

		self.checkResults()
		self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
		self.numiter = self.getNumberOfIterations()
		for i in range(self.numiter):
			iternum = i+1
			apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green")
			self.uploadIteration(iternum)

		reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir'])
		if reconrunid:
			apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
			eulerjump = apEulerJump.ApEulerJump()
			eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid'])
			apRecon.setGoodBadParticlesFromReconId(reconrunid)
		else:
			apDisplay.printWarning("Could not find recon run id")
	def calculateEulerJumpsAndGoodBadParticles(self, uploadIterations):
		''' calculate euler jumps for entire recon, currently based on EMAN ZXZ convention '''
		
		if self.params['commit'] is True:
			reconrunid = self.refinerunq.dbid
					
			### make table entries of good-bad particles
			apRecon.setGoodBadParticlesFromReconId(reconrunid)

			### verify completed refinements and iterations
			refinecomplete = self.verifyNumberOfCompletedRefinements(multiModelRefinementRun=self.multiModelRefinementRun)
			
			if self.multiModelRefinementRun is True: 
				### Euler jumpers calculated from ApMultiModelRefineRunData in multi-model case, looping over values in all single-model refinements			
				multimodelrunid = self.multimodelq.dbid
				if len(uploadIterations) > 1:
					apDisplay.printMsg("calculating euler jumpers for multi-model refinement="+str(multimodelrunid))
					eulerjump = apEulerJump.ApEulerJump()
					### TECHNICALLY, IT DOESN'T MAKE SENSE TO PASS THE RECONRUNID, SINCE PARTICLES CAN BE JUMPING,
					### BUT AS FAR AS I CAN TELL, IT DOESN'T MAKE A DIFFERENCE IN THE RESULTING QUERY, SINCE THE VARIABLE
					### IS ONLY USED TO QUERY FOR STACK PARTICLES, WHICH ARE IDENTICAL IN MULTI-MODEL AND SINGLE-MODEL REFINEMENT
					### CASES. THEREFORE, THE LAST RECONRUNID IS PASSES ... * DMITRY
					eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.runparams['stackid'], multimodelrunid=multimodelrunid)
			else:
				### Euler jumpers calculated from ApRefineRunData in single-model case
				if len(uploadIterations) > 1 or \
				(refinecomplete.itervalues().next()[-1] == self.runparams['numiter'] and len(refinecomplete.itervalues().next())>1): 
					apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
					eulerjump = apEulerJump.ApEulerJump()
					eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.runparams['stackid'])
		
		return
Example #4
0
    def calculateEulerJumpsAndGoodBadParticles(self, uploadIterations):
        ''' calculate euler jumps for entire recon, currently based on EMAN ZXZ convention '''

        if self.params['commit'] is True:
            reconrunid = self.refinerunq.dbid

            ### make table entries of good-bad particles
            apRecon.setGoodBadParticlesFromReconId(reconrunid)

            ### verify completed refinements and iterations
            refinecomplete = self.verifyNumberOfCompletedRefinements(
                multiModelRefinementRun=self.multiModelRefinementRun)

            if self.multiModelRefinementRun is True:
                ### Euler jumpers calculated from ApMultiModelRefineRunData in multi-model case, looping over values in all single-model refinements
                multimodelrunid = self.multimodelq.dbid
                if len(uploadIterations) > 1:
                    apDisplay.printMsg(
                        "calculating euler jumpers for multi-model refinement="
                        + str(multimodelrunid))
                    eulerjump = apEulerJump.ApEulerJump()
                    ### TECHNICALLY, IT DOESN'T MAKE SENSE TO PASS THE RECONRUNID, SINCE PARTICLES CAN BE JUMPING,
                    ### BUT AS FAR AS I CAN TELL, IT DOESN'T MAKE A DIFFERENCE IN THE RESULTING QUERY, SINCE THE VARIABLE
                    ### IS ONLY USED TO QUERY FOR STACK PARTICLES, WHICH ARE IDENTICAL IN MULTI-MODEL AND SINGLE-MODEL REFINEMENT
                    ### CASES. THEREFORE, THE LAST RECONRUNID IS PASSES ... * DMITRY
                    eulerjump.calculateEulerJumpsForEntireRecon(
                        reconrunid,
                        self.runparams['stackid'],
                        multimodelrunid=multimodelrunid)
            else:
                ### Euler jumpers calculated from ApRefineRunData in single-model case
                if len(uploadIterations) > 1 or \
                (refinecomplete.itervalues().next()[-1] == self.runparams['numiter'] and len(refinecomplete.itervalues().next())>1):
                    apDisplay.printMsg("calculating euler jumpers for recon=" +
                                       str(reconrunid))
                    eulerjump = apEulerJump.ApEulerJump()
                    eulerjump.calculateEulerJumpsForEntireRecon(
                        reconrunid, self.runparams['stackid'])

        return
        selectq = " SELECT * FROM `install` WHERE `key`='version'"
        values = projectdb.returnCustomSQL(selectq)
        if values:
                projectdb.updateColumn("install", "value", "'2.0'", 
                        "install.key = 'version'",timestamp=False)
        else:
                insertq = "INSERT INTO `install` (`key`, `value`) VALUES ('version', '2.0')"
                projectdb.executeCustomSQL(insertq)

if __name__ == "__main__":
        projectids = getProjectIds()
        projectdb = dbupgrade.DBUpgradeTools('projectdata', drop=True)
        for projectid in projectids:
                appiondbname = apProject.getAppionDBFromProjectId(projectid, die=False)
                if appiondbname is None:
                        continue
                elif not projectdb.databaseExists(appiondbname):
                        print "\033[31merror database %s does not exist\033[0m"%(appiondbname)
                        time.sleep(1)
                        continue
                if apProject.setDBfromProjectId(projectid, die=False):
                        reconrunids = getReconRunsIds()
                        for reconrunid in reconrunids:
                                apRecon.setGoodBadParticlesFromReconId(reconrunid)
                        # Tomography upgrades
                        createExcludedColumnByFakeInsert()
                        allalignrundata = getTomoAlignmentRuns()
                        for alignrun in allalignrundata:
                                insertTiltsInAlign(alignrun)
        upgradeProjectDB(projectdb,backup=False)
Example #6
0
    def start(self):
        if self.params['rundir'] is None or not os.path.isdir(
                self.params['rundir']):
            apDisplay.printError("upload directory does not exist: " +
                                 str(self.params['rundir']))

        ### create temp directory for extracting data
        self.params['tmpdir'] = os.path.join(self.params['rundir'], "temp")
        apParam.createDirectory(self.params['tmpdir'], warning=True)

        ### make sure that the stack & model IDs exist in database
        emanJobFile = self.findEmanJobFile()
        self.params['stack'] = apStack.getOnlyStackData(self.params['stackid'])
        self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
        self.params['model'] = appiondata.ApInitialModelData.direct_query(
            self.params['modelid'])
        self.params['boxsize'] = apStack.getStackBoxsize(
            self.params['stackid'])

        ### parse out the refinement parameters from the log file
        self.parseLogFile()

        ### parse out the message passing subclassification parameters from the job/log file
        if self.params['package'] == 'EMAN/MsgP':
            self.parseMsgPassingParams()

        ### convert class average files from old to new format
        self.convertClassAvgFiles()

        ### get a list of the files in the directory
        self.listFiles()

        ### create a refinementRun entry in the database
        self.insertRefinementRun()

        if self.params['euleronly'] is False:
            ### insert the Iteration info
            for iteration in self.iterationdatas:
                ### if only uploading one iteration, skip to that one
                if self.params['oneiter'] and int(
                        iteration['num']) != self.params['oneiter']:
                    continue
                ### if beginning at later iteration, skip to that one
                if self.params['startiter'] and int(
                        iteration['num']) < self.params['startiter']:
                    continue
                ### if beginning at later iteration, skip to that one
                if self.params['enditer'] and int(
                        iteration['num']) > self.params['enditer']:
                    continue
                apDisplay.printColor(
                    "\nUploading iteration " + str(iteration['num']) + " of " +
                    str(len(self.iterationdatas)) + "\n", "green")
                for i in range(75):
                    sys.stderr.write("#")
                sys.stderr.write("\n")
                self.insertIteration(iteration)

        ### calculate euler jumps
        if self.params['commit'] is True:
            reconrunid = self.params['refineRun'].dbid
            stackid = self.params['stack'].dbid
            if self.params['oneiter'] is None and len(self.iterationdatas) > 1:
                apDisplay.printMsg("calculating euler jumpers for recon=" +
                                   str(reconrunid))
                eulerjump = apEulerJump.ApEulerJump()
                eulerjump.calculateEulerJumpsForEntireRecon(
                    reconrunid, stackid)
            ### coran keep plot
            if self.params['package'] == 'EMAN/SpiCoran':
                apCoranPlot.makeCoranKeepPlot(reconrunid)
            apRecon.setGoodBadParticlesFromReconId(reconrunid)
	def start(self):
		if self.params['rundir'] is None or not os.path.isdir(self.params['rundir']):
			apDisplay.printError("upload directory does not exist: "+str(self.params['rundir']))


		### create temp directory for extracting data
		self.params['tmpdir'] = os.path.join(self.params['rundir'], "temp")
		apParam.createDirectory(self.params['tmpdir'], warning=True)

		### make sure that the stack & model IDs exist in database
		emanJobFile = self.findEmanJobFile()
		self.params['stack'] = apStack.getOnlyStackData(self.params['stackid'])
		self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
		self.params['model'] = appiondata.ApInitialModelData.direct_query(self.params['modelid'])
		self.params['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])

		### parse out the refinement parameters from the log file
		self.parseLogFile()

		### parse out the message passing subclassification parameters from the job/log file
		if self.params['package'] == 'EMAN/MsgP':
			self.parseMsgPassingParams()

		### convert class average files from old to new format
		self.convertClassAvgFiles()

		### get a list of the files in the directory
		self.listFiles()

		### create a refinementRun entry in the database
		self.insertRefinementRun()

		if self.params['euleronly'] is False:
			### insert the Iteration info
			for iteration in self.iterationdatas:
				### if only uploading one iteration, skip to that one
				if self.params['oneiter'] and int(iteration['num']) != self.params['oneiter']:
					continue
				### if beginning at later iteration, skip to that one
				if self.params['startiter'] and int(iteration['num']) < self.params['startiter']:
					continue
				### if beginning at later iteration, skip to that one
				if self.params['enditer'] and int(iteration['num']) > self.params['enditer']:
					continue
				apDisplay.printColor("\nUploading iteration "+str(iteration['num'])+" of "
					+str(len(self.iterationdatas))+"\n", "green")
				for i in range(75):
					sys.stderr.write("#")
				sys.stderr.write("\n")
				self.insertIteration(iteration)

		### calculate euler jumps
		if self.params['commit'] is True:
			reconrunid = self.params['refineRun'].dbid
			stackid = self.params['stack'].dbid
			if self.params['oneiter'] is None and len(self.iterationdatas) > 1:
				apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
				eulerjump = apEulerJump.ApEulerJump()
				eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, stackid)
			### coran keep plot
			if self.params['package']=='EMAN/SpiCoran':
				apCoranPlot.makeCoranKeepPlot(reconrunid)
			apRecon.setGoodBadParticlesFromReconId(reconrunid)
                               "'2.0'",
                               "install.key = 'version'",
                               timestamp=False)
    else:
        insertq = "INSERT INTO `install` (`key`, `value`) VALUES ('version', '2.0')"
        projectdb.executeCustomSQL(insertq)


if __name__ == "__main__":
    projectids = getProjectIds()
    projectdb = dbupgrade.DBUpgradeTools('projectdata', drop=True)
    for projectid in projectids:
        appiondbname = apProject.getAppionDBFromProjectId(projectid, die=False)
        if appiondbname is None:
            continue
        elif not projectdb.databaseExists(appiondbname):
            print "\033[31merror database %s does not exist\033[0m" % (
                appiondbname)
            time.sleep(1)
            continue
        if apProject.setDBfromProjectId(projectid, die=False):
            reconrunids = getReconRunsIds()
            for reconrunid in reconrunids:
                apRecon.setGoodBadParticlesFromReconId(reconrunid)
            # Tomography upgrades
            createExcludedColumnByFakeInsert()
            allalignrundata = getTomoAlignmentRuns()
            for alignrun in allalignrundata:
                insertTiltsInAlign(alignrun)
    upgradeProjectDB(projectdb, backup=False)