def checkConflicts(self):
		
		### setup correct database after we have read the project id
		if 'projectid' in self.params and self.params['projectid'] is not None:
			apDisplay.printMsg("Using split database")
			# use a project database
			newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
			sinedon.setConfig('appiondata', db=newdbname)
			apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
		
		# DD processes
		self.dd = apDDprocess.DDStackProcessing()
		print self.dd
	
		# get stack data
		self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid'])
		self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
		self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
		
		# query image
		qimage = self.stackparts[0]['particle']['image']

		# DD info
		self.dd.setImageData(qimage)
		self.dd.setDDStackRun(self.params['ddstackid'])
		self.ddstackpath = self.dd.getDDStackRun()['path']['path']
    def checkConflicts(self):

        ### setup correct database after we have read the project id
        if 'projectid' in self.params and self.params['projectid'] is not None:
            apDisplay.printMsg("Using split database")
            # use a project database
            newdbname = apProject.getAppionDBFromProjectId(
                self.params['projectid'])
            sinedon.setConfig('appiondata', db=newdbname)
            apDisplay.printColor("Connected to database: '" + newdbname + "'",
                                 "green")

        # DD processes
        self.dd = apDDprocess.DDStackProcessing()
        print self.dd

        # get stack data
        self.stackdata = appiondata.ApStackData.direct_query(
            self.params['stackid'])
        self.stackparts = apStack.getStackParticlesFromId(
            self.params['stackid'], msg=True)
        self.sessiondata = apStack.getSessionDataFromStackId(
            self.params['stackid'])

        # query image
        qimage = self.stackparts[0]['particle']['image']

        # DD info
        self.dd.setImageData(qimage)
        self.dd.setDDStackRun(self.params['ddstackid'])
        self.ddstackpath = self.dd.getDDStackRun()['path']['path']
	def checkConflicts(self):

		### setup correct database after we have read the project id
		if 'projectid' in self.params and self.params['projectid'] is not None:
			apDisplay.printMsg("Using split database")
			# use a project database
			newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
			sinedon.setConfig('appiondata', db=newdbname)
			apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
		
		### get stack data
		self.stack = {}
		if self.params['stackid'] is not None:
			self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
			self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
			self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
			self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
		else:
			self.stack['data'] = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
			self.stack['apix'] = self.stack['pixelsize']
			self.stack['boxsize'] = self.stack['boxsize']
			self.stack['file'] = self.stack['imagicfile']

		### check conflicts
		if self.params['stackid'] is None and self.params['alignstackid'] is None:
			apDisplay.printError("stack id OR alignstack id was not defined")
		if self.params['stackid'] is not None and self.params['alignstackid'] is not None:
			apDisplay.printError("either specify stack id OR alignstack id, not both")
		if self.params['generations'] is None:
			apDisplay.printError("number of generations was not provided")
		maxparticles = 500000
		if self.params['numpart'] > maxparticles:
			apDisplay.printError("too many particles requested, max: "
				+ str(maxparticles) + " requested: " + str(self.params['numpart']))
		if self.params['numpart'] > apFile.numImagesInStack(self.stack['file']):
			apDisplay.printError("trying to use more particles "+str(self.params['numpart'])
				+" than available "+str(apFile.numImagesInStack(self.stack['file'])))
		if self.params['numpart'] is None:
			self.params['numpart'] = apFile.numImagesInStack(self.stack['file'])

		boxsize = apStack.getStackBoxsize(self.params['stackid'])
		if self.params['ou'] is None:
			self.params['ou'] = (boxsize / 2.0) - 2
		self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2
		if self.params['clipsize'] is not None:
			if self.params['clipsize'] > self.clipsize:
				apDisplay.printError("requested clipsize is too big %d > %d"
					%(self.params['clipsize'],self.clipsize))
			self.clipsize = self.params['clipsize']
		self.mpirun = self.checkMPI()
		if self.mpirun is None:
			apDisplay.printError("There is no MPI installed")
    def checkConflicts(self):

        ### setup correct database after we have read the project id
        if 'projectid' in self.params and self.params['projectid'] is not None:
            apDisplay.printMsg("Using split database")
            # use a project database
            newdbname = apProject.getAppionDBFromProjectId(
                self.params['projectid'])
            sinedon.setConfig('appiondata', db=newdbname)
            apDisplay.printColor("Connected to database: '" + newdbname + "'",
                                 "green")

        if self.params['stackid'] is None:
            apDisplay.printError("stackid was not defined")

        if self.params['expweight'] is False:
            apDisplay.printWarning(
                "Exposure weighting is turned off, make sure this is what you want"
            )
        if self.params['localavg'] is False:
            apDisplay.printWarning(
                "Trajectory local averaging is turned off, make sure this is what you want"
            )

        # DD processes
        self.dd = apDDprocess.DDStackProcessing()
        print self.dd

        # get stack data
        self.stackdata = appiondata.ApStackData.direct_query(
            self.params['stackid'])
        self.stackparts = apStack.getStackParticlesFromId(
            self.params['stackid'], msg=True)
        self.sessiondata = apStack.getSessionDataFromStackId(
            self.params['stackid'])

        # query image
        qimage = self.stackparts[0]['particle']['image']

        # pixel size info
        self.params['apix'] = apStack.getMicrographPixelSizeFromStackId(
            self.params['stackid'])
        self.params['box'] = self.stackdata['boxsize']
        self.params['particleradius'] = self.params[
            'particleradius'] / self.params['apix']
        if self.params['particleradius'] > self.params['box'] / 2.0:
            apDisplay.printWarning(
                "specified particle radius greater than box radius, \
				setting particle radius to 0.8 * boxsize")

        # micrograph & frame info
        frames = qimage['use frames']
        nframes = len(frames)
        if self.params['framelastali'] is None:
            self.params['framelastali'] = frames[-1]
        if self.params['framelastave'] is None:
            self.params['framelastave'] = frames[-1]

        # microscope kV
        self.params['kv'] = qimage['scope']['high tension'] / 1000.0

        # query exposure per frame, if not set here
        if self.params['total_dose'] is not None:
            dose = self.params['total_dose']
        else:
            try:
                dose = apDatabase.getDoseFromImageData(qimage)
            except:
                apDisplay.printError(
                    "dose not specified and not in database, please specify explicitly"
                )
        if self.params['expperframe'] is None and self.params[
                'expweight'] is True:
            if dose is not None:
                self.params['expperframe'] = dose / nframes
            else:
                apDisplay.printError(
                    "exposure per frame needs to be specified, cannot find in database"
                )

        # dimensions
        self.params['framex'] = int(
            apDatabase.getDimensionsFromImageData(qimage)['x'])
        self.params['framey'] = int(
            apDatabase.getDimensionsFromImageData(qimage)['y'])

        # DD info
        self.dd.setImageData(qimage)
        self.dd.setDDStackRun(self.params['ddstackid'])
        self.ddstackpath = self.dd.getDDStackRun()['path']['path']
#==================
#==================
#==================
if __name__ == '__main__':
        print "Usage: apCoranPlot.py <reconid> <projectid>"

        ### setup correct database after we have read the project id
        if len(sys.argv) > 2:
                projectid = int(sys.argv[2])
        else:
                projectid = None
        if projectid is not None:
                apDisplay.printWarning("Using split database")
                # use a project database
                newdbname = apProject.getAppionDBFromProjectId(projectid)
                sinedon.setConfig('appiondata', db=newdbname)
                apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")

        ### run the program
        if len(sys.argv) > 1:
                reconid = int(sys.argv[1])
                makeCoranKeepPlot(reconid)
        else:
                reconids = getAllCoranRecons()
                for reconid in reconids:
                        recondata = apRecon.getRefineRunDataFromID(reconid)
                        reconpath = recondata['path']['path']
                        print reconpath
                        os.chdir(reconpath)
                        try:
	def checkConflicts(self):
		
		### setup correct database after we have read the project id
		if 'projectid' in self.params and self.params['projectid'] is not None:
			apDisplay.printMsg("Using split database")
			# use a project database
			newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
			sinedon.setConfig('appiondata', db=newdbname)
			apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
		
		if self.params['stackid'] is None:
			apDisplay.printError("stackid was not defined")

		if self.params['expweight'] is False:
			apDisplay.printWarning("Exposure weighting is turned off, make sure this is what you want")
		if self.params['localavg'] is False:
			apDisplay.printWarning("Trajectory local averaging is turned off, make sure this is what you want")

		# DD processes
		self.dd = apDDprocess.DDStackProcessing()
		print self.dd
	
		# get stack data
		self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid'])
		self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
		self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
		
		# query image
		qimage = self.stackparts[0]['particle']['image']

		# pixel size info
		self.params['apix'] = apStack.getMicrographPixelSizeFromStackId(self.params['stackid'])
		self.params['box'] = self.stackdata['boxsize']
		self.params['particleradius'] = self.params['particleradius'] / self.params['apix']
		if self.params['particleradius'] > self.params['box'] / 2.0:
			apDisplay.printWarning("specified particle radius greater than box radius, \
				setting particle radius to 0.8 * boxsize")

		# micrograph & frame info
		frames = qimage['use frames']
		nframes = len(frames)
		if self.params['framelastali'] is None:
			self.params['framelastali'] = frames[-1]
		if self.params['framelastave'] is None:
			self.params['framelastave'] = frames[-1]

		# microscope kV
		self.params['kv'] = qimage['scope']['high tension']/1000.0

		# query exposure per frame, if not set here
		if self.params['total_dose'] is not None:
			dose = self.params['total_dose']
		else:
			try:
				dose = apDatabase.getDoseFromImageData(qimage)
			except:
				apDisplay.printError("dose not specified and not in database, please specify explicitly")
		if self.params['expperframe'] is None and self.params['expweight'] is True:
			if dose is not None:
				self.params['expperframe'] = dose / nframes
			else:
				apDisplay.printError("exposure per frame needs to be specified, cannot find in database")
	
		# dimensions
		self.params['framex'] = int(apDatabase.getDimensionsFromImageData(qimage)['x'])
		self.params['framey'] = int(apDatabase.getDimensionsFromImageData(qimage)['y'])

		# DD info
		self.dd.setImageData(qimage)
		self.dd.setDDStackRun(self.params['ddstackid'])
		self.ddstackpath = self.dd.getDDStackRun()['path']['path']
    def checkConflicts(self):

        ### setup correct database after we have read the project id
        if 'projectid' in self.params and self.params['projectid'] is not None:
            apDisplay.printMsg("Using split database")
            # use a project database
            newdbname = apProject.getAppionDBFromProjectId(
                self.params['projectid'])
            sinedon.setConfig('appiondata', db=newdbname)
            apDisplay.printColor("Connected to database: '" + newdbname + "'",
                                 "green")

        ### get stack data
        self.stack = {}
        if self.params['stackid'] is not None:
            self.stack['data'] = apStack.getOnlyStackData(
                self.params['stackid'])
            self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
                self.params['stackid'])
            self.stack['boxsize'] = apStack.getStackBoxsize(
                self.params['stackid'])
            self.stack['file'] = os.path.join(
                self.stack['data']['path']['path'], self.stack['data']['name'])
        else:
            self.stack['data'] = appiondata.ApAlignStackData.direct_query(
                self.params['alignstackid'])
            self.stack['apix'] = self.stack['pixelsize']
            self.stack['boxsize'] = self.stack['boxsize']
            self.stack['file'] = self.stack['imagicfile']

        ### check conflicts
        if self.params['stackid'] is None and self.params[
                'alignstackid'] is None:
            apDisplay.printError("stack id OR alignstack id was not defined")
        if self.params['stackid'] is not None and self.params[
                'alignstackid'] is not None:
            apDisplay.printError(
                "either specify stack id OR alignstack id, not both")
        if self.params['generations'] is None:
            apDisplay.printError("number of generations was not provided")
        maxparticles = 500000
        if self.params['numpart'] > maxparticles:
            apDisplay.printError("too many particles requested, max: " +
                                 str(maxparticles) + " requested: " +
                                 str(self.params['numpart']))
        if self.params['numpart'] > apFile.numImagesInStack(
                self.stack['file']):
            apDisplay.printError(
                "trying to use more particles " + str(self.params['numpart']) +
                " than available " +
                str(apFile.numImagesInStack(self.stack['file'])))
        if self.params['numpart'] is None:
            self.params['numpart'] = apFile.numImagesInStack(
                self.stack['file'])

        boxsize = apStack.getStackBoxsize(self.params['stackid'])
        if self.params['ou'] is None:
            self.params['ou'] = (boxsize / 2.0) - 2
        self.clipsize = int(math.floor(
            boxsize / float(self.params['bin'] * 2))) * 2
        if self.params['clipsize'] is not None:
            if self.params['clipsize'] > self.clipsize:
                apDisplay.printError("requested clipsize is too big %d > %d" %
                                     (self.params['clipsize'], self.clipsize))
            self.clipsize = self.params['clipsize']
        self.mpirun = self.checkMPI()
        if self.mpirun is None:
            apDisplay.printError("There is no MPI installed")
def upgradeProjectDB(projectdb,backup=True):
        ### set version of database
        selectq = " SELECT * FROM `install` WHERE `key`='version'"
        values = projectdb.returnCustomSQL(selectq)
        if values:
                projectdb.updateColumn("install", "value", "'2.0'", 
                        "install.key = 'version'",timestamp=False)
        else:
                insertq = "INSERT INTO `install` (`key`, `value`) VALUES ('version', '2.0')"
                projectdb.executeCustomSQL(insertq)

if __name__ == "__main__":
        projectids = getProjectIds()
        projectdb = dbupgrade.DBUpgradeTools('projectdata', drop=True)
        for projectid in projectids:
                appiondbname = apProject.getAppionDBFromProjectId(projectid, die=False)
                if appiondbname is None:
                        continue
                elif not projectdb.databaseExists(appiondbname):
                        print "\033[31merror database %s does not exist\033[0m"%(appiondbname)
                        time.sleep(1)
                        continue
                if apProject.setDBfromProjectId(projectid, die=False):
                        reconrunids = getReconRunsIds()
                        for reconrunid in reconrunids:
                                apRecon.setGoodBadParticlesFromReconId(reconrunid)
                        # Tomography upgrades
                        createExcludedColumnByFakeInsert()
                        allalignrundata = getTomoAlignmentRuns()
                        for alignrun in allalignrundata:
                                insertTiltsInAlign(alignrun)
示例#9
0
#===============
#===============
if __name__ == "__main__":
	print "Usage: apStackMeanPlot.py <stackid> <#points> <projectid>"
	if len(sys.argv) > 1:
		stackid = int(sys.argv[1])
	else:
		#stackid=1279
		stackid=1291
	if len(sys.argv) > 2:
		gridpoints = int(sys.argv[2])
	else:
		gridpoints=16
	if len(sys.argv) > 3:
		projectid = int(sys.argv[3])
	else:
		projectid=None
	### setup correct database after we have read the project id
	if projectid is not None:
		apDisplay.printWarning("Using split database")
		# use a project database
		newdbname = apProject.getAppionDBFromProjectId(projectid)
		import sinedon
		sinedon.setConfig('appiondata', db=newdbname)
		apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")

	makeStackMeanPlot(stackid, gridpoints)



    def __init__(self,
                 optargs=sys.argv[1:],
                 quiet=False,
                 useglobalparams=True,
                 maxnproc=None):
        """
                Starts a new function and gets all the parameters
                """
        ### setup some expected values
        self.successful_run = False
        self.clusterjobdata = None
        self.params = {}
        sys.stdout.write("\n\n")
        self.quiet = quiet
        self.maxnproc = maxnproc
        self.startmem = mem.active()
        self.t0 = time.time()
        self.createDefaultStats()
        self.timestamp = apParam.makeTimestamp()
        self.argdict = {}
        self.optdict = {}
        apDisplay.printMsg("Time stamp: " + self.timestamp)
        self.functionname = apParam.getFunctionName(sys.argv[0])
        apDisplay.printMsg("Function name: " + self.functionname)
        self.appiondir = apParam.getAppionDirectory()
        apDisplay.printMsg("Appion directory: " + self.appiondir)
        self.parsePythonPath()
        loadavg = os.getloadavg()[0]
        if loadavg > 2.0:
            apDisplay.printMsg("Load average is high " +
                               str(round(loadavg, 2)))
            loadsquared = loadavg * loadavg
            time.sleep(loadavg)
            apDisplay.printMsg("New load average " +
                               str(round(os.getloadavg()[0], 2)))
        self.setLockname('lock')

        ### setup default parser: run directory, etc.
        self.setParams(optargs, useglobalparams)
        #if 'outdir' in self.params and self.params['outdir'] is not None:
        #       self.params['rundir'] = self.params['outdir']

        ### setup correct database after we have read the project id
        if 'projectid' in self.params and self.params['projectid'] is not None:
            apDisplay.printMsg("Using split database")
            # use a project database
            newdbname = apProject.getAppionDBFromProjectId(
                self.params['projectid'])
            sinedon.setConfig('appiondata', db=newdbname)
            apDisplay.printColor("Connected to database: '" + newdbname + "'",
                                 "green")

        ### check if user wants to print help message
        if 'commit' in self.params and self.params['commit'] is True:
            apDisplay.printMsg("Committing data to database")
        else:
            apDisplay.printWarning("Not committing data to database")

        self.checkConflicts()
        if useglobalparams is True:
            self.checkGlobalConflicts()

        ### setup run directory
        self.setProcessingDirName()
        self.setupRunDirectory()

        ### Start pool of threads to run subprocesses.
        ### Later you will use self.process_launcher.launch(...) to
        ### put commands into the queue.
        ### There is currently a timeout built into it that will cause
        ### the threads to die if they have no tasks after 10 seconds.
        self.process_launcher = apThread.ProcessLauncher(
            2, self.params['rundir'])

        ### write function log
        self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))

        ### upload command line parameters to database
        self.uploadScriptData()

        ### any custom init functions go here
        self.onInit()
    if values:
        projectdb.updateColumn("install",
                               "value",
                               "'2.0'",
                               "install.key = 'version'",
                               timestamp=False)
    else:
        insertq = "INSERT INTO `install` (`key`, `value`) VALUES ('version', '2.0')"
        projectdb.executeCustomSQL(insertq)


if __name__ == "__main__":
    projectids = getProjectIds()
    projectdb = dbupgrade.DBUpgradeTools('projectdata', drop=True)
    for projectid in projectids:
        appiondbname = apProject.getAppionDBFromProjectId(projectid, die=False)
        if appiondbname is None:
            continue
        elif not projectdb.databaseExists(appiondbname):
            print "\033[31merror database %s does not exist\033[0m" % (
                appiondbname)
            time.sleep(1)
            continue
        if apProject.setDBfromProjectId(projectid, die=False):
            reconrunids = getReconRunsIds()
            for reconrunid in reconrunids:
                apRecon.setGoodBadParticlesFromReconId(reconrunid)
            # Tomography upgrades
            createExcludedColumnByFakeInsert()
            allalignrundata = getTomoAlignmentRuns()
            for alignrun in allalignrundata:
	def __init__(self,optargs=sys.argv[1:],quiet=False,useglobalparams=True,maxnproc=None):
		"""
		Starts a new function and gets all the parameters
		"""
		### setup some expected values
		self.successful_run = False
		self.clusterjobdata = None
		self.params = {}
		sys.stdout.write("\n\n")
		self.quiet = quiet
		self.maxnproc = maxnproc
		self.startmem = mem.active()
		self.t0 = time.time()
		self.createDefaultStats()
		self.timestamp = apParam.makeTimestamp()
		self.argdict = {}
		self.optdict = {}
		apDisplay.printMsg("Time stamp: "+self.timestamp)
		self.functionname = apParam.getFunctionName(sys.argv[0])
		apDisplay.printMsg("Function name: "+self.functionname)
		self.appiondir = apParam.getAppionDirectory()
		apDisplay.printMsg("Appion directory: "+self.appiondir)
		hostname = apParam.getHostname()
		apDisplay.printMsg("Processing hostname: "+hostname)
		self.parsePythonPath()
# 		loadavg = os.getloadavg()[0]
# 		if loadavg > 2.0:
# 			apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
# 			loadsquared = loadavg*loadavg
# 			time.sleep(loadavg)
# 			apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2)))
		self.setLockname('lock')

		### setup default parser: run directory, etc.
		self.setParams(optargs,useglobalparams)
		#if 'outdir' in self.params and self.params['outdir'] is not None:
		#	self.params['rundir'] = self.params['outdir']

		### setup correct database after we have read the project id
		if 'projectid' in self.params and self.params['projectid'] is not None:
			apDisplay.printMsg("Using split database")
			# use a project database
			newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
			sinedon.setConfig('appiondata', db=newdbname)
			apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")

		### check if user wants to print help message
		if 'commit' in self.params and self.params['commit'] is True:
			apDisplay.printMsg("Committing data to database")
		else:
			apDisplay.printWarning("Not committing data to database")

		self.checkConflicts()
		if useglobalparams is True:
			self.checkGlobalConflicts()

		### setup run directory
		self.setProcessingDirName()
		self.setupRunDirectory()

		### Start pool of threads to run subprocesses.
		### Later you will use self.process_launcher.launch(...) to
		### put commands into the queue.
		### There is currently a timeout built into it that will cause
		### the threads to die if they have no tasks after 10 seconds.
		self.process_launcher = apThread.ProcessLauncher(2, self.params['rundir'])

		### write function log
		self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))

		### upload command line parameters to database
		self.uploadScriptData()

		### any custom init functions go here
		self.onInit()