def setJobStatus(jobid, status): ### clean status newstat = str(status[0]).upper() ### cluster job data clustdata = appiondata.ApAppionJobData.direct_query(jobid) if not clustdata: print "Did not find jobid=%d" % (jobid) return False ### do the query dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) db.autocommit(True) cursor = db.cursor() query = ("UPDATE \n" + " `ApAppionJobData` as job \n" + "SET \n" + " job.`status` = '" + str(newstat) + "' \n" + "WHERE \n" + " job.`DEF_id` = " + str(clustdata.dbid) + " \n") try: cursor.execute(query) except: print "MySQL query failed:\n======\n%s" % (query) return False if getJobStatus(jobid) == newstat: return True return False
def look_up(filename): legname = os.path.basename(filename) if legname.endswith('.mrc'): legname = legname[:-4] # find possible sessions that contain this image session_str = legname.split('_')[0] q = 'SELECT name from SessionData WHERE name LIKE "%s%%"' % (session_str,) conf = sinedon.getConfig('leginondata') db = MySQLdb.connect(**conf) cur = db.cursor() cur.execute(q) sessions = cur.fetchall() session_names = [session[0] for session in sessions] # first look for AcquisitionImageData for session_name in session_names: sessiondata = leginon.leginondata.SessionData(name=session_name) sessiondata = sessiondata.query(results=1)[0] imdata = leginon.leginondata.AcquisitionImageData(session=sessiondata, filename=legname) imdata = imdata.query() if imdata: fullpath = os.path.join(sessiondata['image path'], legname+'.mrc') return fullpath # then look for reference images for imclass in (leginon.leginondata.DarkImageData, leginon.leginondata.NormImageData, leginon.leginondata.BrightImageData): imdata = imclass(filename=legname) imdata = imdata.query() if imdata: sessiondata = imdata[0]['session'] fullpath = os.path.join(sessiondata['image path'], legname+'.mrc') return fullpath raise ValueError('no image found in db: %s' % (filename,))
def makeTables(sinedonname,modulename,dbname=None,xmlfile=None,check_exist=False): ### use alternate db name if desired if dbname is not None: print "setting alternate database name" sinedon.setConfig(sinedonname, db=dbname) ### connect to DB dbconf = sinedon.getConfig(sinedonname) dbd = sqldict.SQLDict(**dbconf) ### import desire module module = __import__(modulename) modbase = re.sub("^.*\.", "", modulename) tableData = getattr(module, modbase) ## hope this works ### get module members funcs = inspect.getmembers(tableData, inspect.isclass) print "Found %d classes in module"%(len(funcs)) #print funcs ### parse members count = 0 if xmlfile is not None: xmlf = open(xmlfile, 'w') xmlf.write("<defaulttables>\n <definition>\n") for func in funcs: ### Check if member is valid len 2 tuple if len(func) != 2: continue ### Check if member is a sinedon Data class if not issubclass(func[1], sinedon.data.Data) or func[0] == "Data": continue ### Create table tablename = func[0] tableclass = func[1]() table = (dbname, tablename) definition, formatedData = sqldict.dataSQLColumns(tableclass, False) create_flag=False if check_exist: try: dbd.diffSQLTable(tablename,definition) except (MySQLdb.ProgrammingError, MySQLdb.OperationalError), e: errno = e.args[0] ## some version of mysqlpython parses the exception differently if not isinstance(errno, int): errno = errno.args[0] ## 1146: table does not exist if errno in (1146,): print tablename,' does not yet exist, and will be created' create_flag=True else: create_flag=True if create_flag: if xmlfile is None: dbd.createSQLTable(table, definition) else: definitionToXml(xmlf, tablename, definition) count += 1
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("enter a stackid ID, e.g. --stackid=773") dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) db.autocommit(True) self.cursor = db.cursor()
def onInit(self): # connect self.dbconf = sinedon.getConfig('appiondata') self.db = MySQLdb.connect(**self.dbconf) self.db.autocommit(True) # create a cursor self.cursor = self.db.cursor()
def setJobStatus(jobid, status): ### clean status newstat = str(status[0]).upper() ### cluster job data clustdata = appiondata.ApAppionJobData.direct_query(jobid) if not clustdata: print "Did not find jobid=%d"%(jobid) return False ### do the query dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) db.autocommit(True) cursor = db.cursor() query = ( "UPDATE \n" +" `ApAppionJobData` as job \n" +"SET \n" +" job.`status` = '"+str(newstat)+"' \n" +"WHERE \n" +" job.`DEF_id` = "+str(clustdata.dbid)+" \n" ) try: cursor.execute(query) except: print "MySQL query failed:\n======\n%s"%(query) return False if getJobStatus(jobid) == newstat: return True return False
def partnum2defid(stackid): """ This function must be used because sinedon would take up too much memory? """ t0 = time.time() stackpartnum = apStack.getNumberStackParticlesFromId(stackid) apDisplay.printMsg("Mapping %d stack particle IDs"%(stackpartnum)) import sinedon import MySQLdb dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) cursor = db.cursor() cursor.execute('SELECT DEF_id, particleNumber FROM ApStackParticleData WHERE `REF|ApStackData|stack` = %s' % (stackid,)) partdict = {} maxnum = 0 while True: row = cursor.fetchone() if row is None: break defid = int(row[0]) #row['DEF_id'] num = int(row[1]) #row['particleNumber'] if num > maxnum: maxnum = num partdict[num] = defid if stackpartnum != maxnum: apDisplay.printError("Expected to get %d particles, but received %d particles"%(stackpartnum, maxnum)) apDisplay.printMsg("Mapped %d stack particle IDs in %s"%(stackpartnum, apDisplay.timeString(time.time()-t0))) return partdict
def __init__(self): """ Need to connect to DB server before moving forward """ appionScript.AppionScript.__init__(self) # connect self.dbconf = sinedon.getConfig('appiondata') self.db = MySQLdb.connect(**self.dbconf) self.db.autocommit(True) # create a cursor self.cursor = self.db.cursor()
def __init__(self, **kwargs): # connection to the project database try: dbparams = sinedon.getConfig('projectdata') except: raise NotConnectedError('no project database') if not dbparams['host']: raise NotConnectedError('no hostname for project database') try: self.db = sqldict.SQLDict(**dbparams) except Exception, e: raise NotConnectedError(e)
def setGoodBadParticlesFromReconId(reconid): """ Goes through existing recons and caches the number of good and bad particles """ import sinedon import MySQLdb dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) db.autocommit(True) cursor = db.cursor() refinerundata = appiondata.ApRefineRunData.direct_query(reconid) refineq = appiondata.ApRefineIterData() refineq['refineRun'] = refinerundata refineiterdatas = refineq.query() r0 = time.time() for refineiterdata in refineiterdatas: t0 = time.time() #print "Iteration %d"%(refineiterdata['iteration']) goodbadq = appiondata.ApRefineGoodBadParticleData() goodbadq['refine'] = refineiterdata goodbaddata = goodbadq.query() if goodbaddata: continue fields = { 'good_refine': getParticleCount(refineiterdata.dbid, cursor, 'refine_keep', True), 'bad_refine': getParticleCount(refineiterdata.dbid, cursor, 'refine_keep', False), 'good_postRefine': getParticleCount(refineiterdata.dbid, cursor, 'postRefine_keep', True), 'bad_postRefine': getParticleCount(refineiterdata.dbid, cursor, 'postRefine_keep', False), } #print fields goodbadq = appiondata.ApRefineGoodBadParticleData() goodbadq['refine'] = refineiterdata goodbadq['good_refine'] = fields['good_refine'] goodbadq['bad_refine'] = fields['bad_refine'] goodbadq['good_postRefine'] = fields['good_postRefine'] goodbadq['bad_postRefine'] = fields['bad_postRefine'] goodbadq.insert() if time.time() - t0 > 2.0: apDisplay.printMsg("Iter completed in %s" % (apDisplay.timeString(time.time() - t0))) if time.time() - r0 > 2.0: apDisplay.printMsg("Refine completed in %s" % (apDisplay.timeString(time.time() - r0))) return
def __init__(self): """ Need to connect to DB server before moving forward """ ### get db config info self.dbconf = sinedon.getConfig('appiondata') ### connect self.db = MySQLdb.connect(**self.dbconf) self.db.autocommit(True) ### create a cursor self.cursor = self.db.cursor() ### keep sinedon version too self.jumperror = False
def readyUploadFlag(self): if self.params['commit'] is False: return config = sinedon.getConfig('appiondata') dbc = MySQLdb.Connect(**config) dbc.autocommit(True) cursor = dbc.cursor() query = (" UPDATE ApMaxLikeJobData " + " SET `finished` = '1' " + " WHERE `DEF_id` = '" + str(self.params['maxlikejobid']) + "'") cursor.execute(query) cursor.close() dbc.close()
def __updateStatusInDB (self, jobid, status): retVal = True #initialize return value to True dbConfig = sinedon.getConfig('appiondata') dbConnection = MySQLdb.connect(**dbConfig) cursor = dbConnection.cursor() updateCommand = "UPDATE ApAppionJobData SET status= '%s' WHERE `DEF_id` = '%s'" % (status, jobid) result = cursor.execute(updateCommand) if not result: retVal = False return retVal
def setImgViewerStatus(imgdata, status=None, msg=True): """ Function that sets the image status in the viewer False: Image was hidden True: Image is an exemplar None: Image is visible see 'ViewerImageStatus' table in dbemdata """ if status is False: statusVal = 'hidden' elif status is True: statusVal = 'exemplar' else: print "skipping set viewer status" return currentstatus = getImgViewerStatus(imgdata) if currentstatus is None: #insert new statusq = leginon.leginondata.ViewerImageStatus() statusq['image'] = imgdata statusq['status'] = statusVal statusq.insert() elif currentstatus != status: #update column dbconf = sinedon.getConfig('leginondata') db = sinedon.sqldb.sqlDB(**dbconf) q = ("UPDATE " + dbconf['db'] + ".`ViewerImageStatus` " + "SET status = '" + statusVal + ("' WHERE `REF|AcquisitionImageData|image`=%d" % (imgdata.dbid, ))) db.execute(q) #check assessment if msg is True: finalassess = getImgViewerStatus(imgdata) imgname = apDisplay.short(imgdata['filename']) if finalassess is True: astr = apDisplay.colorString("exemplar", "green") elif finalassess is False: astr = apDisplay.colorString("hidden", "red") elif finalassess is None: astr = apDisplay.colorString("none", "yellow") apDisplay.printMsg("Final image assessment: " + astr + " (" + imgname + ")") return
def readyUploadFlag(self): if self.params['commit'] is False: return config = sinedon.getConfig('appiondata') dbc = MySQLdb.Connect(**config) dbc.autocommit(True) cursor = dbc.cursor() query = ( " UPDATE ApMaxLikeJobData " +" SET `finished` = '1' " +" WHERE `DEF_id` = '"+str(self.params['maxlikejobid'])+"'" ) cursor.execute(query) cursor.close() dbc.close()
def __updateStatusInDB (self, jobid, status): retVal = True #initialize return value to True dbConfig = sinedon.getConfig('appiondata') dbConnection = MySQLdb.connect(**dbConfig) dbConnection.autocommit(True) cursor = dbConnection.cursor() updateCommand = "UPDATE ApAppionJobData SET status= '%s' WHERE `DEF_id` = '%s'" % (status, jobid) result = cursor.execute(updateCommand) if not result: retVal = False return retVal
def run(self): # setAppionDB apProject.setAppiondbBySessionName(self.sessionname) import sinedon apdbname = sinedon.getConfig('appiondata')['db'] print 'Change apPath in database %s' % apdbname # update query = "update %s.`ApPathData` set `path` = '%s' where %s.`ApPathData`.`path` like '%s'" % (apdbname,self.new_path, apdbname,self.old_path) print query # send the query to the database linked to sinedon module named appiondata directq.complexMysqlQuery('appiondata',query) print 'Path changed from %s to %s' % (self.old_path, self.new_path)
def setImgViewerStatus(imgdata, status=None, msg=True): """ Function that sets the image status in the viewer False: Image was hidden True: Image is an exemplar None: Image is visible see 'ViewerImageStatus' table in dbemdata """ if status is False: statusVal = 'hidden' elif status is True: statusVal = 'exemplar' else: print "skipping set viewer status" return currentstatus = getImgViewerStatus(imgdata) if currentstatus is None: #insert new statusq = leginon.leginondata.ViewerImageStatus() statusq['image'] = imgdata statusq['status'] = statusVal statusq.insert() elif currentstatus != status: #update column dbconf=sinedon.getConfig('leginondata') db=sinedon.sqldb.sqlDB(**dbconf) q= ( "UPDATE "+dbconf['db']+".`ViewerImageStatus` " +"SET status = '"+statusVal + ("' WHERE `REF|AcquisitionImageData|image`=%d" % (imgdata.dbid,))) db.execute(q) #check assessment if msg is True: finalassess = getImgViewerStatus(imgdata) imgname = apDisplay.short(imgdata['filename']) if finalassess is True: astr = apDisplay.colorString("exemplar", "green") elif finalassess is False: astr = apDisplay.colorString("hidden", "red") elif finalassess is None: astr = apDisplay.colorString("none", "yellow") apDisplay.printMsg("Final image assessment: "+astr+" ("+imgname+")") return
def backupSinedonCfg(self): # get the original database host, user info import sinedon self.globalconfigs = sinedon.getConfig('leginondata') print self.globalconfigs # make backup homepath = expanduser("~") self.sinedon_cfg_file = os.path.join(homepath, 'sinedon.cfg') if os.path.isfile(self.sinedon_cfg_file): # make backup self.sinedon_cfg_backup = self.sinedon_cfg_file while os.path.isfile(self.sinedon_cfg_backup): self.sinedon_cfg_backup += 'k' shutil.move(self.sinedon_cfg_file, self.sinedon_cfg_backup) else: self.sinedon_cfg_backup = None
def backupSinedonCfg(self): # get the original database host, user info import sinedon self.globalconfigs = sinedon.getConfig('leginondata') print self.globalconfigs # make backup homepath = expanduser("~") self.sinedon_cfg_file = os.path.join(homepath,'sinedon.cfg') if os.path.isfile(self.sinedon_cfg_file): # make backup self.sinedon_cfg_backup = self.sinedon_cfg_file while os.path.isfile(self.sinedon_cfg_backup): self.sinedon_cfg_backup +='k' shutil.move(self.sinedon_cfg_file,self.sinedon_cfg_backup) else: self.sinedon_cfg_backup = None
def run(self): # setAppionDB apProject.setAppiondbBySessionName(self.sessionname) import sinedon apdbname = sinedon.getConfig('appiondata')['db'] print 'Change apPath in database %s' % apdbname # update query = "update %s.`ApPathData` set `path` = '%s' where %s.`ApPathData`.`path` like '%s'" % ( apdbname, self.new_path, apdbname, self.old_path) print query # send the query to the database linked to sinedon module named appiondata directq.complexMysqlQuery('appiondata', query) print 'Path changed from %s to %s' % (self.old_path, self.new_path)
def getAllCoranRecons(): dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) # create a cursor cursor = db.cursor() query = ( " SELECT DISTINCT refdat.`REF|ApRefineRunData|refineRun` AS reconid " +" FROM `ApRefineIterData` AS refdat " +" WHERE refdat.`SpiCoranGoodClassAvg` IS NOT NULL " +" AND refdat.`iteration` > '7' " ) cursor.execute(query) results = cursor.fetchall() db.close() reconids = [] for row in results: reconids.append(int(row[0])) print "found "+str(len(reconids))+" reconids: ", reconids return reconids
def getTotalNumParticles(reconid, numiter): dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) # create a cursor cursor = db.cursor() query = ( " SELECT stackpart.`particleNumber` AS p " +" FROM `ApRefineParticleData` AS reconpart " +" LEFT JOIN `ApStackParticleData` AS stackpart " +" ON (reconpart.`REF|ApStackParticleData|particle` = stackpart.`DEF_id`) " +" LEFT JOIN `ApRefineIterData` AS refdat " +" ON (reconpart.`REF|ApRefineIterData|refineIter` = refdat.`DEF_id`) " +" WHERE refdat.`REF|ApRefineRunData|refineRun` = '"+str(reconid)+"' " +" AND refdat.`iteration` = '"+str(numiter)+"' " ) cursor.execute(query) numpart = int(cursor.rowcount) db.close() return numpart
def getTotalNumParticles(reconid, numiter): dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) # create a cursor cursor = db.cursor() query = ( " SELECT stackpart.`particleNumber` AS p " + " FROM `ApRefineParticleData` AS reconpart " + " LEFT JOIN `ApStackParticleData` AS stackpart " + " ON (reconpart.`REF|ApStackParticleData|particle` = stackpart.`DEF_id`) " + " LEFT JOIN `ApRefineIterData` AS refdat " + " ON (reconpart.`REF|ApRefineIterData|refineIter` = refdat.`DEF_id`) " + " WHERE refdat.`REF|ApRefineRunData|refineRun` = '" + str(reconid) + "' " + " AND refdat.`iteration` = '" + str(numiter) + "' ") cursor.execute(query) numpart = int(cursor.rowcount) db.close() return numpart
def getAllCoranRecons(): dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) # create a cursor cursor = db.cursor() query = ( " SELECT DISTINCT refdat.`REF|ApRefineRunData|refineRun` AS reconid " + " FROM `ApRefineIterData` AS refdat " + " WHERE refdat.`SpiCoranGoodClassAvg` IS NOT NULL " + " AND refdat.`iteration` > '7' ") cursor.execute(query) results = cursor.fetchall() db.close() reconids = [] for row in results: reconids.append(int(row[0])) print "found " + str(len(reconids)) + " reconids: ", reconids return reconids
def setGoodBadParticlesFromReconId(reconid): """ Goes through existing recons and caches the number of good and bad particles """ import sinedon import MySQLdb dbconf = sinedon.getConfig("appiondata") db = MySQLdb.connect(**dbconf) db.autocommit(True) cursor = db.cursor() refinerundata = appiondata.ApRefineRunData.direct_query(reconid) refineq = appiondata.ApRefineIterData() refineq["refineRun"] = refinerundata refineiterdatas = refineq.query() r0 = time.time() for refineiterdata in refineiterdatas: t0 = time.time() # print "Iteration %d"%(refineiterdata['iteration']) goodbadq = appiondata.ApRefineGoodBadParticleData() goodbadq["refine"] = refineiterdata goodbaddata = goodbadq.query() if goodbaddata: continue fields = { "good_refine": getParticleCount(refineiterdata.dbid, cursor, "refine_keep", True), "bad_refine": getParticleCount(refineiterdata.dbid, cursor, "refine_keep", False), "good_postRefine": getParticleCount(refineiterdata.dbid, cursor, "postRefine_keep", True), "bad_postRefine": getParticleCount(refineiterdata.dbid, cursor, "postRefine_keep", False), } # print fields goodbadq = appiondata.ApRefineGoodBadParticleData() goodbadq["refine"] = refineiterdata goodbadq["good_refine"] = fields["good_refine"] goodbadq["bad_refine"] = fields["bad_refine"] goodbadq["good_postRefine"] = fields["good_postRefine"] goodbadq["bad_postRefine"] = fields["bad_postRefine"] goodbadq.insert() if time.time() - t0 > 2.0: apDisplay.printMsg("Iter completed in %s" % (apDisplay.timeString(time.time() - t0))) if time.time() - r0 > 2.0: apDisplay.printMsg("Refine completed in %s" % (apDisplay.timeString(time.time() - r0))) return
def checkConflicts(self): """ make sure the necessary parameters are set correctly """ if self.cursor is None: # connect self.dbconf = sinedon.getConfig('appiondata') self.db = MySQLdb.connect(**self.dbconf) self.db.autocommit(True) # create a cursor self.cursor = self.db.cursor() if not self.params['reconid']: apDisplay.printError("Enter a Reconstruction Run ID, e.g. --reconid=243") if not self.params['tiltrunid']: self.params['tiltrunid'] = self.getTiltRunIDFromReconID(self.params['reconid']) self.params['symmetry'] = apSymmetry.getSymmetryFromReconRunId(self.params['reconid']) self.params['symmname'] = self.params['symmetry']['eman_name'] if not self.params['stackid']: self.params['stackid'] = apStack.getStackIdFromRecon(self.params['reconid'])
def getParticlesForIter(reconid, iternum): dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) # create a cursor cursor = db.cursor() query = ( " SELECT stackpart.`particleNumber` AS p " +" FROM `ApRefineParticleData` AS reconpart " +" LEFT JOIN `ApStackParticleData` AS stackpart " +" ON (reconpart.`REF|ApStackParticleData|particle` = stackpart.`DEF_id`) " +" LEFT JOIN `ApRefineIterData` AS refdat " +" ON (reconpart.`REF|ApRefineIterData|refineIter` = refdat.`DEF_id`) " +" WHERE refdat.`REF|ApRefineRunData|refineRun` = '"+str(reconid)+"' " +" AND reconpart.`coran_keep` = 1 " # for Coran plot #+" AND reconpart.`thrown_out` IS NULL " # for EMAN plot +" AND refdat.`iteration` = '"+str(iternum)+"' " ) cursor.execute(query) results = cursor.fetchall() db.close() return results
def getParticlesForIter(reconid, iternum): dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) # create a cursor cursor = db.cursor() query = ( " SELECT stackpart.`particleNumber` AS p " + " FROM `ApRefineParticleData` AS reconpart " + " LEFT JOIN `ApStackParticleData` AS stackpart " + " ON (reconpart.`REF|ApStackParticleData|particle` = stackpart.`DEF_id`) " + " LEFT JOIN `ApRefineIterData` AS refdat " + " ON (reconpart.`REF|ApRefineIterData|refineIter` = refdat.`DEF_id`) " + " WHERE refdat.`REF|ApRefineRunData|refineRun` = '" + str(reconid) + "' " + " AND reconpart.`coran_keep` = 1 " # for Coran plot #+" AND reconpart.`thrown_out` IS NULL " # for EMAN plot + " AND refdat.`iteration` = '" + str(iternum) + "' ") cursor.execute(query) results = cursor.fetchall() db.close() return results
def __initDB (self, jobObject, job): retValue = None try: #Determine the appion project database name using the project id. projDBConfig = sinedon.getConfig('projectdata') dbConnection = MySQLdb.connect(**projDBConfig) cursor = dbConnection.cursor() query = "SELECT appiondb from processingdb WHERE `REF|projects|project`=%d" % (jobObject.getProjectId()) queryResult=cursor.execute(query) if queryResult: projDB = cursor.fetchone()[0] projDBConfig = sinedon.setConfig('appiondata', db=projDB) retValue = projDB cursor.close() dbConnection.close() except MySQLdb.DatabaseError, e: sys.stderr.write("Warning: Failure determining project database: %s \n" % (e))
def __initDB (self, jobObject, job): retValue = None try: #Determine the appion project database name using the project id. projDBConfig = sinedon.getConfig('projectdata') dbConnection = MySQLdb.connect(**projDBConfig) dbConnection.autocommit(True) cursor = dbConnection.cursor() query = "SELECT appiondb from processingdb WHERE `REF|projects|project`=%d" % (jobObject.getProjectId()) queryResult=cursor.execute(query) if queryResult: projDB = cursor.fetchone()[0] projDBConfig = sinedon.setConfig('appiondata', db=projDB) retValue = projDB cursor.close() dbConnection.close() except MySQLdb.DatabaseError, e: sys.stderr.write("Warning: Failure determining project database: %s \n" % (e))
def look_up(filename): legname = os.path.basename(filename) if legname.endswith('.mrc'): legname = legname[:-4] # find possible sessions that contain this image session_str = legname.split('_')[0] q = 'SELECT name from SessionData WHERE name LIKE "%s%%"' % (session_str, ) conf = sinedon.getConfig('leginondata') db = MySQLdb.connect(**conf) cur = db.cursor() cur.execute(q) sessions = cur.fetchall() session_names = [session[0] for session in sessions] # first look for AcquisitionImageData for session_name in session_names: sessiondata = leginon.leginondata.SessionData(name=session_name) sessiondata = sessiondata.query(results=1)[0] imdata = leginon.leginondata.AcquisitionImageData(session=sessiondata, filename=legname) imdata = imdata.query() if imdata: fullpath = os.path.join(sessiondata['image path'], legname + '.mrc') return fullpath # then look for reference images for imclass in (leginon.leginondata.DarkImageData, leginon.leginondata.NormImageData, leginon.leginondata.BrightImageData): imdata = imclass(filename=legname) imdata = imdata.query() if imdata: sessiondata = imdata[0]['session'] fullpath = os.path.join(sessiondata['image path'], legname + '.mrc') return fullpath raise ValueError('no image found in db: %s' % (filename, ))
def getEulersForIteration(reconid, iteration=1): """ returns all classdata for a particular refinement iteration this does NOT use sinedon because it goes from taking < 1 min for this method to taking several hours with sinedon """ # connect dbconf = sinedon.getConfig('appiondata') db = MySQLdb.connect(**dbconf) db.autocommit(True) # create a cursor cursor = db.cursor() t0 = time.time() query = ( "SELECT pc.euler1, pc.euler2, pc.`refine_keep`, pc.`postRefine_keep` " +"FROM `ApRefineParticleData` AS pc " +"LEFT JOIN `ApRefineIterData` AS rd " +"ON pc.`REF|ApRefineIterData|refineIter` = rd.`DEF_id` " +"WHERE rd.`REF|ApRefineRunData|refineRun` = "+str(reconid)+" " +"AND rd.`iteration` = "+str(iteration)+" " ) print "querying for euler values at "+time.asctime() cursor.execute(query) numrows = int(cursor.rowcount) apDisplay.printColor("Found "+str(numrows)+" euler values", "cyan") result = cursor.fetchall() apDisplay.printMsg("Fetched data in "+apDisplay.timeString(time.time()-t0)) # sort eulers into classes alldict, eulerdict, postrefinedict = calcDictNative(result) db.close() return alldict, eulerdict, postrefinedict
if __name__ == "__main__": # parse options projectid = None if len(sys.argv) < 3: print "Usage: %s jobid status [projectid]" % (sys.argv[0],) sys.exit() jobid = sys.argv[1] status = sys.argv[2] if len(sys.argv) > 3: projectid = sys.argv[3] # set new db if projectid is not None: pjc = sinedon.getConfig('projectdata') q = "SELECT appiondb FROM processingdb WHERE `REF|projects|project`='%s'" % (projectid,) dbc = MySQLdb.Connect(**pjc) dbc.autocommit(True) cursor = dbc.cursor() result = cursor.execute(q) if result: newdbname, = cursor.fetchone() sinedon.setConfig('appiondata', db=newdbname) cursor.close() dbc.close() # connect to database c = sinedon.getConfig('appiondata') dbc = MySQLdb.Connect(**c)
def getConnection(modulename='leginondata'): if not connections.has_key(modulename): print 'connecting' param = sinedon.getConfig(modulename) connections[modulename] = sqldb.sqlDB(**param) return connections[modulename]
def __init__(self, sessionname): apProject.setAppiondbBySessionName(sessionname) self.sessionname = sessionname import sinedon self.apdbname = sinedon.getConfig('appiondata')['db']
def makeTables(sinedonname, modulename, dbname=None, xmlfile=None, check_exist=False): ### use alternate db name if desired if dbname is not None: print "setting alternate database name" sinedon.setConfig(sinedonname, db=dbname) ### connect to DB dbconf = sinedon.getConfig(sinedonname) dbd = sqldict.SQLDict(**dbconf) ### import desire module module = __import__(modulename) modbase = re.sub("^.*\.", "", modulename) tableData = getattr(module, modbase) ## hope this works ### get module members funcs = inspect.getmembers(tableData, inspect.isclass) print "Found %d classes in module" % (len(funcs)) #print funcs ### parse members count = 0 if xmlfile is not None: xmlf = open(xmlfile, 'w') xmlf.write("<defaulttables>\n <definition>\n") for func in funcs: ### Check if member is valid len 2 tuple if len(func) != 2: continue ### Check if member is a sinedon Data class if not issubclass(func[1], sinedon.data.Data) or func[0] == "Data": continue ### Create table tablename = func[0] tableclass = func[1]() table = (dbname, tablename) definition, formatedData = sqldict.dataSQLColumns(tableclass, False) create_flag = False if check_exist: try: dbd.diffSQLTable(tablename, definition) except (MySQLdb.ProgrammingError, MySQLdb.OperationalError), e: errno = e.args[0] ## some version of mysqlpython parses the exception differently if not isinstance(errno, int): errno = errno.args[0] ## 1146: table does not exist if errno in (1146, ): print tablename, ' does not yet exist, and will be created' create_flag = True else: create_flag = True if create_flag: if xmlfile is None: dbd.createSQLTable(table, definition) else: definitionToXml(xmlf, tablename, definition) count += 1
if __name__ == "__main__": # parse options projectid = None if len(sys.argv) < 3: print "Usage: %s jobid status [projectid]" % (sys.argv[0], ) sys.exit() jobid = sys.argv[1] status = sys.argv[2] if len(sys.argv) > 3: projectid = sys.argv[3] # set new db if projectid is not None: pjc = sinedon.getConfig('projectdata') q = "SELECT appiondb FROM processingdb WHERE `REF|projects|project`='%s'" % ( projectid, ) dbc = MySQLdb.Connect(**pjc) dbc.autocommit(True) cursor = dbc.cursor() result = cursor.execute(q) if result: newdbname, = cursor.fetchone() sinedon.setConfig('appiondata', db=newdbname) cursor.close() dbc.close() # connect to database c = sinedon.getConfig('appiondata')
from appionlib import apDatabase import leginon.leginondata def hideImage(imgid): imgdata = leginon.leginondata.AcquisitionImageData.direct_query(imgid) apDatabase.setImgViewerStatus(imgdata, False) if __name__ == '__main__': # 898749 if len(sys.argv) < 2: print "Usage: hideSquare.py <square id>" squareid = int(sys.argv[1]) dbconf = sinedon.getConfig('leginon.leginondata') db = MySQLdb.connect(**dbconf) db.autocommit(True) cursor = db.cursor() query = ( "SELECT " + " image.`DEF_id`, image.filename " + " FROM AcquisitionImageData AS image " + " LEFT JOIN AcquisitionImageTargetData AS target " + " ON image.`REF|AcquisitionImageTargetData|target` = target.`DEF_id` " + " WHERE target.`REF|AcquisitionImageData|image` = '" + str(squareid) + "' ") print query cursor.execute(query) results = cursor.fetchall() print "results: ", len(results)
import MySQLdb from appionlib import apDisplay from appionlib import apDatabase import leginon.leginondata def hideImage(imgid): imgdata=leginon.leginondata.AcquisitionImageData.direct_query(imgid) apDatabase.setImgViewerStatus(imgdata, False) if __name__ == '__main__': # 898749 if len(sys.argv) < 2: print "Usage: hideSquare.py <square id>" squareid = int(sys.argv[1]) dbconf = sinedon.getConfig('leginon.leginondata') db = MySQLdb.connect(**dbconf) db.autocommit(True) cursor = db.cursor() query = ( "SELECT " +" image.`DEF_id`, image.filename " +" FROM AcquisitionImageData AS image " +" LEFT JOIN AcquisitionImageTargetData AS target " +" ON image.`REF|AcquisitionImageTargetData|target` = target.`DEF_id` " +" WHERE target.`REF|AcquisitionImageData|image` = '"+str(squareid)+"' " ) print query cursor.execute(query) results = cursor.fetchall() print "results: ", len(results)