def convertEkinprjs(local=None, server=None, project=None, username=None): """convert old ekin prjs in a db to new""" if local != None: DB = PDatabase(local=local) elif server != None: DB = PDatabase(server=server, username='******', port=8080, password='******', project=project) for f in DB['userfields']: if DB['userfields'][f]['field_type'] in ekintypes: print f for r in DB.getRecs(): rec = DB[r] if rec.has_key(f): E = rec[f] E.checkDatasets() for d in E.datasets: ek = E.getDataset(d) #ek.prettyPrint() rec[f] = E print DB.getChanged() DB.commit('converted ekin data') return
def createdb(local=None, server=None, project=None, username=None, norecs=1000): """Create and add some test data""" if local != None: if os.path.exists(local): for i in ['.lock', '.index', '']: try: os.remove(local + i) except: pass DB = PDatabase(local=local) elif server != None: DB = PDatabase(server=server, username=username, password='******', project=project) import string import DNAtool.mutation as mutation choices = ['a', 'b', 'c', 'd'] DB.addField('choice', 'text') DB.addField('stab', 'text') DB.addField('activity', 'text') #DB.addField('ekin', 'General') E = EkinProject() data = E.readDataset('Ekin/test.Ekindat') E.insertDataset(data['data'], 'test', fit=data['fit']) print 'creating dummy data..' j = 0 count = 0 for i in range(norecs): if j > 3: j = 0 c = '' for k in range(10): c += random.choice(string.letters) DB.add(c) DB.data[c].choice = choices[j] DB.data[c].DNASeq = simulate_sequence(300) AAseqs3, AAseqs1 = mutation.translate(DB.data[c].DNASeq) DB.addProtseq(c, AAseqs3[1][5:80], 1) DB.data[c].stab = str(round(random.normalvariate(1, 2), 3)) DB.data[c].activity = str(round(random.normalvariate(30, 4), 3)) #DB.data[c].ekin = E j += 1 count += 1 if count > 3000: print 'saving..' DB.commit() DB.db.cacheMinimize() count = 0 DB.commit() return DB
def loadDB(prj=None, remote=False, settings={}): """Load a local or remote db, settings are the remote db arguments such as server, username, password and project name""" if settings.has_key('port'): settings['port'] = int(settings['port']) if remote == True: DB = PDatabase(project=prj, **settings) else: DB = PDatabase(local=prj) return DB
def saveDBCopy(DB, filename, callback=None): """Save local copy of a remote or another local DB""" import copy total = len(DB.getRecs()) if filename == '' or filename == None: return False if os.path.exists(filename): for i in ['.lock','.index','']: os.remove(filename+i) newDB = PDatabase(local=filename) newDB = copyDB(DB, newDB) newDB.commit() newDB.close() return newDB
def PEATSAJobs(prjs, resubmit=False): """Submit PEATSA runs for all projects or merge results if done""" for name in prjs: print name DB = PDatabase(local=os.path.join(savepath,name)) pdb = DB['wt'].Structure PS = PEATSAPlugin() PS.main(DB=DB) if hasattr(DB.meta,'peatsa_jobs') and resubmit == False: if 'mycalc' in DB.meta.peatsa_jobs: print 'job is present' #try to merge results S = PEATTableModel(DB) job,n = PS.getJob('mycalc') PS.mergeResults(job, 'prediction', S) DB.commit() print 'merged results' else: mutlist = [] for p in DB.getRecs(): mutlist.append(DB.get(p).Mutations) #print mutlist pdbfile = PS.writetempPDB() #we add source project data so exp data can be read from summary prjdata = {'server':'enzyme.ucd.ie','username':'******', 'project':name,'password':'******','port':'8080'} PS.submitJob(name='mycalc', pdbname=DB.meta.refprotein, pdbfile=pdbfile, mutations=mutlist, calcs=['stability'], meta={'protein':name,'expcol':'Exp','project':prjdata}) #required to end process PS.jobManager.stopLogging() DB.close() return
def showResults(self, name=None, showtable=True, ax=None, stats=True): """Show results with correlation plot from selected job""" job, name = self.getJob(name) if job == None: print 'job not in DB' return if job.state() != 'Finished': print 'job not finished' return self.matrices = job.data.allMatrices() #print self.matrices['ModellingResults'].csvRepresentation() jobmeta = job.metadata() cols = self.DB.getSimpleFields() expcol = None expdata = None #print jobmeta if jobmeta.has_key('expcol'): expcol = jobmeta['expcol'] if expcol not in cols and jobmeta.has_key('project'): #we may have stored the exp data in another project prjdata = jobmeta['project'] print 'trying to loading exp data from external project(s)' from PEATDB.Base import PDatabase from PEATTables import PEATTableModel tmpdb = PDatabase(**prjdata) print tmpdb S = PEATTableModel(tmpdb) expdata = S.simpleCopy(include=['Mutations']) print expdata #if exp column not known then ask user if expcol == '' or expcol == None: mpDlg = MultipleValDialog(title='Select Experimental Data', initialvalues=[cols], labels=['exp data column:'], types=['list'], parent=self.mainwin) if mpDlg.result == True: expcol = mpDlg.results[0] else: return for m in self.matrices: matrix = self.matrices[m] if matrix == None or not 'Total' in matrix.columnHeaders(): continue ax, mh, x, y = self.plotMerged(matrix, expcol, expdata, m, showtable, ax, name, stats) #need to add this for mousehandler to work.. hack '''from Correlation import MouseHandler mh = MouseHandler(ax, labels=expcol, key='Mutations') mh.connect()''' return ax, mh, x, y
def loadDB(): from PEATDB.Base import PDatabase DB = PDatabase(server='peat.ucd.ie', username='******', password='******', project='titration_db', port=8080) return DB
def convertClass(): """Convert records to proper module name""" project='test' DB=PDatabase(server='localhost',port=8080, username='******',password='******', project=project) for k in DB.getRecs(): r=PEATRecord(k) rec = DB.data[k] print rec.__class__ for f in rec.getFields(): r[f] = rec[f] DB.data[k] = r print DB.data[k].__class__ DB.commit(note='convert') return
def testBlob(): """write a file as binary data to blob and read back""" from ZODB.PersistentMapping import PersistentMapping import mimetypes from PILView import PILViewer DB = PDatabase(server='localhost',port=8090, username='******', password='******') def addfile(fname): myblob = Blob() b=myblob.open('w') o=open(fname) data = o.read() b.write(data) print b.name b.close() return myblob dirlist=os.listdir(os.getcwd()) for f in dirlist: m = mimetypes.guess_type(f)[0] if m != None and 'image' in m: print f b=addfile(f) DB.add(f) DB.data[f]['testfile']=FileRecord(name=f,blob=b) DB.commit() for k in DB.data: if not DB.data[k].has_key('testfile'): continue rec = DB.data[k]['testfile'] myblob = rec.blob f = myblob.open("r") print f.name #app = PILViewer(imgfile=f.name) #app.mainloop() DB.close() return
def testBlob(): """write a file as binary data to blob and read back""" from ZODB.PersistentMapping import PersistentMapping import mimetypes from PILView import PILViewer DB = PDatabase(server='localhost', port=8090, username='******', password='******') def addfile(fname): myblob = Blob() b = myblob.open('w') o = open(fname) data = o.read() b.write(data) print b.name b.close() return myblob dirlist = os.listdir(os.getcwd()) for f in dirlist: m = mimetypes.guess_type(f)[0] if m != None and 'image' in m: print f b = addfile(f) DB.add(f) DB.data[f]['testfile'] = FileRecord(name=f, blob=b) DB.commit() for k in DB.data: if not DB.data[k].has_key('testfile'): continue rec = DB.data[k]['testfile'] myblob = rec.blob f = myblob.open("r") print f.name #app = PILViewer(imgfile=f.name) #app.mainloop() DB.close() return
def setDisplayFields(local=None, server=None, project=None, username=None): """Update display attributes for every cell""" if local != None: DB = PDatabase(local=local) elif server != None: DB = PDatabase(server=server, username=username, port=8080, password='******', project=project) for r in DB.getRecs(): rec = DB[r] for f in rec: x = rec.setDisplayAttribute(f, rec[f]) rec._display[f] = x print f, rec.getDisplayAttribute(f) #DB.commit('displ attrs') return
def convertClass(): """Convert records to proper module name""" project = 'test' DB = PDatabase(server='localhost', port=8080, username='******', password='******', project=project) for k in DB.getRecs(): r = PEATRecord(k) rec = DB.data[k] print rec.__class__ for f in rec.getFields(): r[f] = rec[f] DB.data[k] = r print DB.data[k].__class__ DB.commit(note='convert') return
def speedTest1(): """benchmark read and write""" DB = PDatabase(server='localhost', port=8090) #DB = PDatabase(local='./Data.fs') for i in range(100, 300): DB.add(i) for i in range(100, 290): DB.delete(i) DB.commit() print DB DB.close() return
def checkdb(server=None, project=None, local=None): """test basic ops for the db""" if server != None: DB = PDatabase(server=server, port=8080, username='******', password='******', project=project) else: DB = PDatabase(local=local) print DB #DB.listRecs() '''first = DB.data.keys()[0] rec = DB.data[first] for k in rec.keys(): print type(rec[k])''' print DB['userfields'] print DB.meta DB.close() return
def main(): import os from optparse import OptionParser parser = OptionParser() parser.add_option("-f", "--file", dest="file", help="Open a local db") opts, remainder = parser.parse_args() #test if opts.file != None and os.path.exists(opts.file): path = os.path.abspath(opts.file) from PEATDB.Base import PDatabase DB = PDatabase(local=path) P = PEATSAPlugin() P.main(DB=DB) P.test()
def send2Server(projects): """Send all projects to remote versions""" settings={'server':'enzyme.ucd.ie','username':'******', 'password':'******','port':8080} adminsettings={'host':'enzyme.ucd.ie','user':'******', 'passwd':'nielsen','port':8080} '''for p in projects: print p DB = PDatabase(local=os.path.join(savepath,p)) Utils.createDBonServer(prj=p,settings=adminsettings, access='guest') Utils.copyDBtoServer(DB,p,settings)''' DB = PDatabase(local='summary.fs') Utils.copyDBtoServer(DB,'PotapovDataset',settings) return
def connect(self): """Connect to the peat db""" saveout = sys.stdout logpath = os.path.abspath('out.log') fsock = open(logpath, 'w') sys.stdout = fsock sys.stdout.flush() print self.server DB = PDatabase(server=self.server, port=self.port, username=self.user, password=self.password, project=self.project) sys.stdout.flush() sys.stdout = saveout fsock.close() return DB
def saveDBCopy(DB, filename, callback=None): """Save local copy of a remote or another local DB""" import copy total = len(DB.getRecs()) if filename == '' or filename == None: return False if os.path.exists(filename): for i in ['.lock', '.index', '']: os.remove(filename + i) newDB = PDatabase(local=filename) newDB = copyDB(DB, newDB) newDB.commit() newDB.close() return newDB
def testMemory(): """test memory behaviour of DB, how can cache be managed""" DB = PDatabase(server='localhost', username='******', password='******', project='novo') #DB = PDatabase(local='large.fs') print DB db = DB.db print db.cacheSize() for k in DB.getRecs()[:50]: #print k r = DB[k] r.name if db.cacheSize() > 500: db.cacheMinimize() print db.cacheSize() return
def info(projects): """Just return info in current projects""" total=0 summDB = PDatabase(local='summary.fs') for p in projects: DB = PDatabase(local=os.path.join(savepath,p)) l = DB.length() total += l print '%s has %s records' %(p,l) if p not in summDB.getRecs(): print 'not present in summary project' print '-----------------------' print 'info on %s projects' %len(projects) print 'with total of %s records' %total print '%s mutants' %(total-len(projects)) return
def remodel(): #DB=PDatabase(local='hewlsample.fs') DB = PDatabase(server='localhost', port=8080, username='******', password='******', project='novo') print DB, 'curr ref:', DB.meta.refprotein '''rec= DB['wt+D52N'] print rec.getDisplayAttribute('Structure') rec['Structure'] = rec.Structure''' r = '6 c9' rec = DB[r] DBActions.checkModels(DB=DB, selected=[r]) print rec.Structure #print rec.aaseq '''for r in DB.getRecs(): print r, type(DB.get(r).Structure) print DB.get(r).getDisplayAttribute('Structure')''' #DB.commit(note='modelling') return
def testLoading(): """Test loading times for large DB""" from PEATDB.PEATTables import PEATTableModel t1 = time() DB = PDatabase(local='large.fs') #DB = PDatabase(server='localhost', port=8080, username='******', # password='******', project='large') t2 = time() print round(t2 - t1, 2) print DB.getRecs() t3 = time() print round(t3 - t2, 2) print DB.meta t4 = time() print t4 - t3 M = PEATTableModel(DB) print M t5 = time() print t5 - t4 return
def importOldProj(datadir,local=None, server=None, project=None, username=None): """Import old peat projects""" import PEAT_DB.Database as peatDB from PEAT_DB.PEAT_dict import PEAT_dict, sub_dict import copy if local != None: newDB = PDatabase(local=local) elif server != None: newDB = PDatabase(server=server, username=username, port=8080, password='******', project=project) print newDB PT = peatDB.Database(datadir, Tk=False) oldDB = PT.DB print 'got old peat_db with %s proteins' %len(PT.proteins) print PT.DB.keys() #import meta stuff like userfields, table for p in newDB.meta.special: if not p in PT.DB.keys(): continue print 'adding',p for k in PT.DB[p]: newDB.meta[p][k] = copy.deepcopy(PT.DB[p][k]) newDB.meta._p_changed = 1 for p in PT.proteins: if p in newDB.meta.special: continue name = oldDB[p]['Name'] rec = PEATRecord(name=name) for col in oldDB[p].keys(): cdata = oldDB[p][col] recdata = {} if col == 'name': cdata = oldDB[p]['Name'] if oldDB['userfields'].has_key(col) and oldDB['userfields'][col]['field_type'] in ekintypes: E=EkinProject(data=cdata) E.length = len(E.datasets) if len(E.datasets)==0: continue cdata = E if type(cdata) == sub_dict: for k in cdata.keys(): recdata[k] = copy.deepcopy(cdata[k]) else: recdata = cdata if cdata != '' and cdata != None: rec.addField(col, data=recdata) newDB.add(p,rec) print newDB.meta.userfields #remove any file cols, too hard to import for m in newDB.meta.userfields.keys()[:]: if newDB.meta.userfields[m]['field_type'] == 'File': newDB.deleteField(m) newDB.commit(user='******', note='import') newDB.close() print 'import done' return
def loadDB(self, local=None): from PEATDB.Base import PDatabase if local != None: self.DB = PDatabase(local=local) return
def mergeDBs(DB1, DB2): """Combine two databases""" newDB = PDatabase() return newDB
def summarise(projects): summDB = PDatabase(local='summary.fs') C = CorrelationAnalyser() figs = [] for f in range(4): figs.append(plt.figure()) gs = gridspec.GridSpec(5, 5, wspace=0.3, hspace=0.5) i=0 data=[] print 'processing %s projects' %len(projects) for p in projects: print 'structure:',p DB = PDatabase(local=os.path.join(savepath,p)) S = PEATTableModel(DB) try: exp,pre = S.getColumns(['Exp','prediction'],allowempty=False) errs = [j[0]-j[1] for j in zip(exp,pre)] except: print 'no results' continue #DB.close() #add link to proj summDB.add(p) summDB.addField('project',fieldtype='Project') summDB[p]['project'] = {'server':'enzyme.ucd.ie','username':'******', 'project':p,'password':'******','port':'8080'} print summDB.isChanged() #stats cc,rmse,meanerr = C.getStats(pre,exp) #ttest for mean errs 0 ttp = round(stats.ttest_1samp(errs, 0)[1],2) #normality of errs w,swp = C.ShapiroWilk(errs) x={'name':p,'mutants':len(pre),'rmse':rmse,'corrcoef':cc,'meanerr':meanerr, 'ttest':ttp,'shapirowilk':swp} '''ax = figs[0].add_subplot(gs[0, i]) C.plotCorrelation(pre,exp,title=p,ms=2,axeslabels=False,ax=ax) ax = figs[1].add_subplot(gs[0, i]) C.showHistogram([pre,exp],title=p,labels=['pre','exp'],ax=ax) ax = figs[2].add_subplot(gs[0, i]) C.plotNorm(errs,title=p,lw=1,ax=ax) #qqplot ax = figs[3].add_subplot(gs[0, i]) C.QQplot(errs,title=p,ax=ax)''' #get PDB info parser = PDBParser() descr = parser.getDescription(p) x.update(descr) data.append(x) i+=1 summDB.importDict(data) print summDB.isChanged() summDB.commit() #add all peatsa jobs to summary proj also '''print 'adding peatsa job info' PS = PEATSAPlugin() PS.main(DB=summDB) #summDB.meta.peatsa_jobs = None #from ZODB.PersistentMapping import PersistentMapping #summDB.meta.peatsa_jobs = PersistentMapping() PS.checkJobsDict() PS.jobManager.stopLogging() for p in projects: #print summDB.meta DB = PDatabase(local=os.path.join(savepath,p)) job = DB.meta.peatsa_jobs['mycalc'] summDB.meta.peatsa_jobs[p] = job print job #DB.close() print summDB.isChanged() print summDB.meta.peatsa_jobs summDB.commit()''' #for i in range(len(figs)): # figs[i].savefig('fig%s.png' %i) #plt.show() return
def main(): """Run some analysis""" from optparse import OptionParser parser = OptionParser() app = NMRTitration() DB=None; E=None parser.add_option("-f", "--file", dest="file", help="Open a local db") parser.add_option("-e", "--ekinprj", dest="ekinprj", help="Open an ekin project") parser.add_option("-s", "--server", dest="server", help="field") parser.add_option("-t", "--analysis", dest="analysis", action='store_true', help="titr db analysis", default=False) parser.add_option("-r", "--refit", dest="refit", action='store_true', help="refit specific ekin data", default=False) parser.add_option("-u", "--getexperrs", dest="getexperrs", action='store_true', help="get exp uncertainties", default=False) parser.add_option("-m", "--addmeta", dest="addmeta", action='store_true', help="add meta data for NMR", default=False) parser.add_option("-p", "--protein", dest="protein", help="protein") parser.add_option("-c", "--col", dest="col", help="field") parser.add_option("-a", "--atom", dest="atom", help="atom") parser.add_option("-x", "--export", dest="export", action='store_true', help="export db", default=False) parser.add_option("-b", "--benchmark", dest="benchmark", action='store_true', help="benchmark some stuff", default=False) parser.add_option("-g", "--gui", dest="gui", action='store_true', help="start gui app", default=False) opts, remainder = parser.parse_args() if opts.file != None and os.path.exists(opts.file): app.loadDB(opts.file) elif opts.server != None: DB = PDatabase(server='localhost', username='******', password='******', project='titration_db', port=8080) if opts.gui == True: app.main() app.mainwin.mainloop() return yuncerts = {'H':0.03,'N':0.1,'C':0.2} try: yuncert=yuncerts[opts.atom] except: yuncert=None if opts.ekinprj != None: E = EkinProject() E.openProject(opts.ekinprj) #some tit db analysis if opts.analysis == True and opts.server != None: complete = ['HEWL', 'Bovine Beta-Lactoglobulin', 'Plastocyanin (Anabaena variabilis)', 'Plastocyanin (Phormidium)', 'Glutaredoxin', 'Protein G B1','Xylanase (Bacillus subtilus)'] if opts.col == None: print 'provide a column' else: app.analyseTitDB(DB, opts.col)#, complete) #app.addpKaTables(DB, complete) elif opts.benchmark == True: app.benchmarkExpErr(DB) elif opts.col != None or E != None: app.titDBUtils(DB, opts.col, opts.protein, a=opts.atom, E=E, refit=opts.refit, addmeta=opts.addmeta, getexperrs=opts.getexperrs, yuncert=yuncert) elif opts.export == True: app.exportAll(DB, col=opts.col)
def loadDB(self): """Load a DB""" from PEATDB.Base import PDatabase if local != None: self.DB = PDatabase(local=local) return
def importTest(): DB = PDatabase(local='import.fs') DB.importCSV(filename='testdata.csv') return
def summarystats(projects): """summary stats""" for p in projects: DB = PDatabase(local=os.path.join(savepath,p))
def createProjects(files): """Create multiple projects at once from csv files""" for filename in files: print filename name = os.path.splitext(filename)[0] #create/open db DB = PDatabase(local=os.path.join(savepath,name)) DB.add('wt') #add wt pdb stream = DBActions.fetchPDB(name) DBActions.addPDBFile(DB, 'wt', pdbdata=stream, pdbname=name, gui=False) DB.meta.refprotein = 'wt' DB.meta.info['protein'] = name #import data from csv DB.importCSV(os.path.join(cpath,filename), namefield='Mutations') print 'imported ok' DB.deleteField('PDB') DB.commit() DB.close() print 'done' return
def importOldProj(datadir, local=None, server=None, project=None, username=None): """Import old peat projects""" import PEAT_DB.Database as peatDB from PEAT_DB.PEAT_dict import PEAT_dict, sub_dict import copy if local != None: newDB = PDatabase(local=local) elif server != None: newDB = PDatabase(server=server, username=username, port=8080, password='******', project=project) print newDB PT = peatDB.Database(datadir, Tk=False) oldDB = PT.DB print 'got old peat_db with %s proteins' % len(PT.proteins) print PT.DB.keys() #import meta stuff like userfields, table for p in newDB.meta.special: if not p in PT.DB.keys(): continue print 'adding', p for k in PT.DB[p]: newDB.meta[p][k] = copy.deepcopy(PT.DB[p][k]) newDB.meta._p_changed = 1 for p in PT.proteins: if p in newDB.meta.special: continue name = oldDB[p]['Name'] rec = PEATRecord(name=name) for col in oldDB[p].keys(): cdata = oldDB[p][col] recdata = {} if col == 'name': cdata = oldDB[p]['Name'] if oldDB['userfields'].has_key(col) and oldDB['userfields'][col][ 'field_type'] in ekintypes: E = EkinProject(data=cdata) E.length = len(E.datasets) if len(E.datasets) == 0: continue cdata = E if type(cdata) == sub_dict: for k in cdata.keys(): recdata[k] = copy.deepcopy(cdata[k]) else: recdata = cdata if cdata != '' and cdata != None: rec.addField(col, data=recdata) newDB.add(p, rec) print newDB.meta.userfields #remove any file cols, too hard to import for m in newDB.meta.userfields.keys()[:]: if newDB.meta.userfields[m]['field_type'] == 'File': newDB.deleteField(m) newDB.commit(user='******', note='import') newDB.close() print 'import done' return
from PEATDB.Ekin.Base import EkinProject from PEATDB.DictEdit import DictEditor import os import pickle cols = ['15N NMR', '1H NMR'] nuclnames = {'1H NMR': 'H', '15N NMR': 'N'} complete = [ 'HEWL', 'Bovine Beta-Lactoglobulin', 'Plastocyanin (Anabaena variabilis)', 'Plastocyanin (Phormidium)', 'Glutaredoxin', 'CexCD (Apo)', 'Protein G B1', 'Xylanase (Bacillus subtilus)' ] col = cols[0] nucl = nuclnames[col] t = TitrationAnalyser() #ghost mapping.. DB = PDatabase(server='peat.ucd.ie', username='******', password='******', project='titration_db', port=8080) p = t.extractpKas(DB, col, names=['HEWL'], titratable=False, reliable=False, minspan=0.06) t.mappKas(DB, col, p, names=['HEWL'], nucleus=nucl, calculatespans=False)