def uploadDensity(self, volfile):
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq['path'] = appiondata.ApPathData(
path=os.path.dirname(os.path.abspath(volfile)))
densq['name'] = os.path.basename(volfile)
densq['hidden'] = False
densq['norm'] = True
densq['symmetry'] = self.params['symdata']
#densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25)
densq['pixelsize'] = self.apix
densq['mass'] = self.mass
densq['boxsize'] = apFile.getBoxSize(volfile)[0]
densq['lowpass'] = self.params['lowpass']
#densq['highpass'] = self.params['highpasspart']
#densq['mask'] = self.params['radius']
densq['description'] = "EMDB id %d density" % (self.params['emdbid'])
if self.mass is not None:
densq['description'] += " with mass of %d kDa" % (self.mass)
densq['resolution'] = self.params['lowpass']
densq['session'] = self.sessiondata
densq['md5sum'] = apFile.md5sumfile(volfile)
densq['emdbid'] = self.params['emdbid']
if self.params['commit'] is True:
densq.insert()
return
def insertOtrRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq['type'] = "fsc" fscresq['half'] = self.fscresolution fscresq['fscfile'] = "fscdata"+self.timestamp+".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq['type'] = "rmeasure" rmeasureq['half'] = self.rmeasureresolution rmeasureq['fscfile'] = None ### insert rct run data otrrunq = appiondata.ApOtrRunData() otrrunq['runname'] = self.params['runname'] tempstr = "" for cnum in self.classlist: tempstr += str(cnum)+"," classliststr = tempstr[:-1] otrrunq['classnums'] = classliststr otrrunq['numiter'] = self.params['numiters'] otrrunq['euleriter'] = self.params['refineiters'] otrrunq['maskrad'] = self.params['radius'] otrrunq['lowpassvol'] = self.params['lowpassvol'] otrrunq['highpasspart'] = self.params['highpasspart'] otrrunq['median'] = self.params['median'] otrrunq['description'] = self.params['description'] otrrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) otrrunq['alignstack'] = self.alignstackdata otrrunq['tiltstack'] = apStack.getOnlyStackData(self.params['tiltstackid']) otrrunq['numpart'] = self.numpart otrrunq['fsc_resolution'] = fscresq otrrunq['rmeasure_resolution'] = rmeasureq if self.params['commit'] is True: otrrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['otrrun'] = otrrunq densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25) densq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] densq['boxsize'] = self.getBoxSize() densq['lowpass'] = self.params['lowpassvol'] densq['highpass'] = self.params['highpasspart'] densq['mask'] = self.params['radius'] #densq['iterid'] = self.params['numiters'] densq['description'] = self.params['description'] densq['resolution'] = self.fscresolution densq['rmeasure'] = self.rmeasureresolution densq['session'] = apStack.getSessionDataFromStackId(self.params['tiltstackid']) densq['md5sum'] = apFile.md5sumfile(volfile) if self.params['commit'] is True: densq.insert() return
def getMostRecentDensities(self, num=3):
densityq = appiondata.Ap3dDensityData()
densitydatas = densityq.query(results=num)
if not densitydatas:
apDisplay.printError("No densities found")
densityids = []
for densitydata in densitydatas:
densityids.append(densitydata.dbid)
return densityids
def insert3dDensity(self):
apDisplay.printMsg("committing density to database")
symdata = apSymmetry.findSymmetry(self.params['sym'])
if not symdata:
apDisplay.printError("no symmetry associated with this id\n")
self.params['syminfo'] = symdata
modq = appiondata.Ap3dDensityData()
sessiondata = apDatabase.getSessionDataFromSessionName(
self.params['sessionname'])
modq['session'] = sessiondata
modq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
modq['name'] = self.params['name']
modq['resolution'] = self.params['res']
modq['symmetry'] = symdata
modq['pixelsize'] = self.params['apix']
modq['boxsize'] = self.params['box']
modq['description'] = self.params['description']
modq['lowpass'] = self.params['lp']
modq['highpass'] = self.params['hp']
modq['mask'] = self.params['mask']
modq['imask'] = self.params['imask']
if self.params['reconiterid'] is not None:
iterdata = appiondata.ApRefineIterData.direct_query(
self.params['reconiterid'])
if not iterdata:
apDisplay.printError(
"this iteration was not found in the database\n")
modq['refineIter'] = iterdata
if self.params['reconid'] is not None:
iterdata = appiondata.ApRefineIterData.direct_query(
self.params['reconid'])
if not iterdata:
apDisplay.printError(
"this iteration was not found in the database\n")
modq['refineIter'] = iterdata
### if ampfile specified
if self.params['ampfile'] is not None:
(ampdir, ampname) = os.path.split(self.params['ampfile'])
modq['ampPath'] = appiondata.ApPathData(
path=os.path.abspath(ampdir))
modq['ampName'] = ampname
modq['maxfilt'] = self.params['maxfilt']
modq['handflip'] = self.params['yflip']
modq['norm'] = self.params['norm']
modq['invert'] = self.params['invert']
modq['hidden'] = False
filepath = os.path.join(self.params['rundir'], self.params['name'])
modq['md5sum'] = apFile.md5sumfile(filepath)
if self.params['commit'] is True:
modq.insert()
else:
apDisplay.printWarning("not commiting model to database")
def uploadDensity(self, volfile):
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile)))
densq['name'] = os.path.basename(volfile)
densq['hidden'] = False
densq['norm'] = True
densq['symmetry'] = self.symmdata
densq['pixelsize'] = self.clusterstackdata['clusterrun']['pixelsize']
densq['boxsize'] = self.clusterstackdata['clusterrun']['boxsize']
#densq['lowpass'] = None
#densq['highpass'] = None
densq['mask'] = self.params['mask']
densq['description'] = self.params['description']+"- From eman start-"+self.params['method']
#densq['resolution'] = None
densq['session'] = apStack.getSessionDataFromStackId(self.params['stackid'])
densq['md5sum'] = apFile.md5sumfile(volfile)
densq['eman'] = self.params['method']
if self.params['commit'] is True:
densq.insert()
return
def start(self):
self.params['outputstack'] = os.path.join(self.params['rundir'],
self.params['stackname'])
particles, self.params['refineiter'] = getParticleInfo(
self.params['reconid'], self.params['iter'])
stackdata = particles[0]['particle']['stack']
stack = os.path.join(stackdata['path']['path'], stackdata['name'])
classes, cstats = determineClasses(particles)
rejectlst = []
if self.params['sigma'] is not None:
cutoff = cstats[
'meanquality'] + self.params['sigma'] * cstats['stdquality']
apDisplay.printMsg("Cutoff = " + str(cutoff))
rejectlst = self.removePtclsByQualityFactor(
particles, rejectlst, cutoff)
if self.params['avgjump'] is not None:
rejectlst = self.removePtclsByJumps(particles, rejectlst)
if self.params['rejectlst']:
rejectlst = removePtclsByLst(rejectlst, self.params)
classkeys = classes.keys()
classkeys.sort()
classnum = 0
totalptcls = 0
keepfile = open('keep.lst', 'w')
keepfile.write('#LST\n')
reject = open('reject.lst', 'w')
reject.write('#LST\n')
apDisplay.printMsg("Processing classes")
#loop through classes
for key in classkeys:
# file to hold particles of this class
clsfile = open('clstmp.lst', 'w')
clsfile.write('#LST\n')
classnum += 1
if classnum % 10 == 1:
apDisplay.printMsg(
str(classnum) + " of " + (str(len(classkeys))))
images = EMAN.EMData()
#loop through particles in class
nptcls = 0
for ptcl in classes[key]['particles']:
if ptcl['mirror']:
mirror = 1
else:
mirror = 0
rot = ptcl['euler3']
rot = rot * math.pi / 180
if ptcl['particle']['particleNumber'] not in rejectlst:
l = '%d\t%s\t%f,\t%f,%f,%f,%d\n' % (
ptcl['particle']['particleNumber'] - 1, stack,
ptcl['quality_factor'], rot, ptcl['shiftx'],
ptcl['shifty'], mirror)
keepfile.write(l)
clsfile.write(l)
totalptcls += 1
nptcls += 1
else:
reject.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' %
(ptcl['particle']['particleNumber'] - 1,
stack, ptcl['quality_factor'], rot,
ptcl['shiftx'], ptcl['shifty'], mirror))
#if ptcl['quality_factor']>cstats['meanquality']+3*cstats['stdquality']:
# high.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1,
# stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror))
clsfile.close()
if nptcls < 1:
continue
if self.params['skipavg'] is False:
makeClassAverages('clstmp.lst', self.params['outputstack'],
classes[key], self.params)
if self.params['eotest'] is True:
self.makeEvenOddClasses('clstmp.lst', classes[key])
apDisplay.printMsg("\n")
reject.close()
keepfile.close()
os.remove('clstmp.lst')
# make 3d density file if specified:
if self.params['make3d'] is not None:
self.params['make3d'] = os.path.basename(self.params['make3d'])
outfile = os.path.join(self.params['rundir'],
self.params['make3d'])
apEMAN.make3d(self.params['stackname'],
outfile,
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d %s %s mask=%d norm" %
(outfile, outfile, self.params['mask']))
if self.params['eotest'] is True:
apEMAN.make3d(self.params['oddstack'],
"odd.mrc",
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.make3d(self.params['evenstack'],
"even.mrc",
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d odd.mrc even.mrc fsc=fsc.eotest")
if os.path.exists(outfile):
# run rmeasure
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
box = apVolume.getModelDimensions(outfile)
apDisplay.printMsg('inserting density into database')
symdata = apSymmetry.findSymmetry(self.params['sym'])
if not symdata:
apDisplay.printError('no symmetry associated with this model')
modq = appiondata.Ap3dDensityData()
modq['session'] = apStack.getSessionDataFromStackId(
self.params['stackid'])
modq['name'] = self.params['make3d']
modq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
modq['boxsize'] = box
modq['mask'] = self.params['mask']
modq['pixelsize'] = apix
fscres = apRecon.getResolutionFromFSCFile('fsc.eotest',
box,
apix,
msg=True)
modq['resolution'] = fscres
modq['rmeasure'] = apRecon.runRMeasure(apix, outfile)
modq['md5sum'] = apFile.md5sumfile(outfile)
modq['maxjump'] = self.params['avgjump']
modq['sigma'] = self.params['sigma']
modq['hard'] = self.params['hard']
modq['symmetry'] = symdata
modq['refineIter'] = self.params['refineiter']
if self.params['commit'] is True:
modq.insert()
apChimera.filterAndChimera(outfile,
res=fscres,
apix=apix,
box=box,
chimtype='snapshot',
zoom=self.params['zoom'],
sym=self.params['sym'],
mass=self.params['mass'])
else:
apDisplay.printError(
'no 3d volume was generated - check the class averages:')
apDisplay.printError(self.params['stackname'])
apDisplay.printError(
'hard may be set too high, or avg euler jump set too low for the # of particles'
)
stackstr = str(stackdata.dbid)
reconstr = str(self.params['reconid'])
apDisplay.printColor(
"Make a new stack with only non-jumpers:\n" +
"subStack.py --projectid=" + str(self.params['projectid']) +
" -s " + stackstr + " \\\n " + " -k " +
os.path.join(self.params['rundir'], "keep.lst") + " \\\n " +
" -d 'recon " + reconstr + " sitters' -n sitters" + reconstr +
" -C ", "purple")
