def uploadDensity(self, volfile):
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq['path'] = appiondata.ApPathData(
path=os.path.dirname(os.path.abspath(volfile)))
densq['name'] = os.path.basename(volfile)
densq['hidden'] = False
densq['norm'] = True
densq['symmetry'] = self.params['symdata']
#densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25)
densq['pixelsize'] = self.apix
densq['mass'] = self.mass
densq['boxsize'] = apFile.getBoxSize(volfile)[0]
densq['lowpass'] = self.params['lowpass']
#densq['highpass'] = self.params['highpasspart']
#densq['mask'] = self.params['radius']
densq['description'] = "EMDB id %d density" % (self.params['emdbid'])
if self.mass is not None:
densq['description'] += " with mass of %d kDa" % (self.mass)
densq['resolution'] = self.params['lowpass']
densq['session'] = self.sessiondata
densq['md5sum'] = apFile.md5sumfile(volfile)
densq['emdbid'] = self.params['emdbid']
if self.params['commit'] is True:
densq.insert()
return
def insertMisc(self):
print "inserting into database"
miscq = appiondata.ApMiscData()
if self.params['reconid'] is not None:
miscq['refineRun'] = self.recondata
sessiondata = apRecon.getSessionDataFromReconId(self.params['reconid'])
miscq['session'] = sessiondata
projectid = apProject.getProjectIdFromSessionName(sessiondata['name'])
miscq['REF|projectdata|projects|project'] = projectid
elif self.params['fulltomoid'] is not None:
miscq['fulltomogram'] = self.tomodata
sessiondata = self.tomodata['session']
miscq['session'] = sessiondata
projectid = apProject.getProjectIdFromSessionName(sessiondata['name'])
miscq['REF|projectdata|projects|project'] = projectid
elif self.params['sessionname'] is not None:
miscq['session'] = self.sessiondata
projectid = apProject.getProjectIdFromSessionName(self.params['sessionname'])
miscq['REF|projectdata|projects|project'] = projectid
miscq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
miscq['name'] = self.filename
miscq['description'] = self.params['description']
miscq['md5sum'] = apFile.md5sumfile(self.newfile)
miscq['hidden'] = False
if self.params['commit'] is True:
miscq.insert()
else:
apDisplay.printWarning("not committing to DB")
def uploadDensity(self, volfile): ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = self.params['symdata'] #densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25) densq['pixelsize'] = self.apix densq['mass'] = self.mass densq['boxsize'] = apFile.getBoxSize(volfile)[0] densq['lowpass'] = self.params['lowpass'] #densq['highpass'] = self.params['highpasspart'] #densq['mask'] = self.params['radius'] densq['description'] = "EMDB id %d density"%(self.params['emdbid']) if self.mass is not None: densq['description'] += " with mass of %d kDa"%(self.mass) densq['resolution'] = self.params['lowpass'] densq['session'] = self.sessiondata densq['md5sum'] = apFile.md5sumfile(volfile) densq['emdbid'] = self.params['emdbid'] if self.params['commit'] is True: densq.insert() return
def insertModel(self, mrcname):
apDisplay.printMsg("commiting model to database")
modq = appiondata.ApInitialModelData()
modq['REF|projectdata|projects|project'] = self.params['projectid']
modq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
modq['name'] = os.path.basename(mrcname)
modq['symmetry'] = self.params['symdata']
modq['pixelsize'] = self.params['newapix']
modq['boxsize'] = self.params['newbox']
modq['resolution'] = self.params['res']
modq['hidden'] = False
modq['md5sum'] = apFile.md5sumfile(mrcname)
modq['description'] = self.params['description']
if self.params['densityid'] is not None:
modq['original_density'] = appiondata.Ap3dDensityData.direct_query(
self.params['densityid'])
if self.params['oldmodelid'] is not None:
modq[
'original_model'] = appiondata.ApInitialModelData.direct_query(
self.params['oldmodelid'])
if self.params['commit'] is True:
modq.insert()
else:
apDisplay.printWarning("not commiting model to database")
def insertMisc(self):
print "inserting into database"
miscq = appiondata.ApMiscData()
if self.params['reconid'] is not None:
miscq['refineRun'] = self.recondata
sessiondata = apRecon.getSessionDataFromReconId(
self.params['reconid'])
miscq['session'] = sessiondata
projectid = apProject.getProjectIdFromSessionName(
sessiondata['name'])
miscq['REF|projectdata|projects|project'] = projectid
elif self.params['fulltomoid'] is not None:
miscq['fulltomogram'] = self.tomodata
sessiondata = self.tomodata['session']
miscq['session'] = sessiondata
projectid = apProject.getProjectIdFromSessionName(
sessiondata['name'])
miscq['REF|projectdata|projects|project'] = projectid
elif self.params['sessionname'] is not None:
miscq['session'] = self.sessiondata
projectid = apProject.getProjectIdFromSessionName(
self.params['sessionname'])
miscq['REF|projectdata|projects|project'] = projectid
miscq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
miscq['name'] = self.filename
miscq['description'] = self.params['description']
miscq['md5sum'] = apFile.md5sumfile(self.newfile)
miscq['hidden'] = False
if self.params['commit'] is True:
miscq.insert()
else:
apDisplay.printWarning("not committing to DB")
def insertOtrRun(self, volfile):
### setup resolutions
fscresq = appiondata.ApResolutionData()
fscresq["type"] = "fsc"
fscresq["half"] = self.fscresolution
fscresq["fscfile"] = "fscdata" + self.timestamp + ".fsc"
rmeasureq = appiondata.ApResolutionData()
rmeasureq["type"] = "rmeasure"
rmeasureq["half"] = self.rmeasureresolution
rmeasureq["fscfile"] = None
### insert rct run data
otrrunq = appiondata.ApOtrRunData()
otrrunq["runname"] = self.params["runname"]
tempstr = ""
for cnum in self.classlist:
tempstr += str(cnum) + ","
classliststr = tempstr[:-1]
otrrunq["classnums"] = classliststr
otrrunq["numiter"] = self.params["numiters"]
otrrunq["euleriter"] = self.params["refineiters"]
otrrunq["maskrad"] = self.params["radius"]
otrrunq["lowpassvol"] = self.params["lowpassvol"]
otrrunq["highpasspart"] = self.params["highpasspart"]
otrrunq["median"] = self.params["median"]
otrrunq["description"] = self.params["description"]
otrrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
otrrunq["alignstack"] = self.alignstackdata
otrrunq["tiltstack"] = apStack.getOnlyStackData(self.params["tiltstackid"])
otrrunq["numpart"] = self.numpart
otrrunq["fsc_resolution"] = fscresq
otrrunq["rmeasure_resolution"] = rmeasureq
if self.params["commit"] is True:
otrrunq.insert()
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq["otrrun"] = otrrunq
densq["path"] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile)))
densq["name"] = os.path.basename(volfile)
densq["hidden"] = False
densq["norm"] = True
densq["symmetry"] = appiondata.ApSymmetryData.direct_query(25)
densq["pixelsize"] = apStack.getStackPixelSizeFromStackId(self.params["partstackid"]) * self.params["bin"]
densq["boxsize"] = self.getBoxSize()
densq["lowpass"] = self.params["lowpassvol"]
densq["highpass"] = self.params["highpasspart"]
densq["mask"] = self.params["radius"]
# densq['iterid'] = self.params['numiters']
densq["description"] = self.params["description"]
densq["resolution"] = self.fscresolution
densq["rmeasure"] = self.rmeasureresolution
densq["session"] = apStack.getSessionDataFromStackId(self.params["tiltstackid"])
densq["md5sum"] = apFile.md5sumfile(volfile)
if self.params["commit"] is True:
densq.insert()
return
def insertTemplateImage(params):
for i, name in enumerate(params['templatelist']):
if os.path.basename(name) != name:
apDisplay.printError(
"please contact an appion developer, because the database insert is wrong"
)
#check if template exists
templateq = appiondata.ApTemplateImageData()
templateq['path'] = appiondata.ApPathData(
path=os.path.abspath(params['rundir']))
templateq['templatename'] = name
templateId = templateq.query(results=1)
if templateId:
apDisplay.printWarning(
"template already in database.\nNot reinserting")
continue
#check if duplicate template exists
temppath = os.path.join(params['rundir'], name)
md5sum = apFile.md5sumfile(temppath)
templateq2 = appiondata.ApTemplateImageData()
templateq2['md5sum'] = md5sum
templateId = templateq2.query(results=1)
if templateId:
apDisplay.printWarning(
"template with the same check sum already exists in database.\nNot reinserting"
)
continue
#insert template to database if doesn't exist
print "Inserting", name, "into the template database"
templateq['apix'] = params['apix']
templateq['diam'] = params['diam']
templateq['md5sum'] = md5sum
if 'alignid' in params and params['alignid'] is not None:
templateq['alignstack'] = appiondata.ApAlignStackData.direct_query(
params['alignid'])
if 'clusterid' in params and params['clusterid'] is not None:
templateq[
'clusterstack'] = appiondata.ApClusteringStackData.direct_query(
params['clusterid'])
if 'stackid' in params and params['stackid'] is not None:
templateq['stack'] = appiondata.ApStackData.direct_query(
params['stackid'])
if 'imgnums' in params and params['imgnums'] is not None:
imgnums = params['imgnums'].split(",")
templateq['stack_image_number'] = int(imgnums[i])
templateq['description'] = params['description']
templateq['REF|projectdata|projects|project'] = params['projectId']
## PHP web tools expect 'hidden' field, set it to False initially
templateq['hidden'] = False
if params['commit'] is True:
time.sleep(2)
templateq.insert()
else:
apDisplay.printWarning("Not commiting template to DB")
return
def insertOtrRun(self, volfile): ### setup resolutions fscresq = appiondata.ApResolutionData() fscresq['type'] = "fsc" fscresq['half'] = self.fscresolution fscresq['fscfile'] = "fscdata"+self.timestamp+".fsc" rmeasureq = appiondata.ApResolutionData() rmeasureq['type'] = "rmeasure" rmeasureq['half'] = self.rmeasureresolution rmeasureq['fscfile'] = None ### insert rct run data otrrunq = appiondata.ApOtrRunData() otrrunq['runname'] = self.params['runname'] tempstr = "" for cnum in self.classlist: tempstr += str(cnum)+"," classliststr = tempstr[:-1] otrrunq['classnums'] = classliststr otrrunq['numiter'] = self.params['numiters'] otrrunq['euleriter'] = self.params['refineiters'] otrrunq['maskrad'] = self.params['radius'] otrrunq['lowpassvol'] = self.params['lowpassvol'] otrrunq['highpasspart'] = self.params['highpasspart'] otrrunq['median'] = self.params['median'] otrrunq['description'] = self.params['description'] otrrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) otrrunq['alignstack'] = self.alignstackdata otrrunq['tiltstack'] = apStack.getOnlyStackData(self.params['tiltstackid']) otrrunq['numpart'] = self.numpart otrrunq['fsc_resolution'] = fscresq otrrunq['rmeasure_resolution'] = rmeasureq if self.params['commit'] is True: otrrunq.insert() ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['otrrun'] = otrrunq densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = appiondata.ApSymmetryData.direct_query(25) densq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.params['tiltstackid'])*self.params['tiltbin'] densq['boxsize'] = self.getBoxSize() densq['lowpass'] = self.params['lowpassvol'] densq['highpass'] = self.params['highpasspart'] densq['mask'] = self.params['radius'] #densq['iterid'] = self.params['numiters'] densq['description'] = self.params['description'] densq['resolution'] = self.fscresolution densq['rmeasure'] = self.rmeasureresolution densq['session'] = apStack.getSessionDataFromStackId(self.params['tiltstackid']) densq['md5sum'] = apFile.md5sumfile(volfile) if self.params['commit'] is True: densq.insert() return
def copyTemplatesToOutdir(params, timestamp=None):
newlist = []
for tmpl in params['templatelist']:
base = os.path.basename(tmpl)
old = os.path.abspath(tmpl)
### Rename file for new location
if timestamp is None:
timestamp = apParam.makeTimestamp()
#append the name of the directory to the filename
#basedir = os.path.split(os.path.dirname(old))[1]
#base = basedir+"_"+base
name, ext = os.path.splitext(base)
base = name + "-" + timestamp + ext
new = os.path.join(params['rundir'], base)
if os.path.isfile(new):
mdnew = apFile.md5sumfile(new)
mdold = apFile.md5sumfile(old)
if mdnew != mdold:
apDisplay.printError("a different template with name \'" +
new + "\' already exists!")
elif apDatabase.isTemplateInDB(mdnew):
apDisplay.printWarning("same template with md5sum \'" + mdnew +
"\' already exists in the DB!")
apDisplay.printMsg("skipping template \'" + old + "\'")
else:
apDisplay.printWarning("the same template with name \'" + new +
"\' already exists!")
newlist.append(base)
else:
#template is okay to copy and insert
apDisplay.printMsg("copying file " + old + " to " + new)
shutil.copyfile(old, new)
newlist.append(base)
#and only allow user read access just so they don't get deleted
#os.lchmod(new, 0666)
os.chmod(new, 0666)
params['templatelist'] = newlist
apDisplay.printColor("New template List:", "green")
pprint.pprint(params['templatelist'])
return
def insert3dDensity(self):
apDisplay.printMsg("committing density to database")
symdata = apSymmetry.findSymmetry(self.params['sym'])
if not symdata:
apDisplay.printError("no symmetry associated with this id\n")
self.params['syminfo'] = symdata
modq = appiondata.Ap3dDensityData()
sessiondata = apDatabase.getSessionDataFromSessionName(
self.params['sessionname'])
modq['session'] = sessiondata
modq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
modq['name'] = self.params['name']
modq['resolution'] = self.params['res']
modq['symmetry'] = symdata
modq['pixelsize'] = self.params['apix']
modq['boxsize'] = self.params['box']
modq['description'] = self.params['description']
modq['lowpass'] = self.params['lp']
modq['highpass'] = self.params['hp']
modq['mask'] = self.params['mask']
modq['imask'] = self.params['imask']
if self.params['reconiterid'] is not None:
iterdata = appiondata.ApRefineIterData.direct_query(
self.params['reconiterid'])
if not iterdata:
apDisplay.printError(
"this iteration was not found in the database\n")
modq['refineIter'] = iterdata
if self.params['reconid'] is not None:
iterdata = appiondata.ApRefineIterData.direct_query(
self.params['reconid'])
if not iterdata:
apDisplay.printError(
"this iteration was not found in the database\n")
modq['refineIter'] = iterdata
### if ampfile specified
if self.params['ampfile'] is not None:
(ampdir, ampname) = os.path.split(self.params['ampfile'])
modq['ampPath'] = appiondata.ApPathData(
path=os.path.abspath(ampdir))
modq['ampName'] = ampname
modq['maxfilt'] = self.params['maxfilt']
modq['handflip'] = self.params['yflip']
modq['norm'] = self.params['norm']
modq['invert'] = self.params['invert']
modq['hidden'] = False
filepath = os.path.join(self.params['rundir'], self.params['name'])
modq['md5sum'] = apFile.md5sumfile(filepath)
if self.params['commit'] is True:
modq.insert()
else:
apDisplay.printWarning("not commiting model to database")
def copyTemplatesToOutdir(params, timestamp=None):
newlist = []
for tmpl in params['templatelist']:
base = os.path.basename(tmpl)
old = os.path.abspath(tmpl)
### Rename file for new location
if timestamp is None:
timestamp = apParam.makeTimestamp()
#append the name of the directory to the filename
#basedir = os.path.split(os.path.dirname(old))[1]
#base = basedir+"_"+base
name,ext = os.path.splitext(base)
base = name+"-"+timestamp+ext
new = os.path.join(params['rundir'], base)
if os.path.isfile(new):
mdnew = apFile.md5sumfile(new)
mdold = apFile.md5sumfile(old)
if mdnew != mdold:
apDisplay.printError("a different template with name \'"+new+"\' already exists!")
elif apDatabase.isTemplateInDB(mdnew):
apDisplay.printWarning("same template with md5sum \'"+mdnew+"\' already exists in the DB!")
apDisplay.printMsg("skipping template \'"+old+"\'")
else:
apDisplay.printWarning("the same template with name \'"+new+"\' already exists!")
newlist.append(base)
else:
#template is okay to copy and insert
apDisplay.printMsg("copying file "+old+" to "+new)
shutil.copyfile(old, new)
newlist.append(base)
#and only allow user read access just so they don't get deleted
#os.lchmod(new, 0666)
os.chmod(new, 0666)
params['templatelist'] = newlist
apDisplay.printColor("New template List:","green")
pprint.pprint(params['templatelist'])
return
def insertTemplateImage(params):
for i,name in enumerate(params['templatelist']):
if os.path.basename(name) != name:
apDisplay.printError("please contact an appion developer, because the database insert is wrong")
#check if template exists
templateq=appiondata.ApTemplateImageData()
templateq['path'] = appiondata.ApPathData(path=os.path.abspath(params['rundir']))
templateq['templatename']=name
templateId = templateq.query(results=1)
if templateId:
apDisplay.printWarning("template already in database.\nNot reinserting")
continue
#check if duplicate template exists
temppath = os.path.join(params['rundir'], name)
md5sum = apFile.md5sumfile(temppath)
templateq2=appiondata.ApTemplateImageData()
templateq2['md5sum']=md5sum
templateId = templateq2.query(results=1)
if templateId:
apDisplay.printWarning("template with the same check sum already exists in database.\nNot reinserting")
continue
#insert template to database if doesn't exist
print "Inserting",name,"into the template database"
templateq['apix']=params['apix']
templateq['diam']=params['diam']
templateq['md5sum']=md5sum
if 'alignid' in params and params['alignid'] is not None:
templateq['alignstack'] = appiondata.ApAlignStackData.direct_query(params['alignid'])
if 'clusterid' in params and params['clusterid'] is not None:
templateq['clusterstack'] = appiondata.ApClusteringStackData.direct_query(params['clusterid'])
if 'stackid' in params and params['stackid'] is not None:
templateq['stack'] = appiondata.ApStackData.direct_query(params['stackid'])
if 'imgnums' in params and params['imgnums'] is not None:
imgnums = params['imgnums'].split(",")
templateq['stack_image_number']=int(imgnums[i])
templateq['description']=params['description']
templateq['REF|projectdata|projects|project']=params['projectId']
## PHP web tools expect 'hidden' field, set it to False initially
templateq['hidden'] = False
if params['commit'] is True:
time.sleep(2)
templateq.insert()
else:
apDisplay.printWarning("Not commiting template to DB")
return
def insert3dDensity(self):
apDisplay.printMsg("committing density to database")
symdata=apSymmetry.findSymmetry(self.params['sym'])
if not symdata:
apDisplay.printError("no symmetry associated with this id\n")
self.params['syminfo'] = symdata
modq=appiondata.Ap3dDensityData()
sessiondata = apDatabase.getSessionDataFromSessionName(self.params['sessionname'])
modq['session'] = sessiondata
modq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
modq['name'] = self.params['name']
modq['resolution'] = self.params['res']
modq['symmetry'] = symdata
modq['pixelsize'] = self.params['apix']
modq['boxsize'] = self.params['box']
modq['description'] = self.params['description']
modq['lowpass'] = self.params['lp']
modq['highpass'] = self.params['hp']
modq['mask'] = self.params['mask']
modq['imask'] = self.params['imask']
if self.params['reconiterid'] is not None:
iterdata = appiondata.ApHipIterData.direct_query(self.params['reconiterid'])
if not iterdata:
apDisplay.printError("this iteration was not found in the database\n")
modq['hipIter'] = iterdata
if self.params['reconid'] is not None:
iterdata = appiondata.ApHipIterData.direct_query(self.params['reconid'])
if not iterdata:
apDisplay.printError("this iteration was not found in the database\n")
modq['hipIter'] = iterdata
### if ampfile specified
if self.params['ampfile'] is not None:
(ampdir, ampname) = os.path.split(self.params['ampfile'])
modq['ampPath'] = appiondata.ApPathData(path=os.path.abspath(ampdir))
modq['ampName'] = ampname
modq['maxfilt'] = self.params['maxfilt']
modq['handflip'] = self.params['yflip']
modq['norm'] = self.params['norm']
modq['invert'] = self.params['invert']
modq['hidden'] = False
filepath = os.path.join(self.params['rundir'], self.params['name'])
modq['md5sum'] = apFile.md5sumfile(filepath)
if self.params['commit'] is True:
modq.insert()
else:
apDisplay.printWarning("not commiting model to database")
def insertSubTomogram(fulltomodata,rundata,center,offsetz,dimension,path,name,index,pixelsize,description):
tomoq = appiondata.ApTomogramData(fulltomogram=fulltomodata)
tomoq['session'] = fulltomodata['session']
tomoq['tiltseries'] = fulltomodata['tiltseries']
tomoq['subtomorun'] = rundata
tomoq['path'] = appiondata.ApPathData(path=os.path.abspath(path))
tomoq['name'] = name
tomoq['number'] = index
tomoq['center'] = center
# offsetz is in pixel of the full tomogram
tomoq['offsetz'] = offsetz
# dimension is that of the original tilt images, i.e., before binning of the full tomogram
tomoq['dimension'] = dimension
# pixelsize is of the full_bin * sub_bin tomogram
tomoq['pixelsize'] = pixelsize
tomoq['description'] = description
filepath = os.path.join(path,name+".rec")
tomoq['md5sum'] = apFile.md5sumfile(filepath)
return publish(tomoq)
def insertSubTomogram(fulltomodata, rundata, center, offsetz, dimension, path,
name, index, pixelsize, description):
tomoq = appiondata.ApTomogramData(fulltomogram=fulltomodata)
tomoq['session'] = fulltomodata['session']
tomoq['tiltseries'] = fulltomodata['tiltseries']
tomoq['subtomorun'] = rundata
tomoq['path'] = appiondata.ApPathData(path=os.path.abspath(path))
tomoq['name'] = name
tomoq['number'] = index
tomoq['center'] = center
# offsetz is in pixel of the full tomogram
tomoq['offsetz'] = offsetz
# dimension is that of the original tilt images, i.e., before binning of the full tomogram
tomoq['dimension'] = dimension
# pixelsize is of the full_bin * sub_bin tomogram
tomoq['pixelsize'] = pixelsize
tomoq['description'] = description
filepath = os.path.join(path, name + ".rec")
tomoq['md5sum'] = apFile.md5sumfile(filepath)
return publish(tomoq)
def insertModel(self, mrcname):
apDisplay.printMsg("commiting model to database")
modq=appiondata.ApInitialModelData()
modq['REF|projectdata|projects|project'] = self.params['projectid']
modq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
modq['name'] = os.path.basename(mrcname)
modq['symmetry'] = self.params['symdata']
modq['pixelsize'] = self.params['newapix']
modq['boxsize'] = self.params['newbox']
modq['resolution'] = self.params['res']
modq['hidden'] = False
modq['md5sum'] = apFile.md5sumfile(mrcname)
modq['description'] = self.params['description']
if self.params['densityid'] is not None:
modq['original_density'] = appiondata.Ap3dDensityData.direct_query(self.params['densityid'])
if self.params['oldmodelid'] is not None:
modq['original_model'] = appiondata.ApInitialModelData.direct_query(self.params['oldmodelid'])
if self.params['commit'] is True:
modq.insert()
else:
apDisplay.printWarning("not commiting model to database")
def uploadDensity(self, volfile): ### insert 3d volume density densq = appiondata.Ap3dDensityData() densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile))) densq['name'] = os.path.basename(volfile) densq['hidden'] = False densq['norm'] = True densq['symmetry'] = self.symmdata densq['pixelsize'] = self.clusterstackdata['clusterrun']['pixelsize'] densq['boxsize'] = self.clusterstackdata['clusterrun']['boxsize'] #densq['lowpass'] = None #densq['highpass'] = None densq['mask'] = self.params['mask'] densq['description'] = self.params['description']+"- From eman start-"+self.params['method'] #densq['resolution'] = None densq['session'] = apStack.getSessionDataFromStackId(self.params['stackid']) densq['md5sum'] = apFile.md5sumfile(volfile) densq['eman'] = self.params['method'] if self.params['commit'] is True: densq.insert() return
def uploadDensity(self, volfile):
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq['path'] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile)))
densq['name'] = os.path.basename(volfile)
densq['hidden'] = False
densq['norm'] = True
densq['symmetry'] = self.symmdata
densq['pixelsize'] = self.clusterstackdata['clusterrun']['pixelsize']
densq['boxsize'] = self.clusterstackdata['clusterrun']['boxsize']
#densq['lowpass'] = None
#densq['highpass'] = None
densq['mask'] = self.params['mask']
densq['description'] = self.params['description']+"- From eman start-"+self.params['method']
#densq['resolution'] = None
densq['session'] = apStack.getSessionDataFromStackId(self.params['stackid'])
densq['md5sum'] = apFile.md5sumfile(volfile)
densq['eman'] = self.params['method']
if self.params['commit'] is True:
densq.insert()
return
def uploadDensity(self, volfile):
### insert 3d volume density
densq = appiondata.Ap3dDensityData()
densq["path"] = appiondata.ApPathData(path=os.path.dirname(os.path.abspath(volfile)))
densq["name"] = os.path.basename(volfile)
densq["hidden"] = False
densq["norm"] = True
densq["symmetry"] = self.params["symdata"]
densq["pixelsize"] = self.params["apix"]
densq["mass"] = self.params["mass"]
densq["boxsize"] = self.params["boxsize"]
densq["lowpass"] = self.params["lowpass"]
# densq['highpass'] = self.params['highpasspart']
# densq['mask'] = self.params['radius']
densq["description"] = "PDB id %s density" % (self.params["pdbid"])
if self.params["mass"] is not None:
densq["description"] += " with mass of %d kDa" % (self.params["mass"])
densq["resolution"] = self.params["lowpass"]
densq["session"] = self.sessiondata
densq["md5sum"] = apFile.md5sumfile(volfile)
densq["pdbid"] = self.params["pdbid"]
if self.params["commit"] is True:
densq.insert()
return
def start(self):
self.params["outputstack"] = os.path.join(self.params["rundir"], self.params["stackname"])
particles, self.params["refineiter"] = getParticleInfo(self.params["reconid"], self.params["iter"])
stackdata = particles[0]["particle"]["stack"]
stack = os.path.join(stackdata["path"]["path"], stackdata["name"])
classes, cstats = determineClasses(particles)
rejectlst = []
if self.params["sigma"] is not None:
cutoff = cstats["meanquality"] + self.params["sigma"] * cstats["stdquality"]
apDisplay.printMsg("Cutoff = " + str(cutoff))
rejectlst = self.removePtclsByQualityFactor(particles, rejectlst, cutoff)
if self.params["avgjump"] is not None:
rejectlst = self.removePtclsByJumps(particles, rejectlst)
if self.params["rejectlst"]:
rejectlst = removePtclsByLst(rejectlst, self.params)
classkeys = classes.keys()
classkeys.sort()
classnum = 0
totalptcls = 0
keepfile = open("keep.lst", "w")
keepfile.write("#LST\n")
reject = open("reject.lst", "w")
reject.write("#LST\n")
apDisplay.printMsg("Processing classes")
# loop through classes
for key in classkeys:
# file to hold particles of this class
clsfile = open("clstmp.lst", "w")
clsfile.write("#LST\n")
classnum += 1
if classnum % 10 == 1:
apDisplay.printMsg(str(classnum) + " of " + (str(len(classkeys))))
images = EMAN.EMData()
# loop through particles in class
nptcls = 0
for ptcl in classes[key]["particles"]:
if ptcl["mirror"]:
mirror = 1
else:
mirror = 0
rot = ptcl["euler3"]
rot = rot * math.pi / 180
if ptcl["particle"]["particleNumber"] not in rejectlst:
l = "%d\t%s\t%f,\t%f,%f,%f,%d\n" % (
ptcl["particle"]["particleNumber"] - 1,
stack,
ptcl["quality_factor"],
rot,
ptcl["shiftx"],
ptcl["shifty"],
mirror,
)
keepfile.write(l)
clsfile.write(l)
totalptcls += 1
nptcls += 1
else:
reject.write(
"%d\t%s\t%f,\t%f,%f,%f,%d\n"
% (
ptcl["particle"]["particleNumber"] - 1,
stack,
ptcl["quality_factor"],
rot,
ptcl["shiftx"],
ptcl["shifty"],
mirror,
)
)
# if ptcl['quality_factor']>cstats['meanquality']+3*cstats['stdquality']:
# high.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1,
# stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror))
clsfile.close()
if nptcls < 1:
continue
if self.params["skipavg"] is False:
makeClassAverages("clstmp.lst", self.params["outputstack"], classes[key], self.params)
if self.params["eotest"] is True:
self.makeEvenOddClasses("clstmp.lst", classes[key])
apDisplay.printMsg("\n")
reject.close()
keepfile.close()
os.remove("clstmp.lst")
# make 3d density file if specified:
if self.params["make3d"] is not None:
self.params["make3d"] = os.path.basename(self.params["make3d"])
outfile = os.path.join(self.params["rundir"], self.params["make3d"])
apEMAN.make3d(
self.params["stackname"],
outfile,
sym=self.params["sym"],
mode=self.params["mode"],
hard=self.params["hard"],
)
apEMAN.executeEmanCmd("proc3d %s %s mask=%d norm" % (outfile, outfile, self.params["mask"]))
if self.params["eotest"] is True:
apEMAN.make3d(
self.params["oddstack"],
"odd.mrc",
sym=self.params["sym"],
mode=self.params["mode"],
hard=self.params["hard"],
)
apEMAN.make3d(
self.params["evenstack"],
"even.mrc",
sym=self.params["sym"],
mode=self.params["mode"],
hard=self.params["hard"],
)
apEMAN.executeEmanCmd("proc3d odd.mrc even.mrc fsc=fsc.eotest")
if os.path.exists(outfile):
# run rmeasure
apix = apStack.getStackPixelSizeFromStackId(self.params["stackid"])
box = apVolume.getModelDimensions(outfile)
apDisplay.printMsg("inserting density into database")
symdata = apSymmetry.findSymmetry(self.params["sym"])
if not symdata:
apDisplay.printError("no symmetry associated with this model")
modq = appiondata.Ap3dDensityData()
modq["session"] = self.params["sessionname"]
modq["name"] = self.params["make3d"]
modq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
modq["boxsize"] = box
modq["mask"] = self.params["mask"]
modq["pixelsize"] = apix
fscres = apRecon.getResolutionFromFSCFile("fsc.eotest", box, apix, msg=True)
modq["resolution"] = fscres
modq["rmeasure"] = apRecon.runRMeasure(apix, outfile)
modq["md5sum"] = apFile.md5sumfile(outfile)
modq["maxjump"] = self.params["avgjump"]
modq["sigma"] = self.params["sigma"]
modq["hard"] = self.params["hard"]
modq["symmetry"] = symdata
modq["refineIter"] = self.params["refineiter"]
if self.params["commit"] is True:
modq.insert()
apChimera.filterAndChimera(
outfile,
res=fscres,
apix=apix,
box=box,
chimtype="snapshot",
zoom=self.params["zoom"],
sym=self.params["sym"],
mass=self.params["mass"],
)
else:
apDisplay.printError("no 3d volume was generated - check the class averages:")
apDisplay.printError(self.params["stackname"])
apDisplay.printError("hard may be set too high, or avg euler jump set too low for the # of particles")
stackstr = str(stackdata.dbid)
reconstr = str(self.params["reconid"])
apDisplay.printColor(
"Make a new stack with only non-jumpers:\n"
+ "subStack.py --projectid="
+ str(self.params["projectid"])
+ " -s "
+ stackstr
+ " \\\n "
+ " -k "
+ os.path.join(self.params["rundir"], "keep.lst")
+ " \\\n "
+ " -d 'recon "
+ reconstr
+ " sitters' -n sitters"
+ reconstr
+ " -C ",
"purple",
)
def start(self):
self.params['outputstack'] = os.path.join(self.params['rundir'],
self.params['stackname'])
particles, self.params['refineiter'] = getParticleInfo(
self.params['reconid'], self.params['iter'])
stackdata = particles[0]['particle']['stack']
stack = os.path.join(stackdata['path']['path'], stackdata['name'])
classes, cstats = determineClasses(particles)
rejectlst = []
if self.params['sigma'] is not None:
cutoff = cstats[
'meanquality'] + self.params['sigma'] * cstats['stdquality']
apDisplay.printMsg("Cutoff = " + str(cutoff))
rejectlst = self.removePtclsByQualityFactor(
particles, rejectlst, cutoff)
if self.params['avgjump'] is not None:
rejectlst = self.removePtclsByJumps(particles, rejectlst)
if self.params['rejectlst']:
rejectlst = removePtclsByLst(rejectlst, self.params)
classkeys = classes.keys()
classkeys.sort()
classnum = 0
totalptcls = 0
keepfile = open('keep.lst', 'w')
keepfile.write('#LST\n')
reject = open('reject.lst', 'w')
reject.write('#LST\n')
apDisplay.printMsg("Processing classes")
#loop through classes
for key in classkeys:
# file to hold particles of this class
clsfile = open('clstmp.lst', 'w')
clsfile.write('#LST\n')
classnum += 1
if classnum % 10 == 1:
apDisplay.printMsg(
str(classnum) + " of " + (str(len(classkeys))))
images = EMAN.EMData()
#loop through particles in class
nptcls = 0
for ptcl in classes[key]['particles']:
if ptcl['mirror']:
mirror = 1
else:
mirror = 0
rot = ptcl['euler3']
rot = rot * math.pi / 180
if ptcl['particle']['particleNumber'] not in rejectlst:
l = '%d\t%s\t%f,\t%f,%f,%f,%d\n' % (
ptcl['particle']['particleNumber'] - 1, stack,
ptcl['quality_factor'], rot, ptcl['shiftx'],
ptcl['shifty'], mirror)
keepfile.write(l)
clsfile.write(l)
totalptcls += 1
nptcls += 1
else:
reject.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' %
(ptcl['particle']['particleNumber'] - 1,
stack, ptcl['quality_factor'], rot,
ptcl['shiftx'], ptcl['shifty'], mirror))
#if ptcl['quality_factor']>cstats['meanquality']+3*cstats['stdquality']:
# high.write('%d\t%s\t%f,\t%f,%f,%f,%d\n' % (ptcl['particle']['particleNumber']-1,
# stack,ptcl['quality_factor'],rot,ptcl['shiftx'],ptcl['shifty'],mirror))
clsfile.close()
if nptcls < 1:
continue
if self.params['skipavg'] is False:
makeClassAverages('clstmp.lst', self.params['outputstack'],
classes[key], self.params)
if self.params['eotest'] is True:
self.makeEvenOddClasses('clstmp.lst', classes[key])
apDisplay.printMsg("\n")
reject.close()
keepfile.close()
os.remove('clstmp.lst')
# make 3d density file if specified:
if self.params['make3d'] is not None:
self.params['make3d'] = os.path.basename(self.params['make3d'])
outfile = os.path.join(self.params['rundir'],
self.params['make3d'])
apEMAN.make3d(self.params['stackname'],
outfile,
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d %s %s mask=%d norm" %
(outfile, outfile, self.params['mask']))
if self.params['eotest'] is True:
apEMAN.make3d(self.params['oddstack'],
"odd.mrc",
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.make3d(self.params['evenstack'],
"even.mrc",
sym=self.params['sym'],
mode=self.params['mode'],
hard=self.params['hard'])
apEMAN.executeEmanCmd("proc3d odd.mrc even.mrc fsc=fsc.eotest")
if os.path.exists(outfile):
# run rmeasure
apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
box = apVolume.getModelDimensions(outfile)
apDisplay.printMsg('inserting density into database')
symdata = apSymmetry.findSymmetry(self.params['sym'])
if not symdata:
apDisplay.printError('no symmetry associated with this model')
modq = appiondata.Ap3dDensityData()
modq['session'] = apStack.getSessionDataFromStackId(
self.params['stackid'])
modq['name'] = self.params['make3d']
modq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
modq['boxsize'] = box
modq['mask'] = self.params['mask']
modq['pixelsize'] = apix
fscres = apRecon.getResolutionFromFSCFile('fsc.eotest',
box,
apix,
msg=True)
modq['resolution'] = fscres
modq['rmeasure'] = apRecon.runRMeasure(apix, outfile)
modq['md5sum'] = apFile.md5sumfile(outfile)
modq['maxjump'] = self.params['avgjump']
modq['sigma'] = self.params['sigma']
modq['hard'] = self.params['hard']
modq['symmetry'] = symdata
modq['refineIter'] = self.params['refineiter']
if self.params['commit'] is True:
modq.insert()
apChimera.filterAndChimera(outfile,
res=fscres,
apix=apix,
box=box,
chimtype='snapshot',
zoom=self.params['zoom'],
sym=self.params['sym'],
mass=self.params['mass'])
else:
apDisplay.printError(
'no 3d volume was generated - check the class averages:')
apDisplay.printError(self.params['stackname'])
apDisplay.printError(
'hard may be set too high, or avg euler jump set too low for the # of particles'
)
stackstr = str(stackdata.dbid)
reconstr = str(self.params['reconid'])
apDisplay.printColor(
"Make a new stack with only non-jumpers:\n" +
"subStack.py --projectid=" + str(self.params['projectid']) +
" -s " + stackstr + " \\\n " + " -k " +
os.path.join(self.params['rundir'], "keep.lst") + " \\\n " +
" -d 'recon " + reconstr + " sitters' -n sitters" + reconstr +
" -C ", "purple")
