def insertRotKerDenSOM(self):
inserttime = time.time()
### Preliminary data
projectid = apProject.getProjectIdFromAlignStackId(
self.params['alignstackid'])
alignstackdata = appiondata.ApAlignStackData.direct_query(
self.params['alignstackid'])
numclass = self.params['xdim'] * self.params['ydim']
pathdata = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
### rotKerDen SOM Params object
rotkerdenson = appiondata.ApRotKerDenSOMParamsData()
#rotkerdenson['mask_diam'] = 2.0*self.params['maskrad']
rotkerdenson['x_dimension'] = self.params['xdim']
rotkerdenson['y_dimension'] = self.params['ydim']
#rotkerdenson['convergence'] = self.params['converge']
rotkerdenson['run_seconds'] = time.time() - self.t0
rotkerdenson['initregulfact'] = self.params['initregulfact']
rotkerdenson['finalregulfact'] = self.params['finalregulfact']
rotkerdenson['incrementregulfact'] = self.params['incrementregulfact']
rotkerdenson['spectrainnerradius'] = self.params['spectrainnerradius']
rotkerdenson['spectraouterradius'] = self.params['spectraouterradius']
rotkerdenson['spectralowharmonic'] = self.params['spectralowharmonic']
rotkerdenson['spectrahighharmonic'] = self.params[
'spectrahighharmonic']
### Analysis Run object
analysisq = appiondata.ApAlignAnalysisRunData()
analysisq['runname'] = self.params['runname']
analysisq['path'] = pathdata
analysisq['description'] = self.params['description']
analysisq['alignstack'] = alignstackdata
analysisq['hidden'] = False
### Clustering Run object
clusterrunq = appiondata.ApClusteringRunData()
clusterrunq['runname'] = self.params['runname']
clusterrunq['description'] = self.params['description']
clusterrunq['boxsize'] = alignstackdata['boxsize']
clusterrunq['pixelsize'] = alignstackdata['pixelsize']
clusterrunq['num_particles'] = self.params['numpart']
clusterrunq['alignstack'] = alignstackdata
clusterrunq['analysisrun'] = analysisq
clusterrunq['rotkerdenparams'] = rotkerdenson
### Clustering Stack object
#Stack with cluster averages??????
template = os.path.join(self.params['rundir'],
self.spectraTemporalFilesMask + ".png")
files = glob.glob(template)
imglist = []
for listname in files:
a = apImage.readPNG(listname)
imglist.append(a)
apImagicFile.writeImagic(imglist,
"rotkerdenstack" + self.timestamp + ".hed")
clusterstackq = appiondata.ApClusteringStackData()
clusterstackq[
'avg_imagicfile'] = "rotkerdenstack" + self.timestamp + ".hed"
clusterstackq['num_classes'] = numclass
clusterstackq['clusterrun'] = clusterrunq
clusterstackq['path'] = pathdata
clusterstackq['hidden'] = False
imagicfile = os.path.join(self.params['rundir'],
clusterstackq['avg_imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find average stack file: " +
imagicfile)
### looping over clusters
apDisplay.printColor(
"Inserting particle classification data, please wait", "cyan")
numclass = self.params['xdim'] * self.params['ydim']
for i in range(numclass):
classnum = i + 1
classroot = "%s.%d" % (self.timestamp, classnum - 1)
classdocfile = os.path.join(self.params['rundir'], classroot)
partlist = self.readClassDocFile(classdocfile)
### Clustering Particle object
# MRC image for each code node but plot or image
clusterrefq = appiondata.ApClusteringReferenceData()
clusterrefq['refnum'] = classnum
clusterrefq['avg_mrcfile'] = classroot + ".mrc"
clusterrefq['clusterrun'] = clusterrunq
clusterrefq['path'] = pathdata
clusterrefq['num_particles'] = len(partlist)
### looping over particles
#which particles belong to which code node
sys.stderr.write(".")
for partnum in partlist:
alignpartdata = self.getAlignParticleData(
partnum, alignstackdata)
### Clustering Particle objects
clusterpartq = appiondata.ApClusteringParticleData()
clusterpartq['clusterstack'] = clusterstackq
clusterpartq['alignparticle'] = alignpartdata
clusterpartq['partnum'] = partnum
clusterpartq['refnum'] = classnum
clusterpartq['clusterreference'] = clusterrefq
### finally we can insert parameters
if self.params['commit'] is True:
clusterpartq.insert()
sys.stderr.write("\n")
apDisplay.printMsg("Insertion complete in %s" %
(apDisplay.timeString(time.time() - inserttime)))
def insertKerDenSOM(self, binned=None):
### Preliminary data
projectid = apProject.getProjectIdFromAlignStackId(
self.params['alignstackid'])
alignstackdata = appiondata.ApAlignStackData.direct_query(
self.params['alignstackid'])
numclass = self.params['xdim'] * self.params['ydim']
pathdata = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
### KerDen SOM Params object
kerdenq = appiondata.ApKerDenSOMParamsData()
kerdenq['mask_diam'] = 2.0 * self.params['maskrad']
kerdenq['x_dimension'] = self.params['xdim']
kerdenq['y_dimension'] = self.params['ydim']
kerdenq['convergence'] = self.params['converge']
kerdenq['run_seconds'] = time.time() - self.t0
### Align Analysis Run object
analysisq = appiondata.ApAlignAnalysisRunData()
analysisq['runname'] = self.params['runname']
analysisq['path'] = pathdata
analysisq['description'] = self.params['description']
analysisq['alignstack'] = alignstackdata
analysisq['hidden'] = False
### linked through cluster not analysis
#analysisq['kerdenparams'] = kerdenq
### Clustering Run object
clusterrunq = appiondata.ApClusteringRunData()
clusterrunq['runname'] = self.params['runname']
clusterrunq['description'] = self.params['description']
# what if we binned the aligned stack to get the new one
if binned is None:
boxsize = alignstackdata['boxsize']
pixelsize = alignstackdata['pixelsize']
else:
boxsize = alignstackdata['boxsize'] / binned
pixelsize = alignstackdata['pixelsize'] * binned
clusterrunq['boxsize'] = boxsize
clusterrunq['pixelsize'] = pixelsize
clusterrunq['num_particles'] = self.params['numpart']
clusterrunq['alignstack'] = alignstackdata
clusterrunq['analysisrun'] = analysisq
clusterrunq['kerdenparams'] = kerdenq
### Clustering Stack object
clusterstackq = appiondata.ApClusteringStackData()
clusterstackq[
'avg_imagicfile'] = "kerdenstack" + self.timestamp + ".hed"
clusterstackq['num_classes'] = numclass
clusterstackq['clusterrun'] = clusterrunq
clusterstackq['path'] = pathdata
clusterstackq['hidden'] = False
imagicfile = os.path.join(self.params['rundir'],
clusterstackq['avg_imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find average stack file: " +
imagicfile)
### looping over clusters
apDisplay.printColor(
"Inserting particle classification data, please wait", "cyan")
for i in range(numclass):
classnum = i + 1
classroot = "%s.%d" % (self.timestamp, classnum - 1)
classdocfile = os.path.join(self.params['rundir'], classroot)
partlist = self.readClassDocFile(classdocfile)
### Clustering Particle object
clusterrefq = appiondata.ApClusteringReferenceData()
clusterrefq['refnum'] = classnum
clusterrefq['avg_mrcfile'] = classroot + ".mrc"
clusterrefq['clusterrun'] = clusterrunq
clusterrefq['path'] = pathdata
clusterrefq['num_particles'] = len(partlist)
clusterrefq['ssnr_resolution'] = self.cluster_resolution[i]
### looping over particles
sys.stderr.write(".")
for partnum in partlist:
alignpartdata = self.getAlignParticleData(
partnum, alignstackdata)
### Clustering Particle objects
clusterpartq = appiondata.ApClusteringParticleData()
clusterpartq['clusterstack'] = clusterstackq
clusterpartq['alignparticle'] = alignpartdata
clusterpartq['partnum'] = partnum
clusterpartq['refnum'] = classnum
clusterpartq['clusterreference'] = clusterrefq
### finally we can insert parameters
if self.params['commit'] is True:
clusterpartq.insert()
def insertAffinityPropagationRun(self, classes):
### Preliminary data
numclass = len(classes.keys())
projectid = apProject.getProjectIdFromAlignStackId(self.params['alignstackid'])
alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
pathdata = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
### Affinity Propagation Params object
affpropq = appiondata.ApAffinityPropagationClusterParamsData()
affpropq['mask_diam'] = 2.0*self.params['maskrad']
affpropq['run_seconds'] = time.time()-self.t0
affpropq['preference_type'] = self.params['preftype']
### Align Analysis Run object
analysisq = appiondata.ApAlignAnalysisRunData()
analysisq['runname'] = self.params['runname']
analysisq['path'] = pathdata
analysisq['description'] = self.params['description']
analysisq['alignstack'] = alignstackdata
analysisq['hidden'] = False
### linked through cluster not analysis
### Clustering Run object
clusterrunq = appiondata.ApClusteringRunData()
clusterrunq['runname'] = self.params['runname']
clusterrunq['description'] = self.params['description']
clusterrunq['boxsize'] = alignstackdata['boxsize']
clusterrunq['pixelsize'] = alignstackdata['pixelsize']
clusterrunq['num_particles'] = self.params['numpart']
clusterrunq['alignstack'] = alignstackdata
clusterrunq['analysisrun'] = analysisq
clusterrunq['affpropparams'] = affpropq
### Clustering Stack object
clusterstackq = appiondata.ApClusteringStackData()
clusterstackq['avg_imagicfile'] = "classaverage-"+self.timestamp+".hed"
clusterstackq['num_classes'] = numclass
clusterstackq['clusterrun'] = clusterrunq
clusterstackq['path'] = pathdata
clusterstackq['hidden'] = False
imagicfile = os.path.join(self.params['rundir'], clusterstackq['avg_imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find average stack file: "+imagicfile)
### looping over clusters
apDisplay.printColor("Inserting particle classification data, please wait", "cyan")
for i,classkey in enumerate(classes.keys()):
classnum = i+1
partlist = classes[classkey]
#print "MINIMUM: ", min(partlist)
classroot = "%s.%d"% (self.timestamp, classnum-1)
classdocfile = os.path.join(self.params['rundir'], classroot)
### Clustering Particle object
clusterrefq = appiondata.ApClusteringReferenceData()
clusterrefq['refnum'] = classnum
clusterrefq['clusterrun'] = clusterrunq
clusterrefq['path'] = pathdata
clusterrefq['num_particles'] = len(partlist)
#clusterrefq['ssnr_resolution'] = self.cluster_resolution[i]
### looping over particles
sys.stderr.write(".")
for partnum in partlist:
alignpartdata = self.getAlignParticleData(partnum, alignstackdata)
### Clustering Particle objects
clusterpartq = appiondata.ApClusteringParticleData()
clusterpartq['clusterstack'] = clusterstackq
clusterpartq['alignparticle'] = alignpartdata
clusterpartq['partnum'] = partnum
clusterpartq['refnum'] = classnum
clusterpartq['clusterreference'] = clusterrefq
### finally we can insert parameters
if self.params['commit'] is True:
clusterpartq.insert()
return
def insertRefinementRun(self):
runq = appiondata.ApRefineRunData()
#first two must be unique
runq['runname'] = self.params['runname']
runq['stack'] = self.params['stack']
#Recon upload can be continued
earlyresult = runq.query(results=1)
if earlyresult:
apDisplay.printWarning(
"Run already exists in the database.\nIdentical data will not be reinserted"
)
# empty <> than None for Sinedon query
paramdescription = self.params['description']
if not paramdescription:
paramdescription = None
runq['job'] = self.params['jobinfo']
runq['initialModel'] = self.params['model']
runq['package'] = self.params['package']
runq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
runq['description'] = paramdescription
runq['initialModel'] = self.params['model']
runq['num_iter'] = len(self.iterationdatas)
result = runq.query(results=1)
if earlyresult and not result:
if self.params['commit'] is True:
apDisplay.printError("Refinement Run parameters have changed")
else:
apDisplay.printWarning(
"Refinement Run parameters have changed")
# get stack apix
self.params['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stack'].dbid)
apDisplay.printMsg("inserting Refinement Run into database")
if self.params['commit'] is True:
runq.insert()
else:
apDisplay.printWarning("not committing results to database")
#if we insert runq then this returns no results !!!
# this is a workaround (annoying & bad)
runq = appiondata.ApRefineRunData()
runq['runname'] = self.params['runname']
runq['stack'] = self.params['stack']
runq['job'] = self.params['jobinfo']
runq['initialModel'] = self.params['model']
runq['package'] = self.params['package']
runq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
runq['description'] = paramdescription
runq['package'] = self.params['package']
runq['initialModel'] = self.params['model']
result = runq.query(results=1)
# save run entry in the parameters
if result:
self.params['refineRun'] = result[0]
elif self.params['commit'] is True:
apDisplay.printWarning(
"Refinement Run was not found, setting to inserted values")
self.params['refineRun'] = runq
else:
apDisplay.printWarning(
"Refinement Run was not found, setting to 'None'")
self.params['refineRun'] = None
return True
def insertRefBasedRun(self, partlist, imagicstack, insert=False):
apDisplay.printMsg("committing results to DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup ref based run
refrunq = appiondata.ApRefBasedRunData()
refrunq['runname'] = self.params['runname']
refrunq['xysearch'] = self.params['xysearch']
refrunq['xystep'] = self.params['xystep']
refrunq['first_ring'] = self.params['firstring']
refrunq['last_ring'] = self.params['lastring']
refrunq['num_iter'] = self.params['numiter']
refrunq['invert_templs'] = self.params['inverttemplates']
refrunq['num_templs'] = self.params['numtemplate']
#refrunq['csym', int),
refrunq['run_seconds'] = self.params['runtime']
### finish alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['refbasedrun'] = refrunq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['imagicfile'] = imagicstack
alignstackq['avgmrcfile'] = "average.mrc"
emancmd = "proc2d templatestack%02d.spi templatestack%02d.hed"%(self.params['numiter'],self.params['numiter'])
apEMAN.executeEmanCmd(emancmd)
alignstackq['refstackfile'] = ("templatestack%02d.hed"%(self.params['numiter']))
alignstackq['alignrun'] = alignrunq
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: "+imagicfile)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = self.stack['data']
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = self.stack['apix']*self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert()
### insert reference data
reflist = []
for j in range(self.params['numiter']):
iternum = j+1
for i in range(len(self.templatelist)):
refnum = i+1
templateid = self.templatelist[i]
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = iternum
refq['template'] = apTemplate.getTemplateFromId(templateid)
refq['mrcfile'] = ("templateavg%02d-%02d.mrc"%(iternum,refnum))
refpath = os.path.join(self.params['rundir'], "templates")
refq['path'] = appiondata.ApPathData(path=os.path.abspath(refpath))
refq['alignrun'] = alignrunq
if refnum in self.resdict:
refq['ssnr_resolution'] = self.resdict[refnum]
if insert is True:
refq.insert()
if iternum == self.params['numiter']:
reflist.append(refq)
#refq['varmrcfile', str),
#refq['frc_resolution', float),
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for partdict in partlist:
### see apSpider.alignment.alignStack() for more info
"""
partdict.keys()
'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,...
'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,...
'mirror': checkMirror(templatenum),
'score': float(data[3]),
'rot': float(data[4]),
'xshift': float(data[5]),
'yshift': float(data[6]),
"""
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = reflist[partdict['template']-1]
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['rot']
alignpartq['score'] = partdict['score']
alignpartq['mirror'] = partdict['mirror']
if insert is True:
alignpartq.insert()
return