def checkAffPropRun(self): # create a norefParam object clusterrunq = appiondata.ApClusteringRunData() clusterrunq['runname'] = self.params['runname'] clusterstackq = appiondata.ApClusteringStackData() clusterstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) clusterstackq['clusterrun'] = clusterrunq # ... path makes the run unique: uniquerun = clusterstackq.query(results=1) if uniquerun: apDisplay.printError("Run name '"+self.params['runname']+"' is already in the database")
def insertClusterStack(self, classavg=None, classvar=None, numclass=None, insert=False): clusterstackq = appiondata.ApClusteringStackData() clusterstackq['avg_imagicfile'] = classavg + ".hed" clusterstackq['var_imagicfile'] = classvar + ".hed" clusterstackq['num_classes'] = numclass clusterstackq['clusterrun'] = self.clusterrun clusterstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) clusterstackq['hidden'] = False imagicfile = os.path.join(self.params['rundir'], clusterstackq['avg_imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find average stack file: " + imagicfile) imagicfile = os.path.join(self.params['rundir'], clusterstackq['var_imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find variance stack file: " + imagicfile) apDisplay.printMsg("inserting clustering stack into database") if insert is True: clusterstackq.insert() ### particle class data apDisplay.printColor( "Inserting particle classification data, please wait", "cyan") for i in range(numclass): classnum = i + 1 classdocfile = os.path.join( self.params['rundir'], "cluster/classdoc_%s_%04d.spi" % (self.timestamp, classnum)) partlist = self.readClassDocFile(classdocfile) sys.stderr.write(".") for partnum in partlist: alignpartdata = self.getAlignParticleData(partnum) cpartq = appiondata.ApClusteringParticleData() cpartq['clusterstack'] = clusterstackq cpartq['alignparticle'] = alignpartdata cpartq['partnum'] = partnum cpartq['refnum'] = classnum cpartq['clusterreference'] = None # actual parameters if insert is True: cpartq.insert() return
def insertClusterStackIntoDatabase(self, clusterstackfile, classnum, partlist, num_classes): clusterstackq = appiondata.ApClusteringStackData() clusterstackq['avg_imagicfile'] = clusterstackfile clusterstackq['var_imagicfile'] = None clusterstackq['num_classes'] = num_classes clusterstackq['clusterrun'] = self.clusterrun clusterstackq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) clusterstackq['hidden'] = False if not os.path.isfile(clusterstackfile): apDisplay.printError("could not find average stack file: " + clusterstackfile) apDisplay.printMsg("inserting clustering stack into database") if self.params['commit'] is True: clusterstackq.insert() ### insert particle class & reference data clusterrefq = appiondata.ApClusteringReferenceData() clusterrefq['refnum'] = classnum clusterrefq['clusterrun'] = self.clusterrun clusterrefq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) clusterrefq['num_particles'] = len(partlist) if self.params['align'] is True: if (classnum - 1) in self.resdict: clusterrefq['ssnr_resolution'] = self.resdict[classnum - 1] apDisplay.printColor( "Inserting particle classification data, please wait", "cyan") for i, partnum in enumerate(partlist): cpartq = appiondata.ApClusteringParticleData() if self.runparams['align'] is True and self.params[ 'commit'] is True: alignpartdata = self.getAlignParticleData(int(partnum)) cpartq['alignparticle'] = alignpartdata else: cpartq['alignparticle'] = None cpartq['clusterstack'] = clusterstackq cpartq['partnum'] = int(partnum) cpartq['refnum'] = classnum cpartq['clusterreference'] = clusterrefq # actual parameters if self.params['commit'] is True: cpartq.insert() return
def findSubStackConditionData(stackdata): substackname = stackdata['substackname'] if not substackname: return None,None typedict = { 'alignsub':appiondata.ApAlignStackData(), 'clustersub':appiondata.ApClusteringStackData(), } substacktype = None for type in typedict.keys(): if substackname.find(type) >= 0: substacktype = type break if substacktype is None: return None,None conditionids = re.findall('[0-9]+',substackname) q = typedict[substacktype] return substacktype,q.direct_query(conditionids[-1])
def insertAffinityPropagationRun(self, classes): ### Preliminary data numclass = len(classes.keys()) projectid = apProject.getProjectIdFromAlignStackId(self.params['alignstackid']) alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid']) pathdata = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) ### Affinity Propagation Params object affpropq = appiondata.ApAffinityPropagationClusterParamsData() affpropq['mask_diam'] = 2.0*self.params['maskrad'] affpropq['run_seconds'] = time.time()-self.t0 affpropq['preference_type'] = self.params['preftype'] ### Align Analysis Run object analysisq = appiondata.ApAlignAnalysisRunData() analysisq['runname'] = self.params['runname'] analysisq['path'] = pathdata analysisq['description'] = self.params['description'] analysisq['alignstack'] = alignstackdata analysisq['hidden'] = False ### linked through cluster not analysis ### Clustering Run object clusterrunq = appiondata.ApClusteringRunData() clusterrunq['runname'] = self.params['runname'] clusterrunq['description'] = self.params['description'] clusterrunq['boxsize'] = alignstackdata['boxsize'] clusterrunq['pixelsize'] = alignstackdata['pixelsize'] clusterrunq['num_particles'] = self.params['numpart'] clusterrunq['alignstack'] = alignstackdata clusterrunq['analysisrun'] = analysisq clusterrunq['affpropparams'] = affpropq ### Clustering Stack object clusterstackq = appiondata.ApClusteringStackData() clusterstackq['avg_imagicfile'] = "classaverage-"+self.timestamp+".hed" clusterstackq['num_classes'] = numclass clusterstackq['clusterrun'] = clusterrunq clusterstackq['path'] = pathdata clusterstackq['hidden'] = False imagicfile = os.path.join(self.params['rundir'], clusterstackq['avg_imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find average stack file: "+imagicfile) ### looping over clusters apDisplay.printColor("Inserting particle classification data, please wait", "cyan") for i,classkey in enumerate(classes.keys()): classnum = i+1 partlist = classes[classkey] #print "MINIMUM: ", min(partlist) classroot = "%s.%d"% (self.timestamp, classnum-1) classdocfile = os.path.join(self.params['rundir'], classroot) ### Clustering Particle object clusterrefq = appiondata.ApClusteringReferenceData() clusterrefq['refnum'] = classnum clusterrefq['clusterrun'] = clusterrunq clusterrefq['path'] = pathdata clusterrefq['num_particles'] = len(partlist) #clusterrefq['ssnr_resolution'] = self.cluster_resolution[i] ### looping over particles sys.stderr.write(".") for partnum in partlist: alignpartdata = self.getAlignParticleData(partnum, alignstackdata) ### Clustering Particle objects clusterpartq = appiondata.ApClusteringParticleData() clusterpartq['clusterstack'] = clusterstackq clusterpartq['alignparticle'] = alignpartdata clusterpartq['partnum'] = partnum clusterpartq['refnum'] = classnum clusterpartq['clusterreference'] = clusterrefq ### finally we can insert parameters if self.params['commit'] is True: clusterpartq.insert() return
def insertKerDenSOM(self, binned=None): ### Preliminary data projectid = apProject.getProjectIdFromAlignStackId(self.params['alignstackid']) alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid']) numclass = self.params['xdim']*self.params['ydim'] pathdata = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) ### KerDen SOM Params object kerdenq = appiondata.ApKerDenSOMParamsData() kerdenq['mask_diam'] = 2.0*self.params['maskrad'] kerdenq['x_dimension'] = self.params['xdim'] kerdenq['y_dimension'] = self.params['ydim'] kerdenq['convergence'] = self.params['converge'] kerdenq['run_seconds'] = time.time()-self.t0 ### Align Analysis Run object analysisq = appiondata.ApAlignAnalysisRunData() analysisq['runname'] = self.params['runname'] analysisq['path'] = pathdata analysisq['description'] = self.params['description'] analysisq['alignstack'] = alignstackdata analysisq['hidden'] = False ### linked through cluster not analysis #analysisq['kerdenparams'] = kerdenq ### Clustering Run object clusterrunq = appiondata.ApClusteringRunData() clusterrunq['runname'] = self.params['runname'] clusterrunq['description'] = self.params['description'] # what if we binned the aligned stack to get the new one if binned is None: boxsize = alignstackdata['boxsize'] pixelsize = alignstackdata['pixelsize'] else: boxsize = alignstackdata['boxsize'] / binned pixelsize = alignstackdata['pixelsize'] * binned clusterrunq['boxsize'] = boxsize clusterrunq['pixelsize'] = pixelsize clusterrunq['num_particles'] = self.params['numpart'] clusterrunq['alignstack'] = alignstackdata clusterrunq['analysisrun'] = analysisq clusterrunq['kerdenparams'] = kerdenq ### Clustering Stack object clusterstackq = appiondata.ApClusteringStackData() clusterstackq['avg_imagicfile'] = "kerdenstack"+self.timestamp+".hed" clusterstackq['num_classes'] = numclass clusterstackq['clusterrun'] = clusterrunq clusterstackq['path'] = pathdata clusterstackq['hidden'] = False imagicfile = os.path.join(self.params['rundir'], clusterstackq['avg_imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find average stack file: "+imagicfile) ### looping over clusters apDisplay.printColor("Inserting particle classification data, please wait", "cyan") for i in range(numclass): classnum = i+1 classroot = "%s.%d"% (self.timestamp, classnum-1) classdocfile = os.path.join(self.params['rundir'], classroot) partlist = self.readClassDocFile(classdocfile) ### Clustering Particle object clusterrefq = appiondata.ApClusteringReferenceData() clusterrefq['refnum'] = classnum clusterrefq['avg_mrcfile'] = classroot+".mrc" clusterrefq['clusterrun'] = clusterrunq clusterrefq['path'] = pathdata clusterrefq['num_particles'] = len(partlist) clusterrefq['ssnr_resolution'] = self.cluster_resolution[i] ### looping over particles sys.stderr.write(".") for partnum in partlist: alignpartdata = self.getAlignParticleData(partnum, alignstackdata) ### Clustering Particle objects clusterpartq = appiondata.ApClusteringParticleData() clusterpartq['clusterstack'] = clusterstackq clusterpartq['alignparticle'] = alignpartdata clusterpartq['partnum'] = partnum clusterpartq['refnum'] = classnum clusterpartq['clusterreference'] = clusterrefq ### finally we can insert parameters if self.params['commit'] is True: clusterpartq.insert()
def insertRotKerDenSOM(self): inserttime = time.time() ### Preliminary data projectid = apProject.getProjectIdFromAlignStackId( self.params['alignstackid']) alignstackdata = appiondata.ApAlignStackData.direct_query( self.params['alignstackid']) numclass = self.params['xdim'] * self.params['ydim'] pathdata = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) ### rotKerDen SOM Params object rotkerdenson = appiondata.ApRotKerDenSOMParamsData() #rotkerdenson['mask_diam'] = 2.0*self.params['maskrad'] rotkerdenson['x_dimension'] = self.params['xdim'] rotkerdenson['y_dimension'] = self.params['ydim'] #rotkerdenson['convergence'] = self.params['converge'] rotkerdenson['run_seconds'] = time.time() - self.t0 rotkerdenson['initregulfact'] = self.params['initregulfact'] rotkerdenson['finalregulfact'] = self.params['finalregulfact'] rotkerdenson['incrementregulfact'] = self.params['incrementregulfact'] rotkerdenson['spectrainnerradius'] = self.params['spectrainnerradius'] rotkerdenson['spectraouterradius'] = self.params['spectraouterradius'] rotkerdenson['spectralowharmonic'] = self.params['spectralowharmonic'] rotkerdenson['spectrahighharmonic'] = self.params[ 'spectrahighharmonic'] ### Analysis Run object analysisq = appiondata.ApAlignAnalysisRunData() analysisq['runname'] = self.params['runname'] analysisq['path'] = pathdata analysisq['description'] = self.params['description'] analysisq['alignstack'] = alignstackdata analysisq['hidden'] = False ### Clustering Run object clusterrunq = appiondata.ApClusteringRunData() clusterrunq['runname'] = self.params['runname'] clusterrunq['description'] = self.params['description'] clusterrunq['boxsize'] = alignstackdata['boxsize'] clusterrunq['pixelsize'] = alignstackdata['pixelsize'] clusterrunq['num_particles'] = self.params['numpart'] clusterrunq['alignstack'] = alignstackdata clusterrunq['analysisrun'] = analysisq clusterrunq['rotkerdenparams'] = rotkerdenson ### Clustering Stack object #Stack with cluster averages?????? template = os.path.join(self.params['rundir'], self.spectraTemporalFilesMask + ".png") files = glob.glob(template) imglist = [] for listname in files: a = apImage.readPNG(listname) imglist.append(a) apImagicFile.writeImagic(imglist, "rotkerdenstack" + self.timestamp + ".hed") clusterstackq = appiondata.ApClusteringStackData() clusterstackq[ 'avg_imagicfile'] = "rotkerdenstack" + self.timestamp + ".hed" clusterstackq['num_classes'] = numclass clusterstackq['clusterrun'] = clusterrunq clusterstackq['path'] = pathdata clusterstackq['hidden'] = False imagicfile = os.path.join(self.params['rundir'], clusterstackq['avg_imagicfile']) if not os.path.isfile(imagicfile): apDisplay.printError("could not find average stack file: " + imagicfile) ### looping over clusters apDisplay.printColor( "Inserting particle classification data, please wait", "cyan") numclass = self.params['xdim'] * self.params['ydim'] for i in range(numclass): classnum = i + 1 classroot = "%s.%d" % (self.timestamp, classnum - 1) classdocfile = os.path.join(self.params['rundir'], classroot) partlist = self.readClassDocFile(classdocfile) ### Clustering Particle object # MRC image for each code node but plot or image clusterrefq = appiondata.ApClusteringReferenceData() clusterrefq['refnum'] = classnum clusterrefq['avg_mrcfile'] = classroot + ".mrc" clusterrefq['clusterrun'] = clusterrunq clusterrefq['path'] = pathdata clusterrefq['num_particles'] = len(partlist) ### looping over particles #which particles belong to which code node sys.stderr.write(".") for partnum in partlist: alignpartdata = self.getAlignParticleData( partnum, alignstackdata) ### Clustering Particle objects clusterpartq = appiondata.ApClusteringParticleData() clusterpartq['clusterstack'] = clusterstackq clusterpartq['alignparticle'] = alignpartdata clusterpartq['partnum'] = partnum clusterpartq['refnum'] = classnum clusterpartq['clusterreference'] = clusterrefq ### finally we can insert parameters if self.params['commit'] is True: clusterpartq.insert() sys.stderr.write("\n") apDisplay.printMsg("Insertion complete in %s" % (apDisplay.timeString(time.time() - inserttime)))
def insertClusterStack(self, numclusters): pathdata = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) ### clusterStack object clusterstackq = appiondata.ApClusteringStackData() clusterstackq['num_classes'] = numclusters clusterstackq['avg_imagicfile'] = self.params['classumfile'] clusterstackq['clusterrun'] = self.clusterrun clusterstackq['ignore_images'] = self.params['ignore_images'] clusterstackq['ignore_members'] = self.params['ignore_members'] clusterstackq['num_factors'] = self.params['num_eigenimages'] clusterstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir'])) clusterstackq['hidden'] = False ### first insertion into database, if commit is checked apDisplay.printMsg("inserting clustering stack into database") if self.params['commit'] is True: clusterstackq.insert() else: apDisplay.printWarning("not committing results to DB") ### inserting particles into database if self.params['commit'] is True: apDisplay.printColor("Inserting particle classification data, please wait", "cyan") ### read .cls file that contains information regarding particle classification clsfile = os.path.join(self.params['rundir'], self.params['classfile']) cls_particle_data, classqualities, particles_in_class = self.readClassesFile(clsfile, numclusters) for i in range(numclusters): ### insert the particles cls_num = i + 1 cls_quality = classqualities[i] num_particles = particles_in_class[i] particles = cls_particle_data[i] # ### calculate SSNR resolution for class # partlist = "" # stackfile = self.params['classumfile'] # self.calcResolution(partlist, stackfile, self.params['apix']) ### Clustering Particle object clusterrefq = appiondata.ApClusteringReferenceData() clusterrefq['refnum'] = cls_num clusterrefq['clusterrun'] = self.clusterrun clusterrefq['path'] = pathdata clusterrefq['num_particles'] = num_particles for partnum in particles: alignpartdata = self.getAlignParticleData(partnum) cpartq = appiondata.ApClusteringParticleData() cpartq['clusterstack'] = clusterstackq cpartq['alignparticle'] = alignpartdata cpartq['refnum'] = cls_num cpartq['partnum'] = partnum cpartq['clusterreference'] = clusterrefq cpartq['imagic_cls_quality'] = cls_quality if self.params['commit'] is True: cpartq.insert() lastnum = partnum return