def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['start'] is None: apDisplay.printError("a number of starting classes was not provided") if self.params['end'] is None: apDisplay.printError("a number of ending classes was not provided") if self.params['runname'] is None: apDisplay.printError("run name was not defined") self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) # check for virtual stack self.params['virtualdata'] = None if not os.path.isfile(stackfile): vstackdata = apStack.getVirtualStackParticlesFromId(self.params['stackid']) npart = len(vstackdata['particles']) self.params['virtualdata'] = vstackdata else: npart = apFile.numImagesInStack(stackfile) if self.params['numpart'] is None: self.params['numpart'] = npart elif self.params['numpart'] > npart: apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) self.boxsize = apStack.getStackBoxsize(self.params['stackid']) self.workingboxsize = math.floor(self.boxsize/self.params['bin']) if not self.params['mask']: self.params['mask'] = (self.boxsize/2)-2 self.workingmask = math.floor(self.params['mask']/self.params['bin']) if self.params['mramethod'] == 'imagic': self.imagicroot = apIMAGIC.checkImagicExecutablePath() self.imagicversion = apIMAGIC.getImagicVersion(self.imagicroot)
def checksizes(self): newnots = apFile.numImagesInStack(self.params['notstack']) newtilts = apFile.numImagesInStack(self.params['tiltstack']) stackdata1 = apStack.getOnlyStackData(self.params['stack1'], msg=False) stackpath1 = os.path.join(stackdata1['path']['path'], stackdata1['name']) oldnots = apFile.numImagesInStack(stackpath1) stackdata2 = apStack.getOnlyStackData(self.params['stack2'], msg=False) stackpath2 = os.path.join(stackdata2['path']['path'], stackdata2['name']) oldtilts = apFile.numImagesInStack(stackpath2) if newtilts != oldtilts: apDisplay.printWarning( "tilted stack are different sizes: %d vs. %d part" % (newtilts, oldtilts)) else: apDisplay.printMsg("tilted stack are the sames sizes: %d part" % (newtilts)) if newnots != oldnots: apDisplay.printWarning( "untilted stack are different sizes: %d vs. %d part" % (newnots, oldnots)) else: apDisplay.printMsg("untilted stack are the sames sizes: %d part" % (newnots))
def makeNewStack(oldstack, newstack, listfile=None, remove=False, bad=False): if not os.path.isfile(oldstack): apDisplay.printWarning("could not find old stack: "+oldstack) if os.path.isfile(newstack): if remove is True: apDisplay.printWarning("removing old stack: "+newstack) time.sleep(2) apFile.removeStack(newstack) else: apDisplay.printError("new stack already exists: "+newstack) apDisplay.printMsg("creating a new stack\n\t"+newstack+ "\nfrom the oldstack\n\t"+oldstack+"\nusing list file\n\t"+str(listfile)) emancmd = "proc2d "+oldstack+" "+newstack if listfile is not None: emancmd += " list="+listfile apEMAN.executeEmanCmd(emancmd, verbose=True) if bad is True and listfile is not None: ### run only if num bad particles < num good particles newstacknumpart = apFile.numImagesInStack(newstack) oldstacknumpart = apFile.numImagesInStack(oldstack) if newstacknumpart > oldstacknumpart/2: ### create bad.hed stack with all bad particles badstack = os.path.join(os.path.dirname(newstack), "bad.hed") emancmd = "proc2d %s %s exclude=%s"%(oldstack, badstack, listfile) apEMAN.executeEmanCmd(emancmd, verbose=True) else: apDisplay.printMsg("Rejecting more particles than keeping, not creating a bad stack") return
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['description'] is None: apDisplay.printError("run description was not defined") if self.params['templatelist'] is None: apDisplay.printError("template list was not provided") if self.params['lastring'] is None: apDisplay.printError("a last ring radius was not provided") if self.params['runname'] is None: apDisplay.printError("run name was not defined") stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid'])/self.params['bin'] if self.params['lastring'] > boxsize/2-2: apDisplay.printError("last ring radius is too big for boxsize " +str(self.params['lastring'])+" > "+str(boxsize/2-2)) if self.params['lastring']+self.params['xysearch'] > boxsize/2-2: apDisplay.printError("last ring plus xysearch radius is too big for boxsize " +str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2)) ### convert / check template data self.templatelist = self.params['templatelist'].strip().split(",") if not self.templatelist or type(self.templatelist) != type([]): apDisplay.printError("could not parse template list="+self.params['templatelist']) self.params['numtemplate'] = len(self.templatelist) apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates")
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['numrefs'] is None: apDisplay.printError("a number of classes was not provided") maxparticles = 500000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid']) self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" %(self.params['clipsize'],self.clipsize)) self.clipsize = self.params['clipsize'] if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) self.mpirun = self.checkMPI() if self.mpirun is None: apDisplay.printError("There is no MPI installed") if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() if self.params['nproc'] < 2: apDisplay.printError("Only the MPI version of CL2D is currently supported, must run with > 1 CPU")
def checkConflicts(self): if self.params["stackid"] is None: apDisplay.printError("stack id was not defined") if self.params["description"] is None: apDisplay.printError("run description was not defined") if self.params["runname"] is None: apDisplay.printError("run name was not defined") maxparticles = 150000 if self.params["numpart"] > maxparticles: apDisplay.printError( "too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params["numpart"]) ) if self.params["initmethod"] not in self.initmethods: apDisplay.printError( "unknown initialization method defined: " + str(self.params["initmethod"]) + " not in " + str(self.initmethods) ) stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False) stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"]) if self.params["numpart"] > apFile.numImagesInStack(stackfile): apDisplay.printError( "trying to use more particles " + str(self.params["numpart"]) + " than available " + str(apFile.numImagesInStack(stackfile)) )
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") #if self.params['description'] is None: # apDisplay.printError("run description was not defined") if self.params['numrefs'] is None: apDisplay.printError("a number of classes was not provided") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if not self.params['fastmode'] in self.fastmodes: apDisplay.printError("fast mode must be on of: "+str(self.fastmodes)) if not self.params['converge'] in self.convergemodes: apDisplay.printError("converge mode must be on of: "+str(self.convergemodes)) maxparticles = 500000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid']) self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" %(self.params['clipsize'],self.clipsize)) self.clipsize = self.params['clipsize'] if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile)
def appendParticleListToStackFile(partlist, mergestackfile, msg=True): """ takes a list of 2D numpy arrays and add to stack file due to hack, we must re-number the stack later """ ### initialization t0 = time.time() root = os.path.splitext(mergestackfile)[0] mergeheaderfile = root + ".hed" mergedatafile = root + ".img" ### merge data files premergesize = apFile.fileSize(mergedatafile) mergedata = file(mergedatafile, 'ab') for partarray in partlist: part32bit = numpy.asarray(partarray, dtype=numpy.float32) mergedata.write(part32bit.tostring()) mergedata.close() finalsize = apFile.fileSize(mergedatafile) addsize = len(part32bit.tostring() * len(partlist)) if finalsize != addsize + premergesize: apDisplay.printError( "size mismatch %s vs. %s + %s = %s" % (apDisplay.bytes(finalsize), apDisplay.bytes(addsize), apDisplay.bytes(premergesize), apDisplay.bytes(premergesize + addsize))) elif msg is True: apDisplay.printMsg( "size match %s vs. %s + %s = %s" % (apDisplay.bytes(finalsize), apDisplay.bytes(addsize), apDisplay.bytes(premergesize), apDisplay.bytes(premergesize + addsize))) ### merge header files premergenumpart = apFile.numImagesInStack(mergeheaderfile) mergehead = open(mergeheaderfile, 'ab') count = 0 for partarray in partlist: count += 1 headerstr = makeHeaderStrFromArray(premergenumpart + count, partarray) mergehead.write(headerstr) mergehead.close() numberStackFile(mergeheaderfile, msg=msg) finalnumpart = apFile.numImagesInStack(mergeheaderfile) addpart = len(partlist) if finalnumpart != addpart + premergenumpart: apDisplay.printError("size mismatch %d vs. %d + %d = %d" % (finalnumpart, addpart, premergenumpart, addpart + premergenumpart)) elif msg is True: apDisplay.printMsg("size match %d vs. %d + %d = %d" % (finalnumpart, addpart, premergenumpart, addpart + premergenumpart)) return True
def checkConflicts(self): apDisplay.printMsg("/bin/rm -f *.hed *.img *.doc *.vol *.mrc *.spi *.log *~ *.pickle *.hist *.proj *.xmp *.sel *.basis *.original; rm -fr volume*") ### check for missing and duplicate entries #if self.params['alignid'] is None and self.params['clusterid'] is None: # apDisplay.printError("Please provide either --cluster-id or --align-id") if self.params['alignid'] is not None and self.params['clusterid'] is not None: apDisplay.printError("Please provide only one of either --cluster-id or --align-id") if self.params['modelstr'] is None and self.params['nvol'] is None: apDisplay.printError("Please provide model numbers or number of volumes") elif self.params['modelstr'] is not None: modellist = self.params['modelstr'].split(",") self.params['modelids'] = [] for modelid in modellist: self.params['modelids'].append(int(modelid)) ### get the stack ID from the other IDs if self.params['alignid'] is not None: self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignid']) self.params['stackid'] = self.alignstackdata['stack'].dbid elif self.params['clusterid'] is not None: self.clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid']) self.alignstackdata = self.clusterstackdata['clusterrun']['alignstack'] self.params['stackid'] = self.alignstackdata['stack'].dbid if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") #if self.params['description'] is None: # apDisplay.printError("run description was not defined") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if not self.params['fastmode'] in self.fastmodes: apDisplay.printError("fast mode must be on of: "+str(self.fastmodes)) maxparticles = 150000 minparticles = 50 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) if self.params['numpart'] < minparticles: apDisplay.printError("not enough particles requested, min: " + str(minparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) ### find number of processors if self.params['nproc'] is None: self.nproc = apParam.getNumProcessors() else: self.nproc = self.params['nproc'] self.mpirun = self.checkMPI()
def checkConflicts(self): if self.params["stackid"] is None: apDisplay.printError("Stack id was not defined") if self.params["mandir"] is None: apDisplay.printError("Directory containing mandatory files was not defined") if self.params["description"] is None: apDisplay.printError("Run description was not defined") stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False) stackfile = os.path.join(stackdata["path"]["path"], stackdata["name"]) if self.params["numpart"] is None: self.params["numpart"] = apFile.numImagesInStack(stackfile) apDisplay.printWarning("Number of particles not defined, processing full stack") if self.params["numpart"] > apFile.numImagesInStack(stackfile): apDisplay.printError( "trying to use more particles " + str(self.params["numpart"]) + " than available " + str(apFile.numImagesInStack(stackfile)) ) if self.params["replen"] is None: apDisplay.printError("Helical repeat was not defined") if self.params["subunits"] is None: apDisplay.printError("Number of subunits was not defined") if self.params["diameter"] is None: apDisplay.printError("Filament diameter was not defined") if self.params["diaminner"] is None: apDisplay.printError("Inner filament diameter was not defined") if self.params["rescut"] is None: apDisplay.printError("Phase residual cutoff was not defined") if self.params["filval"] is None: apDisplay.printError("Filter value was not defined") if self.params["bin"] is None: self.params["bin"] = 1 apDisplay.printWarning("Binning factor was not defined, binning by one") if self.params["cont"] is None: apDisplay.printError("Must specify if a contrast change is needed") if self.params["prehip"] is None: apDisplay.printError("Must specify if prehip needs to be executed or not") step = apStack.getStackPixelSizeFromStackId(self.params["stackid"]) self.params["step"] = step onerep = math.floor(self.params["replen"] / step) calcyht2 = math.floor(self.params["diameter"] / step) boxsize = apStack.getStackBoxsize(self.params["stackid"]) self.params["boxsize"] = boxsize if self.params["xlngth"] < onerep: apDisplay.printError("Filament segment length can not be less than one helical repeat") if self.params["xlngth"] > boxsize: apDisplay.printError("Filament segment length can not be greater than stack box size") if self.params["yht2"] < calcyht2: apDisplay.printError("Filament box height can not be less than filament diameter") if self.params["padval"] < self.params["xlngth"]: apDisplay.printError("Pad value can not be less than the filament segment length") if not self.checkval(self.params["yht2"]): apDisplay.printError("Filament box height must be a power of two") if not self.checkval(self.params["padval"]): apDisplay.printError("Pad value must be a power of two")
def checkConflicts(self): ### setup correct database after we have read the project id if 'projectid' in self.params and self.params['projectid'] is not None: apDisplay.printMsg("Using split database") # use a project database newdbname = apProject.getAppionDBFromProjectId(self.params['projectid']) sinedon.setConfig('appiondata', db=newdbname) apDisplay.printColor("Connected to database: '"+newdbname+"'", "green") ### get stack data self.stack = {} if self.params['stackid'] is not None: self.stack['data'] = apStack.getOnlyStackData(self.params['stackid']) self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid']) self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name']) else: self.stack['data'] = appiondata.ApAlignStackData.direct_query(self.params['alignstackid']) self.stack['apix'] = self.stack['pixelsize'] self.stack['boxsize'] = self.stack['boxsize'] self.stack['file'] = self.stack['imagicfile'] ### check conflicts if self.params['stackid'] is None and self.params['alignstackid'] is None: apDisplay.printError("stack id OR alignstack id was not defined") if self.params['stackid'] is not None and self.params['alignstackid'] is not None: apDisplay.printError("either specify stack id OR alignstack id, not both") if self.params['generations'] is None: apDisplay.printError("number of generations was not provided") maxparticles = 500000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) if self.params['numpart'] > apFile.numImagesInStack(self.stack['file']): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(self.stack['file']))) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(self.stack['file']) boxsize = apStack.getStackBoxsize(self.params['stackid']) if self.params['ou'] is None: self.params['ou'] = (boxsize / 2.0) - 2 self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" %(self.params['clipsize'],self.clipsize)) self.clipsize = self.params['clipsize'] self.mpirun = self.checkMPI() if self.mpirun is None: apDisplay.printError("There is no MPI installed")
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("Stack id was not defined") if self.params['mandir'] is None: apDisplay.printError("Directory containing mandatory files was not defined") if self.params['description'] is None: apDisplay.printError("Run description was not defined") stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) apDisplay.printWarning("Number of particles not defined, processing full stack") if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart'])+" than available "+str(apFile.numImagesInStack(stackfile))) if self.params['replen'] is None: apDisplay.printError("Helical repeat was not defined") if self.params['subunits'] is None: apDisplay.printError("Number of subunits was not defined") if self.params['diameter'] is None: apDisplay.printError("Filament diameter was not defined") if self.params['diaminner'] is None: apDisplay.printError("Inner filament diameter was not defined") if self.params['rescut'] is None: apDisplay.printError("Phase residual cutoff was not defined") if self.params['filval'] is None: apDisplay.printError("Filter value was not defined") if self.params['bin'] is None: self.params['bin'] = 1 apDisplay.printWarning("Binning factor was not defined, binning by one") if self.params['cont'] is None: apDisplay.printError("Must specify if a contrast change is needed") if self.params['prehip'] is None: apDisplay.printError("Must specify if prehip needs to be executed or not") step = apStack.getStackPixelSizeFromStackId(self.params['stackid']) self.params['step'] = step onerep = math.floor(self.params['replen'] / step) calcyht2 = math.floor(self.params['diameter'] / step) boxsize = apStack.getStackBoxsize(self.params['stackid']) self.params['boxsize'] = boxsize if self.params['xlngth'] < onerep: apDisplay.printError("Filament segment length can not be less than one helical repeat") if self.params['xlngth'] > boxsize: apDisplay.printError("Filament segment length can not be greater than stack box size") if self.params['yht2'] < calcyht2: apDisplay.printError("Filament box height can not be less than filament diameter") if self.params['padval'] < self.params['xlngth']: apDisplay.printError("Pad value can not be less than the filament segment length") if not self.checkval(self.params['yht2']): apDisplay.printError("Filament box height must be a power of two") if not self.checkval(self.params['padval']): apDisplay.printError("Pad value must be a power of two")
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['numrefs'] is None: apDisplay.printError("a number of classes was not provided") maxparticles = 500000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) # check for virtual stack self.params['virtualdata'] = None if not os.path.isfile(stackfile): vstackdata = apStack.getVirtualStackParticlesFromId( self.params['stackid']) npart = len(vstackdata['particles']) self.params['virtualdata'] = vstackdata else: npart = apFile.numImagesInStack(stackfile) if self.params['numpart'] > npart: apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid']) self.clipsize = int(math.floor( boxsize / float(self.params['bin'] * 2))) * 2 if self.params['clipsize'] is not None: if self.params['clipsize'] > self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" % (self.params['clipsize'], self.clipsize)) self.clipsize = self.params['clipsize'] if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) self.mpirun = self.checkMPI() if self.mpirun is None: apDisplay.printError("There is no MPI installed") if self.params['nproc'] is None: self.params['nproc'] = self.params['nodes'] * self.params['ppn'] if self.params['nproc'] < 2: apDisplay.printError( "Only the MPI version of CL2D is currently supported, must run with > 1 CPU" )
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['description'] is None: apDisplay.printError("run description was not defined") ### convert / check template data if self.params['templatelist'] is None: apDisplay.printError("template list was not provided") self.templatelist = self.params['templatelist'].strip().split(",") if not self.templatelist or type(self.templatelist) != type([]): apDisplay.printError("could not parse template list=" + self.params['templatelist']) self.params['numtemplate'] = len(self.templatelist) apDisplay.printMsg("Found " + str(self.params['numtemplate']) + " templates") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if not self.params['fastmode'] in self.fastmodes: apDisplay.printError("fast mode must be on of: " + str(self.fastmodes)) if not self.params['converge'] in self.convergemodes: apDisplay.printError("converge mode must be on of: " + str(self.convergemodes)) maxparticles = 150000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid']) self.clipsize = int(math.floor( boxsize / float(self.params['bin'] * 2))) * 2 if self.params['clipsize'] is not None: if self.params['clipsize'] >= self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" % (self.params['clipsize'], self.clipsize)) self.clipsize = self.params['clipsize'] if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile)
def initValues(self, stackfile, numrequest=None): ### check for stack if not os.path.isfile(stackfile): apDisplay.printError("stackfile does not exist: " + stackfile) ### amount of free memory on machine (converted to bytes) self.freememory = mem.free() * 1024 self.message("Free memory: %s" % (apDisplay.bytes(self.freememory))) ### box size of particle self.boxsize = apFile.getBoxSize(stackfile)[0] self.message("Box size: %d" % (self.boxsize)) ### amount of memory used per particles (4 bytes per pixel) self.memperpart = self.boxsize**2 * 4.0 self.message("Memory used per part: %s" % (apDisplay.bytes(self.memperpart))) ### maximum number particles that fit into memory self.maxpartinmem = self.freememory / self.memperpart self.message("Max particles in memory: %d" % (self.maxpartinmem)) ### number particles to fit into memory self.partallowed = int(self.maxpartinmem / 20.0) self.message("Particles allowed in memory: %d" % (self.partallowed)) ### number particles in stack numpart = apFile.numImagesInStack(stackfile) if self.numpart is None or self.numpart > numpart: self.numpart = numpart if numrequest is not None and self.numpart > numrequest: self.numpart = numrequest self.message("Number of particles in stack: %d" % (self.numpart)) if self.numpart > self.partallowed: numchucks = math.ceil(self.numpart / float(self.partallowed)) self.stepsize = int(self.numpart / numchucks) else: numchucks = 1 self.stepsize = self.numpart self.message("Particle loop num chunks: %d" % (numchucks)) self.message("Particle loop step size: %d" % (self.stepsize))
def convertSpiderStack(self, spiderstack): """ takes the stack file and creates a spider file ready for processing """ emancmd = "proc2d " if not os.path.isfile(spiderstack): apDisplay.printError("stackfile does not exist: " + spiderstack) emancmd += spiderstack + " " imagicstack = os.path.join(self.params['rundir'], "alignstack.hed") apFile.removeFile(imagicstack, warn=True) emancmd += imagicstack + " " starttime = time.time() apDisplay.printColor( "Running spider stack conversion this can take a while", "cyan") apEMAN.executeEmanCmd(emancmd, verbose=True) apDisplay.printColor( "finished eman in " + apDisplay.timeString(time.time() - starttime), "cyan") numpart = apFile.numImagesInStack(imagicstack) if numpart != self.params['numpart']: apDisplay.printError( "The number of particles returned is different from the number requested" ) return imagicstack
def getClassification(self, classdoc, classname): D = {} tmp = {} fh = open(classdoc) nclasses = apFile.numImagesInStack(classname) ### all classes, assuming that some of them were discarded by the minp parameter for i in range(self.params['ncls']): tmp[(i + 1)] = [] for line in fh: params = line.strip().split() particleNumber = int(params[0]) classNumber = int(float(params[2])) tmp[classNumber].append(particleNumber) fh.close() ### rename the dictionary i = 1 for oldkey, val in tmp.iteritems(): ### ascending order if len(val) > 0: D[i] = val # print oldkey, i, val i += 1 return D
def start(self): ### copy stack to rundir newstack = os.path.join(self.params['rundir'], "start.hed") if os.path.isfile(newstack): apDisplay.printError("Stack already exists") emancmd = "proc2d %s %s" % (self.params['stackfile'], newstack) if self.params['normalize'] is True: emancmd += " edgenorm" apEMAN.executeEmanCmd(emancmd) ### set final parameters boxsize = apFile.getBoxSize(newstack) apDisplay.printMsg("Boxsize: %i" % boxsize[0]) if not boxsize or boxsize <= 0: apDisplay.printError("Could not determine stack size") else: self.boxsize = boxsize[0] self.numpart = apFile.numImagesInStack(newstack) apDisplay.printMsg("Num part: %i" % self.numpart) if not self.numpart or self.numpart <= 0: apDisplay.printError("Could not determine number of particles") if self.numpart <= 1: apDisplay.printError("Not enough particles to upload") apStack.averageStack(newstack) #self.params['commit'] = False self.createStackData()
def initValues(self, stackfile, numrequest=None): ### check for stack if not os.path.isfile(stackfile): apDisplay.printError("stackfile does not exist: "+stackfile) ### amount of free memory on machine (converted to bytes) self.freememory = mem.free()*1024 self.message("Free memory: %s"%(apDisplay.bytes(self.freememory))) ### box size of particle self.boxsize = apFile.getBoxSize(stackfile)[0] self.message("Box size: %d"%(self.boxsize)) ### amount of memory used per particles (4 bytes per pixel) self.memperpart = self.boxsize**2 * 4.0 self.message("Memory used per part: %s"%(apDisplay.bytes(self.memperpart))) ### maximum number particles that fit into memory self.maxpartinmem = self.freememory/self.memperpart self.message("Max particles in memory: %d"%(self.maxpartinmem)) ### number particles to fit into memory self.partallowed = int(self.maxpartinmem/20.0) self.message("Particles allowed in memory: %d"%(self.partallowed)) ### number particles in stack numpart = apFile.numImagesInStack(stackfile) if self.numpart is None or self.numpart > numpart: self.numpart = numpart if numrequest is not None and self.numpart > numrequest: self.numpart = numrequest self.message("Number of particles in stack: %d"%(self.numpart)) if self.numpart > self.partallowed: numchucks = math.ceil(self.numpart/float(self.partallowed)) self.stepsize = int(self.numpart/numchucks) else: numchucks = 1 self.stepsize = self.numpart self.message("Particle loop num chunks: %d"%(numchucks)) self.message("Particle loop step size: %d"%(self.stepsize))
def useClusterForTemplate(self): apDisplay.printMsg("Using clustering stack to make templates") clusterstackdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterid']) self.params['apix'] = clusterstackdata['clusterrun']['alignstack']['pixelsize'] stackfile = os.path.join(clusterstackdata['path']['path'], clusterstackdata['avg_imagicfile']) self.params['templatelist'] = [] stackid = clusterstackdata['clusterrun']['alignstack']['stack'].dbid sessiondata = apStack.getSessionDataFromStackId(stackid) self.params['sessionname'] = sessiondata['name'] ### check to see if stackimagenum is within the boundary of the stack numpart = apFile.numImagesInStack(stackfile) for i in self.params['imgnums'].split(","): partnum = int(i) if partnum > numpart or partnum < 0: apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart)) apDisplay.printMsg("Extracting image %d from cluster stack" % partnum) templatename = "cluster%d-average%d.mrc"%(self.params['clusterid'],partnum) abstemplatepath= os.path.join(clusterstackdata['path']['path'], templatename) ### run proc2d with params['stackimgnum'] emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum) ### create template apDisplay.printMsg("creating "+templatename) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'].append(abstemplatepath)
def getNumAlignedParticles(self): t0 = time.time() self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid']) oldalignedstack = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['imagicfile']) numpart = apFile.numImagesInStack(oldalignedstack) apDisplay.printMsg("numpart="+str(numpart)+" in "+apDisplay.timeString(time.time()-t0)) return numpart
def getClassification(self, classdoc, classname): D = {} tmp = {} fh = open(classdoc) nclasses = apFile.numImagesInStack(classname) ### all classes, assuming that some of them were discarded by the minp parameter for i in range(self.params['ncls']): tmp[(i+1)] = [] for line in fh: params = line.strip().split() particleNumber = int(params[0]) classNumber = int(float(params[2])) tmp[classNumber].append(particleNumber) fh.close() ### rename the dictionary i = 1 for oldkey, val in tmp.iteritems(): ### ascending order if len(val) > 0: D[i] = val # print oldkey, i, val i += 1 return D
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['rad'] is None: apDisplay.printError("no particle radius set") if self.params['modelid'] is None: apDisplay.printError("model id was not defined") if self.params['lastring'] is None: apDisplay.printError("a last ring radius was not provided") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if self.params['incr'] is None: apDisplay.printError("angular increments are not specified") stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) numstackp = apFile.numImagesInStack(stackfile) if self.params['numpart'] > numstackp: apDisplay.printError("trying to use more particles "+str(numstackp) +" than available "+str(numstackp)) elif self.params['numpart'] == 0: apDisplay.printWarning("using all "+str(numstackp)+" particles") self.params['numpart'] = numstackp boxsize = apStack.getStackBoxsize(self.params['stackid']) if self.params['lastring'] > boxsize/2-2: apDisplay.printError("last ring radius is too big for boxsize " +str(self.params['lastring'])+" > "+str(boxsize/2-2)) if self.params['lastring']+self.params['xysearch'] > boxsize/2-2: apDisplay.printError("last ring plus xysearch radius is too big for boxsize " +str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2)) if (self.params['xysearch'] % self.params['xystep']) > 0: apDisplay.printError("translational search (xy-search) must be divisible by search step (xy-step)")
def start(self): """ This is the core of your function. """ stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) original_stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) filtered_stackfile = os.path.join(self.params['rundir'], self.timestamp+".hed") apFile.removeStack(filtered_stackfile, warn=False) apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) boxsize = apStack.getStackBoxsize(self.params['stackid']) emancmd = "proc2d %s %s apix=%.3f "%(original_stackfile, filtered_stackfile, apix) if self.params['lowpass'] is not None: emancmd += " lp=%.3f "%(self.params['lowpass']) if self.params['highpass'] is not None: emancmd += " hp=%.3f "%(self.params['highpass']) if self.params['bin'] is not None and self.params['bin'] > 1: ## determine a multiple of the bin that is divisible by 2 and less than the boxsize clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2*self.params['bin'] emancmd += " shrink=%d clip=%d,%d "%(self.params['bin'], clipsize, clipsize) emancmd += " last=%d "%(self.params['numpart']-1) apEMAN.executeEmanCmd(emancmd, verbose=True, showcmd=True) ### confirm that it worked if self.params['numpart'] != apFile.numImagesInStack(filtered_stackfile): apDisplay.printError("Missing particles in stack") ### run the radon transform code self.radonAlign(filtered_stackfile) ### insert info into database self.commitToDatabase()
def fillSimilarityMatrix(self, alignedstack): ### Get initial correlation values ### this is really, really slow similarfile = "similarities.dat" simstack = similarityStack() simstack.similarfile = similarfile simstack.start(alignedstack) if not os.path.isfile(similarfile): apDisplay.printError("Failed to create similarity file") simf = open(similarfile, 'r') simlist = [] count = 0 for line in simf: count += 1 sline = line.strip() slist = sline.split() ccval = float(slist[2]) simlist.append(ccval) simf.close() apDisplay.printMsg("There are %d lines in the sim file: %s"%(count, similarfile)) numpart = apFile.numImagesInStack(alignedstack) if count != numpart*(numpart-1): ### we have a valid file already apDisplay.printError("There are only %d lines need to have %d"%(count, numpart*(numpart-1))) return similarfile, simlist
def start(self): ### copy stack to rundir newstack = os.path.join(self.params['rundir'], "start.hed") if os.path.isfile(newstack): apDisplay.printError("Stack already exists") emancmd = "proc2d %s %s"%(self.params['stackfile'], newstack) if self.params['normalize'] is True: emancmd += " edgenorm" apEMAN.executeEmanCmd(emancmd) ### set final parameters boxsize = apFile.getBoxSize(newstack) apDisplay.printMsg("Boxsize: %i"%boxsize[0]) if not boxsize or boxsize <= 0: apDisplay.printError("Could not determine stack size") else: self.boxsize = boxsize[0] self.numpart = apFile.numImagesInStack(newstack) apDisplay.printMsg("Num part: %i"%self.numpart) if not self.numpart or self.numpart <= 0: apDisplay.printError("Could not determine number of particles") if self.numpart <= 1: apDisplay.printError("Not enough particles to upload") apStack.averageStack(newstack) #self.params['commit'] = False self.createStackData()
def start(self): print self.params if self.params['clusterId'] is not None: self.clusterdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterId']) self.useClusterForTemplateStack() elif self.params['alignId'] is not None: self.aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignId']) self.useClusterForTemplateStack() else: apDisplay.printMsg("Using local file: '"+str(self.params['templatestack'])+"' to upload template") # copy templates to final location if str(self.params['templatestack'])[-4:] == ".img" or str(self.params['templatestack'])[-4:] == ".hed": self.params['templatestack'] = self.params['templatestack'][:-4] if str(self.params['runname'])[-4:] == ".img" or str(self.params['runname'])[-4:0] == ".hed": self.params['runname'] = self.params['runname'][:-4] shutil.copyfile(str(self.params['templatestack'])+".img", os.path.join(self.params['rundir'], str(self.params['runname'])+".img")) shutil.copyfile(str(self.params['templatestack'])+".hed", os.path.join(self.params['rundir'], str(self.params['runname'])+".hed")) self.numimages = apFile.numImagesInStack(os.path.join(self.params['rundir'], str(self.params['runname'])+".hed")) # insert templates to database if self.params['commit'] is True: self.uploadTemplateStack(insert=True) else: apDisplay.printWarning("not committing results to DB")
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['description'] is None: apDisplay.printError("run description was not defined") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if self.params['partrad'] is None: apDisplay.printError("particle radius was not defined") ### deal with particles maxparticles = 150000 self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile))) ### get num processors if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() ### convert / check template data if self.params['templatelist'] is None: apDisplay.printError("template list was not provided") self.templatelist = self.params['templatelist'].strip().split(",") if not self.templatelist or type(self.templatelist) != type([]): apDisplay.printError("could not parse template list=" + self.params['templatelist']) self.params['numtemplate'] = len(self.templatelist) apDisplay.printMsg("Found " + str(self.params['numtemplate']) + " templates") ### convert / check template data if self.params['orientref'] is None: apDisplay.printError("reference for orientation was not provided")
def checksizes(self): newnots = apFile.numImagesInStack(self.params['notstack']) newtilts = apFile.numImagesInStack(self.params['tiltstack']) stackdata1 = apStack.getOnlyStackData(self.params['stack1'], msg=False) stackpath1 = os.path.join(stackdata1['path']['path'], stackdata1['name']) oldnots = apFile.numImagesInStack(stackpath1) stackdata2 = apStack.getOnlyStackData(self.params['stack2'], msg=False) stackpath2 = os.path.join(stackdata2['path']['path'], stackdata2['name']) oldtilts = apFile.numImagesInStack(stackpath2) if newtilts != oldtilts: apDisplay.printWarning("tilted stack are different sizes: %d vs. %d part"%(newtilts,oldtilts)) else: apDisplay.printMsg("tilted stack are the sames sizes: %d part"%(newtilts)) if newnots != oldnots: apDisplay.printWarning("untilted stack are different sizes: %d vs. %d part"%(newnots,oldnots)) else: apDisplay.printMsg("untilted stack are the sames sizes: %d part"%(newnots))
def appendStackFileToStackFile(stackfile, mergestackfile, msg=True): """ takes two stack files and merges them into second file """ ### initialization t0 = time.time() root = os.path.splitext(mergestackfile)[0] mergeheaderfile = root + ".hed" mergedatafile = root + ".img" root = os.path.splitext(stackfile)[0] stackheaderfile = root + ".hed" stackdatafile = root + ".img" ### merge data files addnumpart = apFile.numImagesInStack(stackheaderfile) addsize = apFile.fileSize(stackdatafile) premergenumpart = apFile.numImagesInStack(mergeheaderfile) premergesize = apFile.fileSize(mergedatafile) fout = file(mergedatafile, 'ab') fin = file(stackdatafile, 'rb') shutil.copyfileobj(fin, fout, 65536) fin.close() fout.close() finalsize = apFile.fileSize(mergedatafile) if finalsize != addsize + premergesize: apDisplay.printError( "size mismatch %s vs. %s + %s = %s" % (apDisplay.bytes(finalsize), apDisplay.bytes(addsize), apDisplay.bytes(premergesize), apDisplay.bytes(premergesize + addsize))) ### merge header files fout = file(mergeheaderfile, 'ab') fin = file(stackheaderfile, 'rb') shutil.copyfileobj(fin, fout, 65536) fin.close() fout.close() numberStackFile(mergeheaderfile, msg=msg) finalnumpart = apFile.numImagesInStack(mergeheaderfile) if finalnumpart != addnumpart + premergenumpart: apDisplay.printError("size mismatch %d vs. %d + %d = %d" % (finalnumpart, addnumpart, premergenumpart, addnumpart + premergenumpart))
def numberStackFile(oldheadfile, startnum=0, msg=True): """ Takes an Imagic Stack header file and numbers the particles based on http://www.imagescience.de/formats/formats.htm """ if msg is True: apDisplay.printMsg("saving particle numbers to stack header") t0 = time.time() newheadfile = (os.path.splitext(oldheadfile)[0] + "-temp.hed") numimg = apFile.numImagesInStack(oldheadfile) #print numimg numimg_fbyte = intToFourByte(numimg - 1) of = open(oldheadfile, 'rb') nf = open(newheadfile, 'wb') i = startnum while i < numimg: imgnum_fbyte = intToFourByte(i) data = of.read(1024) ### start new string headerstr = "" ### first image number headerstr += intToFourByte(i) ### number of images, less one headerstr += numimg_fbyte ### always 0,1 ??? headerstr += intToFourByte(0) headerstr += intToFourByte(1) ### creation date: day, month, year, hour, min, sec headerstr += intToFourByte(time.localtime()[2]) headerstr += intToFourByte( time.localtime()[1]) #eman always uses month-1? headerstr += intToFourByte(time.localtime()[0]) headerstr += intToFourByte(time.localtime()[3]) headerstr += intToFourByte(time.localtime()[4]) headerstr += intToFourByte(time.localtime()[5]) ### append other header info, 4 character per item headerstr += data[10 * 4:61 * 4] ### first image number, EMAN does this headerstr += intToFourByte(i) ### append other header info, 4 character per item headerstr += data[62 * 4:] #print "" #print "old=", fourByteToInt(data[0:4]), fourByteToInt(data[4:8]), fourByteToInt(data[8:12]) #print "new=", fourByteToInt(headerstr[0:4]), fourByteToInt(headerstr[4:8]), fourByteToInt(headerstr[8:12]) nf.write(headerstr) i += 1 if not os.path.isfile(newheadfile): apDisplay.printError("failed to number particles in stack file") apFile.removeFile(oldheadfile) shutil.move(newheadfile, oldheadfile) if msg is True: apDisplay.printMsg("completed %d particles in %s" % (numimg, apDisplay.timeString(time.time() - t0))) return True
def fillSimilarityMatrix(stackfile): ### Get initial correlation values ### this is really, really slow numpart = apFile.numImagesInStack(stackfile) similarfile = "similarities.dat" if os.path.isfile(similarfile): simf = open(similarfile, 'r') simlist = [] count = 0 for line in simf: count += 1 sline = line.strip() slist = sline.split() ccval = float(slist[2]) simlist.append(ccval) simf.close() apDisplay.printMsg("There are %d lines in the sim file: %s"%(count, similarfile)) if count == numpart*(numpart-1): ### we have a valid file already return similarfile, simlist ### read data and estimate time imagicdict = apImagicFile.readImagic(stackfile) partarray = imagicdict['images'] numpart = partarray.shape[0] boxsize = partarray.shape[1] #timeper = 27.0e-9 timeper = 17.0e-9 apDisplay.printMsg("Computing CC values in about %s" %(apDisplay.timeString(timeper*numpart**2*boxsize**2))) ### Computing CC values simf = open(similarfile, 'w') cctime = time.time() simlist = [] for i in range(0, numpart): if i % 100 == 99: sys.stderr.write(".") for j in range(i+1, numpart): ccval = self.getCCValue(partarray[i],partarray[j]) str1 = "%05d %05d %.10f\n" % (i+1, j+1, ccval) simf.write(str1) str2 = "%05d %05d %.10f\n" % (j+1, i+1, ccval) simf.write(str2) simlist.append(ccval) sys.stderr.write("\n") simf.close() del partarray del imagicdict['images'] apDisplay.printMsg("CC calc time: %s :: %s per part :: %s per part per pixel" %(apDisplay.timeString(time.time()-cctime), apDisplay.timeString((time.time()-cctime)/numpart**2), apDisplay.timeString((time.time()-cctime)/numpart**2/boxsize**2))) return similarfile, simlist
def appendToStack(self, stackdata): addfile = os.path.join( stackdata['path']['path'], stackdata['name'] ) addnumpart = apFile.numImagesInStack(addfile) orignumpart = apFile.numImagesInStack(self.combinefile) cmd = "proc2d "+addfile+" "+self.combinefile apDisplay.printMsg("adding "+str(addnumpart)+" particles to "+str(orignumpart)+" particles") apEMAN.executeEmanCmd(cmd, verbose=True, showcmd=True) newnumpart = apFile.numImagesInStack(self.combinefile) apDisplay.printMsg("added "+str(addnumpart)+" particles from stackid="+str(stackdata.dbid)+" to " +str(orignumpart)+" particles giving a new combined stack of "+str(newnumpart)+" particles") if addnumpart + orignumpart < newnumpart: apDisplay.printError("Error in stack merging, too few particles added") return
def numberStackFile(oldheadfile, startnum=0): """ Takes an Imagic Stack header file and numbers the particles based on http://www.imagescience.de/formats/formats.htm """ apDisplay.printMsg("saving particle numbers to stack header") t0 = time.time() newheadfile = (os.path.splitext(oldheadfile)[0]+"-temp.hed") numimg = apFile.numImagesInStack(oldheadfile) #print numimg numimg_fbyte = intToFourByte(numimg-1) of = open(oldheadfile, 'rb') nf = open(newheadfile, 'wb') i = startnum while i < numimg: imgnum_fbyte = intToFourByte(i) data = of.read(1024) ### start new string headerstr = "" ### first image number headerstr += intToFourByte(i) ### number of images, less one headerstr += numimg_fbyte ### always 0,1 ??? headerstr += intToFourByte(0) headerstr += intToFourByte(1) ### creation date: day, month, year, hour, min, sec headerstr += intToFourByte(time.localtime()[2]) headerstr += intToFourByte(time.localtime()[1]) #eman always uses month-1? headerstr += intToFourByte(time.localtime()[0]) headerstr += intToFourByte(time.localtime()[3]) headerstr += intToFourByte(time.localtime()[4]) headerstr += intToFourByte(time.localtime()[5]) ### append other header info, 4 character per item headerstr += data[10*4:61*4] ### first image number, EMAN does this headerstr += intToFourByte(i) ### append other header info, 4 character per item headerstr += data[62*4:] #print "" #print "old=", fourByteToInt(data[0:4]), fourByteToInt(data[4:8]), fourByteToInt(data[8:12]) #print "new=", fourByteToInt(headerstr[0:4]), fourByteToInt(headerstr[4:8]), fourByteToInt(headerstr[8:12]) nf.write(headerstr) i += 1 if not os.path.isfile(newheadfile): apDisplay.printError("failed to number particles in stack file") apFile.removeFile(oldheadfile) shutil.move(newheadfile, oldheadfile) apDisplay.printMsg("completed %d particles in %s"%(numimg, apDisplay.timeString(time.time()-t0))) return True
def wrtieEvenParticles(self, stackfile, outfile=None): numpart = apFile.numImagesInStack(stackfile) evenPartList = 2 * numpy.arange(numpart / 2) + 2 self.stacksToMerge = [] if outfile is None: root = os.path.splitext(stackfile)[0] outfile = root + ".even.hed" self.start(stackfile, partlist=evenPartList) mergeStacks(self.stacksToMerge, outfile, self.msg) for tempstackfile in self.stacksToMerge: apFile.removeStack(tempstackfile, warn=self.msg) return outfile
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['description'] is None: apDisplay.printError("run description was not defined") ### convert / check template data if self.params['templatelist'] is None: apDisplay.printError("template list was not provided") self.templatelist = self.params['templatelist'].strip().split(",") if not self.templatelist or type(self.templatelist) != type([]): apDisplay.printError("could not parse template list="+self.params['templatelist']) self.params['numtemplate'] = len(self.templatelist) apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if not self.params['fastmode'] in self.fastmodes: apDisplay.printError("fast mode must be on of: "+str(self.fastmodes)) if not self.params['converge'] in self.convergemodes: apDisplay.printError("converge mode must be on of: "+str(self.convergemodes)) maxparticles = 150000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) boxsize = apStack.getStackBoxsize(self.params['stackid']) self.clipsize = int(math.floor(boxsize/float(self.params['bin']*2)))*2 if self.params['clipsize'] is not None: if self.params['clipsize'] >= self.clipsize: apDisplay.printError("requested clipsize is too big %d > %d" %(self.params['clipsize'],self.clipsize)) self.clipsize = self.params['clipsize'] if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile)
def makeNewStack(oldstack, newstack, listfile=None, remove=False, bad=False): """ selects particular particles from a stack """ if not os.path.isfile(oldstack): apDisplay.printWarning("could not find old stack: " + oldstack) if os.path.isfile(newstack): if remove is True: apDisplay.printWarning("removing old stack: " + newstack) time.sleep(2) apFile.removeStack(newstack) else: apDisplay.printError("new stack already exists: " + newstack) apDisplay.printMsg("creating a new stack\n\t" + newstack + "\nfrom the oldstack\n\t" + oldstack + "\n") a = proc2dLib.RunProc2d() a.setValue('infile', oldstack) a.setValue('outfile', newstack) if listfile is not None: a.setValue('list', listfile) a.run() if bad is True and listfile is not None: ### run only if num bad particles < num good particles newstacknumpart = apFile.numImagesInStack(newstack) oldstacknumpart = apFile.numImagesInStack(oldstack) if newstacknumpart > oldstacknumpart / 2: ### create bad.hed stack with all bad particles badstack = os.path.join(os.path.dirname(newstack), "bad.hed") emancmd = "proc2d %s %s exclude=%s" % (oldstack, badstack, listfile) apEMAN.executeEmanCmd(emancmd, verbose=True) else: apDisplay.printMsg( "Rejecting more particles than keeping, not creating a bad stack" ) return
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['description'] is None: apDisplay.printError("run description was not defined") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if self.params['partrad'] is None: apDisplay.printError("particle radius was not defined") ### deal with particles maxparticles = 150000 self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(self.stackdata['path']['path'], self.stackdata['name']) if self.params['numpart'] is None: self.params['numpart'] = apFile.numImagesInStack(stackfile) if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: "+str(maxparticles)+" requested: "+str(self.params['numpart'])) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles "+str(self.params['numpart']) +" than available "+str(apFile.numImagesInStack(stackfile))) ### get num processors if self.params['nproc'] is None: self.params['nproc'] = apParam.getNumProcessors() ### convert / check template data if self.params['templatelist'] is None: apDisplay.printError("template list was not provided") self.templatelist = self.params['templatelist'].strip().split(",") if not self.templatelist or type(self.templatelist) != type([]): apDisplay.printError("could not parse template list="+self.params['templatelist']) self.params['numtemplate'] = len(self.templatelist) apDisplay.printMsg("Found "+str(self.params['numtemplate'])+" templates") ### convert / check template data if self.params['orientref'] is None: apDisplay.printError("reference for orientation was not provided")
def stackToEmanMrc(stackfile): numpart = apFile.numImagesInStack(stackfile) partlist = [] for i in range(numpart): mrcfile = "eman%02d.mrc" % (i) emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, mrcfile, i, i) apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False) if not os.path.isfile(mrcfile): print "Failed to create MRC" sys.exit(1) part = mrc.read(mrcfile) apFile.removeFile(mrcfile) partlist.append(part) return partlist
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['description'] is None: apDisplay.printError("run description was not defined") if self.params['runname'] is None: apDisplay.printError("run name was not defined") maxparticles = 150000 if self.params['numpart'] > maxparticles: apDisplay.printError("too many particles requested, max: " + str(maxparticles) + " requested: " + str(self.params['numpart'])) if self.params['initmethod'] not in self.initmethods: apDisplay.printError("unknown initialization method defined: " + str(self.params['initmethod']) + " not in " + str(self.initmethods)) stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) if self.params['numpart'] > apFile.numImagesInStack(stackfile): apDisplay.printError("trying to use more particles " + str(self.params['numpart']) + " than available " + str(apFile.numImagesInStack(stackfile)))
def stackToEmanMrc(stackfile): numpart = apFile.numImagesInStack(stackfile) partlist = [] for i in range(numpart): mrcfile = "eman%02d.mrc"%(i) emancmd = "proc2d %s %s first=%d last=%d"%(stackfile, mrcfile, i, i) apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False) if not os.path.isfile(mrcfile): print "Failed to create MRC" sys.exit(1) part = mrc.read(mrcfile) apFile.removeFile(mrcfile) partlist.append(part) return partlist
def useStackForTemplate(self): apDisplay.printMsg("Using stack to make templates") sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) self.params['sessionname'] = sessiondata['name'] self.params['apix'] = apStack.getStackPixelSizeFromStackId( self.params['stackid']) stackdata = apStack.getOnlyStackData(self.params['stackid']) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) self.params['templatelist'] = [] if self.params['avgstack'] is True: ### averaging all particles together apDisplay.printMsg("Averaging all images in stack") templatename = "stack%d-average.mrc" % (self.params['stackid']) oldtemplatepath = os.path.join(stackdata['path']['path'], "average.mrc") abstemplatepath = os.path.join(stackdata['path']['path'], templatename) if os.path.isfile(oldtemplatepath): shutil.copy(oldtemplatepath, abstemplatepath) else: ### average all images using proc2d emancmd = "proc2d %s %s average" % (stackfile, abstemplatepath) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'] = [abstemplatepath] elif self.params['imgnums'] is not None: ### check to see if stackimagenum is within the boundary of the stack numpart = apFile.numImagesInStack(stackfile) for i in self.params['imgnums'].split(","): partnum = int(i) if partnum > numpart or partnum < 0: apDisplay.printError( "'imgnums' is NOT within the boundary of the stack: %d > %d" % (partnum, numpart)) apDisplay.printMsg("Extracting image %d from stack" % partnum) templatename = "stack%d-particle%d.mrc" % ( self.params['stackid'], partnum) abstemplatepath = os.path.join(self.params['rundir'], templatename) ### run proc2d with params['stackimgnum'] emancmd = "proc2d %s %s first=%d last=%d" % ( stackfile, abstemplatepath, partnum, partnum) ### create template apDisplay.printMsg("creating " + templatename) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'].append(abstemplatepath)
def makeNewStack(oldstack, newstack, listfile=None, remove=False, bad=False): """ selects particular particles from a stack """ if not os.path.isfile(oldstack): apDisplay.printWarning("could not find old stack: "+oldstack) if os.path.isfile(newstack): if remove is True: apDisplay.printWarning("removing old stack: "+newstack) time.sleep(2) apFile.removeStack(newstack) else: apDisplay.printError("new stack already exists: "+newstack) apDisplay.printMsg("creating a new stack\n\t"+newstack+ "\nfrom the oldstack\n\t"+oldstack+"\n") a = proc2dLib.RunProc2d() a.setValue('infile',oldstack) a.setValue('outfile',newstack) if listfile is not None: a.setValue('list',listfile) a.run() if bad is True and listfile is not None: ### run only if num bad particles < num good particles newstacknumpart = apFile.numImagesInStack(newstack) oldstacknumpart = apFile.numImagesInStack(oldstack) if newstacknumpart > oldstacknumpart/2: ### create bad.hed stack with all bad particles badstack = os.path.join(os.path.dirname(newstack), "bad.hed") emancmd = "proc2d %s %s exclude=%s"%(oldstack, badstack, listfile) apEMAN.executeEmanCmd(emancmd, verbose=True) else: apDisplay.printMsg("Rejecting more particles than keeping, not creating a bad stack") return
def appendToStack(self, stackdata): addfile = os.path.join(stackdata['path']['path'], stackdata['name']) addnumpart = apFile.numImagesInStack(addfile) orignumpart = apFile.numImagesInStack(self.combinefile) cmd = "proc2d " + addfile + " " + self.combinefile apDisplay.printMsg("adding " + str(addnumpart) + " particles to " + str(orignumpart) + " particles") apEMAN.executeEmanCmd(cmd, verbose=True, showcmd=True) newnumpart = apFile.numImagesInStack(self.combinefile) apDisplay.printMsg("added " + str(addnumpart) + " particles from stackid=" + str(stackdata.dbid) + " to " + str(orignumpart) + " particles giving a new combined stack of " + str(newnumpart) + " particles") if addnumpart + orignumpart < newnumpart: apDisplay.printError( "Error in stack merging, too few particles added") return
def postProcessParticleStack(self, imgdata, imgstackfile, boxedpartdatas, parttree_length): ### phase flipping t0 = time.time() if self.params['phaseflipped'] is True: if self.params['fliptype'] == 'emantilt': ### ctf correct individual particles with tilt using Eman imgstackfile = self.tiltPhaseFlipParticles(imgdata, imgstackfile, boxedpartdatas) self.ctftimes.append(time.time()-t0) elif self.params['fliptype'] == 'emanpart': ### ctf correct individual particles using Eman imgstackfile = self.phaseFlipParticles(imgdata, imgstackfile) self.ctftimes.append(time.time()-t0) else: apDisplay.printMsg("phase flipped whole image already") else: apDisplay.printMsg("not phase flipping particles") numpart = apFile.numImagesInStack(imgstackfile) apDisplay.printMsg(str(numpart)+" particles were boxed out from "+self.shortname) if parttree_length != numpart: apDisplay.printError("There is a mismatch in the number of particles expected and that were boxed") ### rectangular box masking if self.params['boxmask'] is not None: # create a stack of rectangular masks, which will be applied to # the particles as they are added to the final stack pixsz = apDatabase.getPixelSize(imgdata)*self.params['bin'] boxsz = self.boxsize/self.params['bin'] bxmask = int(self.params['bxmask']/pixsz) bymask = int(self.params['bymask']/pixsz) if self.params['bimask'] > 0: bimask = int(self.params['bimask']/pixsz) else: bimask = None falloff = self.params['falloff']/pixsz # adjust masks for falloff bxmask -= falloff/2 bymask = (bymask/2)-(falloff/2) self.params['boxmaskf'] = os.path.splitext(imgstackfile)[0]+"-mask.hed" apBoxer.boxMaskStack(self.params['boxmaskf'], boxedpartdatas, boxsz, bxmask, bymask, falloff, bimask, norotate=self.params['rotate']) return imgstackfile
def postLoop(self): if len(self.stackarray) > 0: stackroot = self.outstackfile[:-4] stackname = "%s-%d.hed"%(stackroot, self.imgnum) apDisplay.printMsg("writing single particles to file "+stackname) self.stacklist.append(stackname) apFile.removeStack(stackname, warn=False) apImagicFile.writeImagic(self.stackarray, stackname, msg=False) ### merge stacks apFile.removeStack(self.outstackfile, warn=False) apImagicFile.mergeStacks(self.stacklist, self.outstackfile) filepart = apFile.numImagesInStack(self.outstackfile) if filepart != self.imgnum: apDisplay.printError("number merged particles (%d) not equal number expected particles (%d)"% (filepart, numpart)) for stackname in self.stacklist: apFile.removeStack(stackname, warn=False) ### summarize apDisplay.printColor("merged %d particles in %s"%(self.imgnum, apDisplay.timeString(time.time()-self.starttime)), "cyan")
def start(self): self.writeParams() stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False) stackpath = os.path.abspath(stackdata["path"]["path"]) stackfile = os.path.join(stackpath, "start.hed") self.params["localstack"] = os.path.join(stackpath, self.timestamp + ".hed") proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(self.params["step"]) if self.params["bin"] > 1: proccmd += " shrink=%d" % int(self.params["bin"]) proccmd += " last=" + str(self.params["numpart"] - 1) apParam.runCmd(proccmd, "EMAN", verbose=True) if self.params["numpart"] != apFile.numImagesInStack(self.params["localstack"]): apDisplay.printError("Missing particles in stack") apXmipp.breakupStackIntoSingleFiles(self.params["localstack"], filetype="mrc") if self.params["prehip"] == "yes": if self.params["rise"] is not None: self.writeLLBO1() elif self.params["ll1"] is not None: self.writeLLBO2() elif self.params["rise"] is None and self.params["ll1"] is None: apDisplay.printError( "You must specify either rise and twist or (1,0) and (0,1) layer lines to run preHIP" ) apDisplay.printMsg("now running s_prehip to set up input files") apDisplay.printMsg( "Make sure the layer line/bessel order assignment is correct. If it is not, you may need to adjust some of the input variables." ) cmd = "s_prehip" proc = subprocess.Popen(cmd) proc.wait() else: apDisplay.printMsg("now running s_hip2: Phoelix programs") cmd = "s_hip2" proc = subprocess.Popen(cmd) proc.wait() # apParam.runCmd(cmd, package="Phoelix") self.putFilesInStack() self.insertHipRunData() self.insertHipParamsData() self.insertHipIterData() self.insertHipParticleData()
def useStackForTemplate(self): apDisplay.printMsg("Using stack to make templates") sessiondata = apStack.getSessionDataFromStackId(self.params['stackid']) self.params['sessionname'] = sessiondata['name'] self.params['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid']) stackdata = apStack.getOnlyStackData(self.params['stackid']) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) self.params['templatelist'] = [] if self.params['avgstack'] is True: ### averaging all particles together apDisplay.printMsg("Averaging all images in stack") templatename = "stack%d-average.mrc"%(self.params['stackid']) oldtemplatepath = os.path.join(stackdata['path']['path'], "average.mrc") abstemplatepath = os.path.join(stackdata['path']['path'], templatename) if os.path.isfile(oldtemplatepath): shutil.copy(oldtemplatepath, abstemplatepath) else: ### average all images using proc2d emancmd = "proc2d %s %s average" % (stackfile, abstemplatepath) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'] = [abstemplatepath] elif self.params['imgnums'] is not None: ### check to see if stackimagenum is within the boundary of the stack numpart = apFile.numImagesInStack(stackfile) for i in self.params['imgnums'].split(","): partnum = int(i) if partnum > numpart or partnum < 0: apDisplay.printError("'imgnums' is NOT within the boundary of the stack: %d > %d"%(partnum,numpart)) apDisplay.printMsg("Extracting image %d from stack" % partnum) templatename = "stack%d-particle%d.mrc"%(self.params['stackid'],partnum) abstemplatepath= os.path.join(self.params['rundir'], templatename) ### run proc2d with params['stackimgnum'] emancmd = "proc2d %s %s first=%d last=%d" % (stackfile, abstemplatepath, partnum, partnum) ### create template apDisplay.printMsg("creating "+templatename) apEMAN.executeEmanCmd(emancmd) if os.path.isfile(abstemplatepath): self.params['templatelist'].append(abstemplatepath)
def getParticleParams(self): apDisplay.printColor("now getting particle shift / rotation / correlation parameters", "cyan") f = open(os.path.join(self.params['rundir'], "outparams.plt"), "r") lines = f.readlines() strip = [line.strip() for line in lines] split = [params.split() for params in strip] params = [] for list in split: numberlist = [eval(p) for p in list] ### figure out if the particle is mirrored, based on which reference it belongs to numrefs = apFile.numImagesInStack(os.path.join(self.params['rundir'], "references.hed")) half = numrefs / 2 if numberlist[4] > half and self.params['mirror'] is True: mirror = 1 else: mirror = 0 numberlist.append(mirror) params.append(numberlist) return params