def createChimeraVolumeSnapshot(self, volume, iteration, reference_number=1): """ create chimera snapshot images of volume """ if self.params["snapfilter"]: resolution = self.params["snapfilter"] else: newfscfile = os.path.join( self.resultspath, "recon_%s_it%.3d_vol%.3d.fsc" % (self.params["timestamp"], iteration, reference_number), ) try: resolution = apRecon.getResolutionFromGenericFSCFile( newfscfile, self.runparams["boxsize"], self.runparams["apix"] ) except: apDisplay.printWarning("Failed to get resolution from generic FSC file: " + newfscfile) resolution = 30 apDisplay.printWarning("Running Chimera Snapshot with resolution: %d " % resolution) apChimera.filterAndChimera( volume, resolution, self.runparams["apix"], self.runparams["boxsize"], "snapshot", self.params["contour"], self.params["zoom"], sym="c1", mass=self.params["mass"], )
def checkConflicts(self): if self.params['stackid'] is None: apDisplay.printError("stack id was not defined") if self.params['rad'] is None: apDisplay.printError("no particle radius set") if self.params['modelid'] is None: apDisplay.printError("model id was not defined") if self.params['lastring'] is None: apDisplay.printError("a last ring radius was not provided") if self.params['runname'] is None: apDisplay.printError("run name was not defined") if self.params['incr'] is None: apDisplay.printError("angular increments are not specified") stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False) stackfile = os.path.join(stackdata['path']['path'], stackdata['name']) numstackp = apFile.numImagesInStack(stackfile) if self.params['numpart'] > numstackp: apDisplay.printError("trying to use more particles "+str(numstackp) +" than available "+str(numstackp)) elif self.params['numpart'] == 0: apDisplay.printWarning("using all "+str(numstackp)+" particles") self.params['numpart'] = numstackp boxsize = apStack.getStackBoxsize(self.params['stackid']) if self.params['lastring'] > boxsize/2-2: apDisplay.printError("last ring radius is too big for boxsize " +str(self.params['lastring'])+" > "+str(boxsize/2-2)) if self.params['lastring']+self.params['xysearch'] > boxsize/2-2: apDisplay.printError("last ring plus xysearch radius is too big for boxsize " +str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2)) if (self.params['xysearch'] % self.params['xystep']) > 0: apDisplay.printError("translational search (xy-search) must be divisible by search step (xy-step)")
def _initializeDoneDict(self): """ reads or creates a done dictionary """ self.donedictfile = os.path.join(self.params['rundir'] , self.functionname+".donedict") if os.path.isfile(self.donedictfile) and self.params['continue'] == True: ### unpickle previously done dictionary apDisplay.printMsg("Reading old done dictionary: "+os.path.basename(self.donedictfile)) f = open(self.donedictfile,'r') self.donedict = cPickle.load(f) f.close() if not 'commit' in self.donedict or self.donedict['commit'] == self.params['commit']: ### all is well apDisplay.printMsg("Found "+str(len(self.donedict))+" done dictionary entries") return elif self.donedict['commit'] is True and self.params['commit'] is not True: ### die apDisplay.printError("Commit flag was enabled and is now disabled, create a new runname") else: ### set up fresh dictionary apDisplay.printWarning("'--commit' flag was changed, creating new done dictionary") ### set up fresh dictionary self.donedict = {} self.donedict['commit'] = self.params['commit'] apDisplay.printMsg("Creating new done dictionary: "+os.path.basename(self.donedictfile)) ### write donedict to file f = open(self.donedictfile, 'w', 0666) cPickle.dump(self.donedict, f) f.close() return
def runChimeraScript(chimscript, xvfb=False): if not chimscript or not os.path.isfile(chimscript): print chimscript apDisplay.printError("Could not find file: apChimSnapshot.py") #apDisplay.printColor("Trying to use chimera for model imaging","cyan") if xvfb is True: port = apParam.resetVirtualFrameBuffer() time.sleep(1) if 'CHIMERA' in os.environ and os.path.isdir(os.environ['CHIMERA']): chimpath = os.environ['CHIMERA'] os.environ['CHIMERA'] = chimpath os.environ['CHIMERAPATH'] = os.path.join(chimpath,"share") os.environ['LD_LIBRARY_PATH'] = os.path.join(chimpath,"lib")+":"+os.environ['LD_LIBRARY_PATH'] chimexe = os.path.join(chimpath,"bin/chimera") if not os.path.isfile(chimexe): apDisplay.printWarning("Could not find chimera at: "+chimexe) else: chimpath = None chimexe = "chimera" #apDisplay.printWarning("'CHIMERA' environmental variable is unset") rendercmd = (chimexe+" --debug python:"+chimscript) logf = open("chimeraRun.log", "a") apDisplay.printColor("running Chimera:\n "+rendercmd, "cyan") if xvfb is True: print "import -verbose -display :%d -window root screencapture.png"%(port) proc = subprocess.Popen(rendercmd, shell=True, stdout=logf, stderr=logf) proc.wait() logf.close() if xvfb is True: apParam.killVirtualFrameBuffer(port) return
def start(self): """ this is the main component of the script where all the processing is done """ ### initialize some variables self.runq = None self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid']) apDisplay.printMsg("Pixel size: %.5f"%(self.apix)) self.boxsize = apStack.getStackBoxsize(self.params['stackid']) apDisplay.printMsg("Box size: %d"%(self.boxsize)) self.checkResults() self.stackmapping = apRecon.partnum2defid(self.params['stackid']) self.numiter = self.getNumberOfIterations() for i in range(self.numiter): iternum = i+1 apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green") self.uploadIteration(iternum) reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir']) if reconrunid: apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid)) eulerjump = apEulerJump.ApEulerJump() eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid']) apRecon.setGoodBadParticlesFromReconId(reconrunid) else: apDisplay.printWarning("Could not find recon run id")
def start(self): self.setFileName() scale = float(self.params['oldapix'])/self.params['newapix'] mrcname = os.path.join(self.params['rundir'], self.params['name']+".mrc") origmodel = self.params['file'] if os.path.isfile(mrcname): apDisplay.printError("File exists") if (abs(self.params['oldapix'] - self.params['newapix']) > 1.0e-2 or abs(self.params['oldbox'] - self.params['newbox']) > 1.0e-1): ### rescale old model to a new size apDisplay.printWarning("rescaling original model to a new size") scale = float(self.params['oldapix'])/self.params['newapix'] apDisplay.printMsg("rescaling model by scale factor of %.4f"%(scale)) apVolume.rescaleVolume(origmodel, mrcname, self.params['oldapix'], self.params['newapix'], self.params['newbox']) else: ### simple upload, just copy file to models folder apDisplay.printMsg("copying original model to a new location: "+mrcname) shutil.copyfile(origmodel, mrcname) if self.params['viper2eman'] is True: apVolume.viper2eman(mrcname, mrcname, apix=self.params['newapix']) ### render chimera images of model contour = self.params['contour'] if self.params['mass'] is not None: apChimera.setVolumeMass(mrcname, self.params['newapix'], self.params['mass']) contour = 1.0 apChimera.renderSnapshots(mrcname, contour=contour, zoom=self.params['zoom'], sym=self.params['symdata']['eman_name']) self.insertModel(mrcname)
def fixAmpContrast(self, defocus, raddata, PSDarray, lowerbound, upperbound): if self.ctfvalues['amplitude_contrast'] > 0 and self.ctfvalues['amplitude_contrast'] < self.maxAmpCon: return defocus bad = True newdefocus = defocus count = 0 while count < 20: count += 1 amplitudecontrast = sinefit.refineAmplitudeContrast(raddata[lowerbound:upperbound]*1e10, newdefocus, PSDarray[lowerbound:upperbound], self.ctfvalues['cs'], self.wavelength, msg=self.debug) if amplitudecontrast is None: apDisplay.printWarning("FAILED to fix amplitude contrast") return defocus elif amplitudecontrast < 0: apDisplay.printColor("Amp Cont: %.3f too small, decrease defocus %.3e"% (amplitudecontrast, newdefocus), "blue") scaleFactor = 0.999 - abs(amplitudecontrast)/5. newdefocus = newdefocus*scaleFactor elif amplitudecontrast > self.maxAmpCon: apDisplay.printColor("Amp Cont: %.3f too large, increase defocus %.3e"% (amplitudecontrast, newdefocus), "cyan") scaleFactor = 1.001 + abs(amplitudecontrast - self.maxAmpCon)/5. newdefocus = newdefocus*scaleFactor else: apDisplay.printColor("Amp Cont: %.3f in range!!! defocus %.3e"% (amplitudecontrast, newdefocus), "green") avgres = self.getResolution(newdefocus, raddata, PSDarray, lowerbound) self.ctfvalues['amplitude_contrast'] = amplitudecontrast return newdefocus avgres = self.getResolution(newdefocus, raddata, PSDarray, lowerbound) if avgres < self.bestres: defocus = newdefocus apDisplay.printWarning("FAILED to fix amplitude contrast") return defocus
def getGenerationFiles(self): if self.files is None: files = glob.glob("class_averages_generation_*.hdf") if len(files) == 0: apDisplay.printWarning("Could not find class_averages_generation_*.hdf files.") self.files = sorted(files, key=lambda a: self.generationFromFile(a)) return self.files
def insertFSCData(self, fscfile, refineIterData): """ current functions read the inverse pixel number and correlation value. This function, therefore, converts generic FSC parameters into pixel / correlation format """ if not os.path.isfile(fscfile): apDisplay.printWarning("Could not open FSC file: " + fscfile) ### skip commented out lines and read FSC data f = open(fscfile, "r") fscfileinfo = f.readlines() f.close() for i, info in enumerate(fscfileinfo): if info[0] == "#": pass else: fscfileinfo = fscfileinfo[i:] break ### insert FSC data into database with the convention: pixel number, FSC numinserts = 0 for j, info in enumerate(fscfileinfo): fscq = appiondata.ApFSCData() fscq["refineIter"] = refineIterData fscq["pix"] = j + 1 fscq["value"] = float(info.split()[1]) numinserts += 1 if self.params["commit"] is True: fscq.insert() if self.params["commit"] is True: apDisplay.printMsg("inserted " + str(numinserts) + " rows of FSC data into database") return
def setVolumeMass(volumefile, apix=1.0, mass=1.0, rna=0.0): """ set the contour of 1.0 to the desired mass (in kDa) of the macromolecule based on its density use RNA to set the percentage of RNA in the structure """ if isValidVolume(volumefile) is False: apDisplay.printError("Volume file is not valid") procbin = apParam.getExecPath("proc2d") emandir = os.path.dirname(procbin) volumebin = os.path.join(emandir, "volume") if not os.path.isfile(volumebin): apDisplay.printWarning("failed to find volume program") return False command = "%s %s %.3f set=%.3f"%( volumebin, volumefile, apix, mass ) t0 = time.time() proc = subprocess.Popen(command, shell=True) proc.wait() if time.time()-t0 < 0.01: apDisplay.printWarning("failed to scale by mass in "+apDisplay.timeString(time.time()-t0)) return False apDisplay.printMsg("finished scaling by mass in "+apDisplay.timeString(time.time()-t0)) return True
def checkSelOrDocFileRootDirectory(self, sel_doc_file, old_rootdir, new_rootdir): ''' necessary when files are transferred from one file system to another, e.g., the root directory on Garibaldi is different from that on Guppy''' try: f = open(sel_doc_file, "r") except: apDisplay.printWarning("Unable to open file: %s" % sel_doc_file) return lines = f.readlines() newlines = [] f.close() ### replace old root directory with new root directory count = 0 for line in lines: if re.search(old_rootdir, line): newline = re.sub(old_rootdir, new_rootdir, line) count += 1 else: newline = line newlines.append(newline) if count > 0: apDisplay.printMsg("changing root directory from %s to %s in %s" % (old_rootdir, new_rootdir, sel_doc_file)) f = open(sel_doc_file, "w") f.writelines(newlines) f.close() return
def start(self): ### determine which iterations to upload; last iter is defaulted to infinity uploadIterations = self.verifyUploadIterations() ### upload each iteration for iteration in uploadIterations: for j in range(self.runparams['NumberOfReferences']): ### general error checking, these are the minimum files that are needed vol = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.mrc" % (self.params['timestamp'], iteration, j+1)) particledatafile = os.path.join(self.resultspath, "particle_data_%s_it%.3d_vol%.3d.txt" % (self.params['timestamp'], iteration, j+1)) if not os.path.isfile(vol): apDisplay.printError("you must have an mrc volume file in the 'external_package_results' directory") if not os.path.isfile(particledatafile): apDisplay.printError("you must have a particle data file in the 'external_package_results' directory") ### make chimera snapshot of volume try: self.createChimeraVolumeSnapshot(vol, iteration, j+1) except: apDisplay.printWarning("could not create Chimera volume snapshots") ### instantiate database objects self.insertRefinementRunData(iteration, j+1) self.insertRefinementIterationData(iteration, j+1) ### calculate Euler jumps self.calculateEulerJumpsAndGoodBadParticles(uploadIterations)
def getBoxSize(filename, msg=False): """ return boxsize of stack in pixels """ if not os.path.isfile(filename): if msg is True: apDisplay.printWarning("file does not exist") return (1,1,1) proc = subprocess.Popen("iminfo %s"%(filename), shell=True, stdout=subprocess.PIPE) proc.wait() lines = "" for line in proc.stdout: sline = line.strip() lines += line m = re.match("^Image\(s\) are ([0-9]+)x([0-9]+)x([0-9]+)", sline) if m and m.groups() and len(m.groups()) > 1: xdim = int(m.groups()[0]) ydim = int(m.groups()[1]) zdim = int(m.groups()[2]) return (xdim,ydim,zdim) m = re.match("^0\.\s+([0-9]+)x([0-9]+)\s+", sline) if m and m.groups() and len(m.groups()) > 1: xdim = int(m.groups()[0]) ydim = int(m.groups()[1]) return (xdim,ydim,1) m = re.match("^0\.\s+([0-9]+)x([0-9]+)x([0-9]+)\s+", sline) if m and m.groups() and len(m.groups()) > 1: xdim = int(m.groups()[0]) ydim = int(m.groups()[1]) zdim = int(m.groups()[2]) return (xdim,ydim,zdim) if msg is True: apDisplay.printWarning("failed to get boxsize: "+lines) return (1,1,1)
def readPartDocFile(self, reflist): partlist = [] docfile = "part"+self.params['timestamp']+".doc" if not os.path.isfile(docfile) or apFile.fileSize(docfile) < 50: apDisplay.printWarning("Could not find doc file "+docfile+" to read particle angles") lastdocfile = "iter%03d/part%s_it%06d.doc"%(self.lastiter, self.params['timestamp'], self.lastiter) if not os.path.isfile(lastdocfile) or apFile.fileSize(lastdocfile) < 50: apDisplay.printError("Could not find backup doc file") apDisplay.printColor("Found a backup copy of docfile", "green") shutil.copy(lastdocfile, docfile) f = open(docfile, "r") mininplane = 360.0 for line in f: if line[:2] == ' ;': continue spidict = operations.spiderInLine(line) origpartdict = self.spidict2partdict(spidict) partdict = self.adjustPartDict(origpartdict, reflist) if partdict['inplane'] < mininplane: mininplane = partdict['inplane'] partlist.append(partdict) apDisplay.printMsg("minimum inplane: "+str(mininplane)) for partdict in partlist: partdict['inplane'] = partdict['inplane']-mininplane apDisplay.printMsg("read rotation and shift parameters for "+str(len(partlist))+" particles") if len(partlist) < 1: apDisplay.printError("Did not find any particles in doc file: "+docfile) return partlist
def importPicks(self, picks1, picks2, tight=False, msg=True): t0 = time.time() #print picks1 #print self.currentpicks1 curpicks1 = numpy.asarray(self.currentpicks1) curpicks2 = numpy.asarray(self.currentpicks2) #print curpicks1 # get picks apTiltTransform.setPointsFromArrays(curpicks1, curpicks2, self.data) pixdiam = self.data['pixdiam'] if tight is True: pixdiam /= 4.0 #print self.data, pixdiam list1, list2 = apTiltTransform.alignPicks2(picks1, picks2, self.data, limit=pixdiam, msg=msg) if list1.shape[0] == 0 or list2.shape[0] == 0: apDisplay.printWarning("No new picks were found") # merge picks newpicks1, newpicks2 = apTiltTransform.betterMergePicks(curpicks1, list1, curpicks2, list2, msg=msg) newparts = newpicks1.shape[0] - curpicks1.shape[0] # copy over picks self.currentpicks1 = newpicks1 self.currentpicks2 = newpicks2 if msg is True: apDisplay.printMsg("Inserted "+str(newparts)+" new particles in "+apDisplay.timeString(time.time()-t0)) return True
def readDictFromXml(filename): if not os.path.isfile(filename): apDisplay.printWarning("Failed to open file "+filename+" for reading") return None dom = xml.dom.minidom.parse(filename) xmldict = nodeToDict(dom.childNodes[0]) return xmldict
def _startLoop(self, info): """ appionLoop OVERRIDE initilizes several parameters for a new image and checks if it is okay to start processing image """ if info is None: self.stats['lastimageskipped'] = True self.stats['skipcount'] += 1 return False name = info['filename'] # check to see if image of the same name is already in leginon imgq = leginon.leginondata.AcquisitionImageData(session=self.session, filename=name) results = imgq.query(readimages=False) if results: apDisplay.printWarning("File %s.mrc exists at the destination" % name) apDisplay.printWarning("Skip Uploading") self.stats['lastimageskipped'] = True self.stats['skipcount'] += 1 return False #calc images left self.stats['imagesleft'] = self.stats['imagecount'] - self.stats['count'] #only if an image was processed last if(self.stats['lastcount'] != self.stats['count']): apDisplay.printColor( "\nStarting image "+str(self.stats['count'])\ +" ( skip:"+str(self.stats['skipcount'])+", remain:"\ +str(self.stats['imagesleft'])+" ) file: "\ +apDisplay.short(name), "green") self.stats['lastcount'] = self.stats['count'] self._checkMemLeak() # check to see if image has already been processed if self._alreadyProcessed(info): return False self.stats['waittime'] = 0 return True
def getResolutionData(self, avgpath, iternum): evenvol = os.path.join(avgpath, "avglist3_%dp1.mrc" % (self.params["rescut"])) oddvol = os.path.join(avgpath, "avglist3_%dp2.mrc" % (self.params["rescut"])) evenhed = os.path.join(avgpath, "avglist3_%dp1.hed" % (self.params["rescut"])) oddhed = os.path.join(avgpath, "avglist3_%dp2.hed" % (self.params["rescut"])) emancmd1 = "proc3d %s %s" % (evenvol, evenhed) apEMAN.executeEmanCmd(emancmd1, verbose=True) emancmd2 = "proc3d %s %s" % (oddvol, oddhed) apEMAN.executeEmanCmd(emancmd2, verbose=True) fscfile = "fsc.eotest.%d" % (iternum) fscpath = os.path.join(avgpath, fscfile) emancmd3 = "proc3d %s %s fsc=%s" % (evenhed, oddhed, fscpath) apEMAN.executeEmanCmd(emancmd3, verbose=True) if not os.path.isfile(fscpath): apDisplay.printWarning("Could not find FSC file: " + fscpath) return None f = open(fscpath, "r") xy = f.readlines() lines = len(xy) boxsize = lines * 2.0 f.close() # calculate the resolution: halfres = apRecon.calcRes(fscpath, boxsize, self.params["step"]) apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms" % (halfres), "cyan") # save to database resq = appiondata.ApResolutionData() resq["half"] = halfres resq["fscfile"] = fscpath return resq
def processParticles(self, imgdata, partdatas, shiftdata): self.shortname = apDisplay.short(imgdata['filename']) ### if only selected points along helix, ### fill in points with helical step if self.params['helicalstep']: apix = apDatabase.getPixelSize(imgdata) partdatas = self.fillWithHelicalStep(partdatas, apix) ### run batchboxer self.boxedpartdatas, self.imgstackfile, self.partmeantree = self.boxParticlesFromImage(imgdata, partdatas, shiftdata) if self.boxedpartdatas is None: self.stats['lastpeaks'] = 0 apDisplay.printWarning("no particles were boxed from "+self.shortname+"\n") self.badprocess = True return None self.stats['lastpeaks'] = len(self.boxedpartdatas) apDisplay.printMsg("do not break function now otherwise it will corrupt stack") #time.sleep(1.0) ### merge image particles into big stack totalpart = self.mergeImageStackIntoBigStack(self.imgstackfile, imgdata) ### create a stack average every so often if self.stats['lastpeaks'] > 0: totalPartices = self.existingParticleNumber+self.stats['peaksum']+self.stats['lastpeaks'] logpeaks = math.log(totalPartices) if logpeaks > self.logpeaks: self.logpeaks = math.ceil(logpeaks) numpeaks = math.ceil(math.exp(self.logpeaks)) apDisplay.printMsg("writing averaging stack, next average at %d particles"%(numpeaks)) mrc.write(self.summedParticles/float(totalPartices), "average.mrc") return totalpart
def insertFSCData(self, iternum, refineIterData): fscfile = 'fsc.eotest.%d'%(iternum) fscpath = os.path.join(self.params['rundir'], "iter%03d"%(iternum), fscfile) if not os.path.isfile(fscpath): apDisplay.printWarning("Could not find FSC file: "+fscpath) return None f = open(fscpath, 'r') apDisplay.printMsg("inserting FSC Data into database") numinserts = 0 for line in f: fscq = appiondata.ApFSCData() fscq['refineIter'] = refineIterData sline = line.strip() bits = sline.split('\t') fscq['pix'] = int(bits[0]) fscq['value'] = float(bits[1]) numinserts+=1 if self.params['commit'] is True: fscq.insert() apDisplay.printMsg("inserted "+str(numinserts)+" rows of FSC data into database") f.close()
def findLastCompletedIteration(self): #recondir = os.path.join(self.params['rundir'], "recon") recondir = self.params['rundir'] iternum = 0 stop = False while stop is False: ## check if iteration is complete iternum += 1 class1Volume = "recon_it%03d_half1_model.star"%(iternum) class2Volume = "recon_it%03d_half2_model.star"%(iternum) class1Volumepath = os.path.join(recondir,class1Volume) class2Volumepath = os.path.join(recondir,class2Volume) if not os.path.isfile(class1Volumepath) or not os.path.isfile(class2Volumepath): apDisplay.printWarning("Model.star file %s or %s is missing"%(class1Volumepath, class2Volumepath)) stop = True break ### set last working iteration numiter = iternum-1 if numiter < 1: apDisplay.printError("No iterations were found") apDisplay.printColor("Found %d complete iterations"%(numiter), "green") return numiter
def readBatchUploadInfo(self): # in this example, the batch script file should be separated by tab # see example in function readUploadInfo for format batchfilename = self.params['batchscript'] if not os.path.exists(batchfilename): apDisplay.printError('Batch file %s not exist' % batchfilename) return [] batchfile = open(batchfilename,'r') lines = batchfile.readlines() batchfile.close() batchinfo = [] count = 0 for line in lines: count += 1 #remove white space at ends sline = line.strip() if ' ' in sline: apDisplay.printWarning("There is a space in the batch file on line %d"%(count)) #remove white space at ends cols = sline.split('\t') if len(cols) > 1: batchinfo.append(cols) else: apDisplay.printWarning("Skipping line %d"%(count)) return batchinfo
def convertFSCFileForIteration(self, iteration): ''' RElion FSC info is in recon_it002_model.star and the final recon_model.star file, convert this to 3DEM format For the final iteration, we need to use recon_model.star rather than the iteration specific file to get the actual values. ''' lastiter = self.findLastCompletedIteration() if iteration == lastiter: fscfile = os.path.join(self.projmatchpath, "recon_model.star"%(iteration)) else: fscfile = os.path.join(self.projmatchpath, "recon_it%.3d_half1_model.star"%(iteration)) f = starFile.StarFile( fscfile ) try: f.read() except e: if iteration == lastiter: apDisplay.printWarning("%s file could not be opened, data will NOT be inserted into the database. Relion did NOT run to completion." % fscfile) else: apDisplay.printWarning("%s file could not be opened, data will NOT be inserted into the database" % fscfile) return False dataBlock = f.getDataBlock("data_model_class_1") loopDict = dataBlock.getLoopDict() newfscfile = open(os.path.join(self.resultspath, "recon_%s_it%.3d_vol001.fsc" % (self.params['timestamp'], iteration)), "w") newfscfile.write("### column (1) inverse Angstroms, column (2) Fourier Shell Correlation (FSC)") for valueSet in loopDict: resolution = float(valueSet["_rlnResolution"]) # already in inverse angstrom fsc = float(valueSet["_rlnGoldStandardFsc"]) # should go from a value of 1 to 0 newfscfile.write("%.6f\t%.6f\n" % (resolution, fsc)) newfscfile.close() return True
def sortFolder(self): numsort = 0 apDisplay.printMsg("Sorting files into clean folders") ### move files for all particle iterations files = [] for i in range(self.lastiter+1): iterdir = "iter%03d"%(i) apParam.createDirectory(iterdir, warning=False) wildcard = "part*_it*%03d_*.*"%(i) files.extend(glob.glob(wildcard)) wildcard = "part*_it*%03d.*"%(i) files.extend(glob.glob(wildcard)) for filename in files: if os.path.isfile(filename): numsort += 1 shutil.move(filename,iterdir) if numsort < 3: apDisplay.printWarning("Problem in iteration file sorting, are they already sorted?") apDisplay.printMsg("Sorted "+str(numsort)+" iteration files") ### move files for all reference iterations refsort = 0 refdir = "refalign" apParam.createDirectory(refdir, warning=False) wildcard = "ref*_it*.*" files = glob.glob(wildcard) for filename in files: refsort += 1 shutil.move(filename, refdir) #if refsort < 5: # apDisplay.printError("Problem in reference file sorting") apDisplay.printMsg("Sorted "+str(refsort)+" reference files") return
def writeTiltFile(tiltfile, seriesname, imagedict, azimuth, refimg ): ''' Create the geometry object (tilt angles, dir of tilt axis, origins of ROI) --- 1. read the params from the database 2. write params to text file ''' try: f = open(tiltfile,'w') except: apDisplay.printWarning("Failed to create %s for writing the protomo2 tilt geometry parameter file" % (tiltfile, )) raise f.write('\n') f.write ("TILT SERIES %s\n" % seriesname) f.write ("\n") f.write (" AXIS\n") f.write ("\n") f.write (" TILT AZIMUTH %g\n" % azimuth) f.write ("\n") keys = imagedict.keys() keys.sort() for n in keys: f.write (" IMAGE %-5d FILE %s ORIGIN [ %8.3f %8.3f ] TILT ANGLE %8.3f ROTATION %8.3f\n" % (n, imagedict[n]['filename'], imagedict[n]['x'], imagedict[n]['y'], imagedict[n]['tilt'], imagedict[n]['rotation'])) f.write ("\n") f.write (" REFERENCE IMAGE %d\n" % refimg) f.write ("\n") f.write ("\n") f.write (" END\n") f.close()
def start(self): print self.params if self.params['clusterId'] is not None: self.clusterdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterId']) self.useClusterForTemplateStack() elif self.params['alignId'] is not None: self.aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignId']) self.useClusterForTemplateStack() else: apDisplay.printMsg("Using local file: '"+str(self.params['templatestack'])+"' to upload template") # copy templates to final location if str(self.params['templatestack'])[-4:] == ".img" or str(self.params['templatestack'])[-4:] == ".hed": self.params['templatestack'] = self.params['templatestack'][:-4] if str(self.params['runname'])[-4:] == ".img" or str(self.params['runname'])[-4:0] == ".hed": self.params['runname'] = self.params['runname'][:-4] shutil.copyfile(str(self.params['templatestack'])+".img", os.path.join(self.params['rundir'], str(self.params['runname'])+".img")) shutil.copyfile(str(self.params['templatestack'])+".hed", os.path.join(self.params['rundir'], str(self.params['runname'])+".hed")) self.numimages = apFile.numImagesInStack(os.path.join(self.params['rundir'], str(self.params['runname'])+".hed")) # insert templates to database if self.params['commit'] is True: self.uploadTemplateStack(insert=True) else: apDisplay.printWarning("not committing results to DB")
def writeTileFile2(tiltfile, seriesname, imagelist, origins, tilts, azimuth, refimg): try: f = open(tiltfile,'w') except: apDisplay.printWarning("Failed to create %s for writing the protomo2 tilt geometry parameter file" % (tiltfile, )) raise f.write('\n') f.write ("TILT SERIES %s\n" % seriesname) f.write ("\n") f.write (" AXIS\n") f.write ("\n") f.write (" TILT AZIMUTH %g\n" % azimuth) f.write ("\n") for n in range(len(imagelist)): imagename=os.path.splitext(os.path.basename(imagelist[n]))[0] #strip off path and extension from mrc file f.write (" IMAGE %-5d FILE %s ORIGIN [ %8.3f %8.3f ] TILT ANGLE %8.3f ROTATION %8.3f\n" % (n+1, imagename, origins[n]['x'], origins[n]['y'], tilts[n], 0)) f.write ("\n") f.write (" REFERENCE IMAGE %d\n" % refimg) f.write ("\n") f.write ("\n") f.write (" END\n") f.close()
def loopCommitToDatabase(self, imgdata): """ Over-writes the particleLoop commit and uses the appionLoop commit """ tiltdata = apTiltPair.getTiltPair(imgdata) if tiltdata is None: apDisplay.printWarning("Tilt data not found; not commiting data") return False ### insert the runid self.commitRunToDatabase(imgdata['session'], True) if self.assess is not None: #note runid is overrided to be 'run1' apDatabase.insertImgAssessmentStatus(imgdata, self.params['runname'], self.assess) apDatabase.insertImgAssessmentStatus(tiltdata, self.params['runname'], self.assess) if len(self.peaktree1) < 3 or len(self.peaktree1) < 3: apDisplay.printWarning("Not enough particle picks; not commiting transform or particle data") return False ### insert the transform transdata = apTiltPair.insertTiltTransform(imgdata, tiltdata, self.tiltparams, self.params) ### insert the particles self.insertParticlePeakPairs(imgdata, tiltdata, transdata)
def start(self): sessionname = self.params['sessionname'] runname = self.params['runname'] preset = self.params['preset'] sessionq = leginondata.SessionData(name=sessionname) presetq=leginondata.PresetData(name=preset) imgquery = leginondata.AcquisitionImageData() imgquery['preset'] = presetq imgquery['session'] = sessionq imgtree = imgquery.query(readimages=False) partq = appiondata.ApContourData() sessiond = sessionq.query() selectionid = apParticle.getSelectionIdFromName(runname, sessionname) if not selectionid: apDisplay.printWarning('No Object Tracing Run found in database, Skipping.......') return selectionrundata = apParticle.getSelectionRunDataFromID(selectionid) file = open('contourpickerData-' + sessionname + '.txt','w') file.write('session_id ' + runname + '\n') file.write('usr_id ' + os.getlogin() + '\n') file.write('experiment_name ' + sessionname + '\n') file.write('experiment_description ' + sessiond[0]['comment'].strip() + '\n') file.write('nimages ' + str(len(imgtree)) + '\n') for imgdata in imgtree: file.write('START_IMAGE' + '\n') partq['image'] = imgdata partq['selectionrun'] = selectionrundata partd = partq.query() if len(partd)>0: file.write('image_refID ' + str(partd[0]['image'].dbid) + '\n') file.write('image_name ' + imgdata['filename'] + '\n') if len(partd)>0: file.write('time_roi ' + str(partd[0].timestamp) + '\n') #file.write('time_roi ' + partd[0]['DEF_timestamp'] + '\n') file.write('dfac = 1\n') maxversion = 0 numparticles = 0 for part in partd: if int(part['version'])>maxversion: maxversion = int(part['version']) for part in partd: if int(part['version'])==maxversion: numparticles+=1 file.write('version_id ' + str(maxversion) + '\n') file.write('ncontours ' + str(numparticles) + '\n') pointq = appiondata.ApContourPointData() for part in partd: if int(part['version'])==maxversion: # file.write('contour_number ' + ) file.write('method_used ' + part['method'] + ' ') pointq['contour'] = part pointd = pointq.query() for point in pointd: file.write(str(point['x']) + ',' + str(point['y']) + ';') file.write('\n') file.write('END_IMAGE' + '\n')
def start(self): ### universal particle counter self.partnum = 1 ### final stack file self.combinefile = os.path.join( self.params['rundir'], self.params['stackfilename'] ) if os.path.isfile(self.combinefile): apDisplay.printError("A stack with name "+self.params['stackfilename']+" and path " +self.params['rundir']+" already exists.") ### loop through stacks for stackstr in self.params['stackids']: stackid = int(stackstr) ### get stack data stackdata = apStack.getOnlyStackData(stackid) ### append particle to stack file self.appendToStack(stackdata) if self.params['commit'] is True: ### insert stack data apDisplay.printColor("inserting new stack particles from stackid="+str(stackid), "cyan") self.commitStack(stackid) else: apDisplay.printWarning("not committing data to database") apStack.averageStack(stack=self.combinefile)
def averageSubStack(partlist, stackfile, bin=1): if len(partlist) > 300: partlist = partlist[:300] boxsize = apImagicFile.getBoxsize(stackfile) if len(partlist) == 0: binboxsize = boxsize/bin blank = numpy.ones((binboxsize, binboxsize), dtype=numpy.float32) return blank if not os.path.isfile(stackfile): apDisplay.printWarning("could not find stack, "+stackfile) return False partdatalist = apImagicFile.readParticleListFromStack(stackfile, partlist, boxsize, msg=False) partdataarray = numpy.asarray(partdatalist) finaldata = partdataarray.mean(0) if bin > 1: finaldata = apImage.binImg(finaldata, bin) return finaldata
def _initializeDoneDict(self): """ reads or creates a done dictionary """ self.donedictfile = os.path.join(self.params['rundir'], self.functionname + ".donedict") if os.path.isfile( self.donedictfile) and self.params['continue'] == True: ### unpickle previously done dictionary apDisplay.printMsg("Reading old done dictionary: " + os.path.basename(self.donedictfile)) f = open(self.donedictfile, 'r') self.donedict = cPickle.load(f) f.close() try: if self.donedict['commit'] == self.params['commit']: ### all is well apDisplay.printMsg("Found " + str(len(self.donedict)) + " done dictionary entries") return elif self.donedict['commit'] is True and self.params[ 'commit'] is not True: ### die apDisplay.printError( "Commit flag was enabled and is now disabled, create a new runname" ) else: ### set up fresh dictionary apDisplay.printWarning( "'--commit' flag was changed, creating new done dictionary" ) except KeyError: apDisplay.printMsg("Found " + str(len(self.donedict)) + " done dictionary entries") ### set up fresh dictionary self.donedict = {} self.donedict['commit'] = self.params['commit'] apDisplay.printMsg("Creating new done dictionary: " + os.path.basename(self.donedictfile)) ### write donedict to file f = open(self.donedictfile, 'w', 0666) cPickle.dump(self.donedict, f) f.close() return
def insertCoranRun(self, insert=False): # create a AlignAnalysisRun object analysisq = appiondata.ApAlignAnalysisRunData() analysisq['runname'] = self.params['runname'] analysisq['path'] = appiondata.ApPathData( path=os.path.abspath(self.params['rundir'])) # ... path makes the run unique: uniquerun = analysisq.query(results=1) if uniquerun and insert is True: apDisplay.printError("Run name '"+self.params['runname']+"' for align stack id="+\ str(self.params['alignstackid'])+"\nis already in the database") coranq = appiondata.ApCoranRunData() coranq['num_factors'] = self.params['numfactors'] coranq['mask_diam'] = 2.0 * self.params['maskrad'] coranq['run_seconds'] = self.runtime # finish AlignAnalysisRun object analysisq['description'] = self.params['description'] analysisq['alignstack'] = self.alignstackdata analysisq['hidden'] = False analysisq['coranrun'] = coranq apDisplay.printMsg( "inserting Align Analysis Run parameters into database") if insert is True: analysisq.insert() ### eigen data for i in range(self.params['numfactors']): factnum = i + 1 eigenq = appiondata.ApCoranEigenImageData() eigenq['coranRun'] = coranq eigenq['factor_num'] = factnum path = os.path.join(self.params['rundir'], "coran") eigenq['path'] = appiondata.ApPathData(path=os.path.abspath(path)) imgname = ("eigenimg%02d.png" % (factnum)) eigenq['image_name'] = imgname if not os.path.isfile(os.path.join(path, imgname)): apDisplay.printWarning(imgname + " does not exist") continue if insert is True: eigenq['percent_contrib'] = self.contriblist[i] eigenq.insert() return
def checkConflicts(self): super(MakeDDParticleMovieLoop, self).checkConflicts() if self.params['bin'] != 1: apDisplay.printError( 'binning is not yet implemented, please use bin=1') #if self.params['denoise'] and not self.params['ddstack']: # apDisplay.printError('denoise only works with ddstack') if self.params['denoise'] and self.params['framestep'] > 1: apDisplay.printWarning( 'Interval of frames must be one when denoising') apDisplay.printWarning('Forcing it to 1....') self.params['framestep'] = 1 if not self.params['nframe']: # force nframe to a large member so that checkIsDD will consider it as dd data # This number will limit the total processing frames if smaller than # the actual number of frames in the images self.params['nframe'] = 1000
def moveStack(filename1, filename2, warn=True): ### replace one imagic stack with another rootname1 = os.path.splitext(filename1)[0] rootname2 = os.path.splitext(filename2)[0] for ext in (".hed", ".img"): inf = rootname1 + ext outf = rootname2 + ext if os.path.isfile(inf): if warn is True: apDisplay.printWarning("replacing stack file '%s' with '%s' " % (outf, inf)) time.sleep(1) try: shutil.move(inf, outf) except: apDisplay.printWarning('%s could not be replaced with %s' % (outf, inf))
def totalLeastSquares(X, Y, W=None, epsilon=1e-5, maxiter=500): """ iterative refine weights of observations, de-emphasizing bad fitting points """ t0 = time.time() ### check the input if checkMatrixSizes(X, Y) is False: return None ### setup weights if necessary if W is None: #W = numpy.random.random((m)) # weights for the observations #W = W*float(M)/W.sum() W = numpy.ones(Y.shape) # even weights for the observations ### solve it err0 = None sys.stderr.write("running total least squares") for i in range(maxiter): sys.stderr.write(".") beta = weightedLeastSquares(X, Y, W) if beta is None: if i < 2: return None beta = weightedLeastSquaresSVD(X, Y, W) ## calculate the absolute mean error err = numpy.absolute(numpy.dot(X, beta) - Y).ravel() #print "totalLeastSquares iter %d error: %.4f"%(i, err.mean()) ## fit to a normal distribution normerr = (err - err.min()) / err.std() ## calculate new weights based on W = numpy.exp(-1 * normerr**2) if W.sum() == 0: apDisplay.printWarning("Failed to set weights") return beta ## see if we can stop if err0 is not None: change = numpy.absolute(err - err0).mean() if change < epsilon: break err0 = err apDisplay.printMsg("totalLeastSquares completed in %s in %d iterations" % (apDisplay.timeString(time.time() - t0), i)) return beta
def getTargets(self, imgdata, scratchdir='', handlefiles='direct'): targetdict={} #copy flatfields brightrefpath=imgdata['bright']['session']['image path'] brightrefname=imgdata['bright']['filename']+'.mrc' brightref=os.path.join(brightrefpath,brightrefname) darkrefpath=imgdata['dark']['session']['image path'] darkrefname=imgdata['dark']['filename']+'.mrc' darkref=os.path.join(darkrefpath,darkrefname) #################################### do away with override flatfield option framesdirname=imgdata['filename']+'.frames' apDisplay.printMsg('Copying frames %s' % (framesdirname)) framespath=imgdata['session']['frame path'] framespathname=os.path.join(framespath,framesdirname) if handlefiles == 'direct': targetdict['brightref']=brightref targetdict['darkref']=darkref targetdict['framespathname']=framespathname targetdict['outpath']=self.params['rundir'] elif handlefiles == 'copy': shutil.copy(brightref,scratchdir) shutil.copy(darkref,scratchdir) targetdict['brightref']=os.path.join(scratchdir,brightrefname) targetdict['darkref']=os.path.join(scratchdir,darkrefname) try: shutil.copytree(framespathname,os.path.join(scratchdir, framesdirname)) except: apDisplay.printWarning('there was a problem copying the frames for %s' % (imgdata['filename'])) targetdict['framespathname']=os.path.join(scratchdir,framesdirname) targetdict['outpath']=os.path.join(scratchdir,imgdata['filename']) elif handlefiles == 'link': os.symlink(brightref,os.path.join(scratchdir,brightrefname)) os.symlink(darkref,os.path.join(scratchdir,darkrefname)) os.symlink(framespathname,os.path.join(scratchdir, framesdirname)) targetdict['brightref']=os.path.join(scratchdir,brightrefname) targetdict['darkref']=os.path.join(scratchdir,darkrefname) targetdict['framespathname']=os.path.join(scratchdir,framesdirname) targetdict['outpath']=os.path.join(scratchdir,imgdata['filename']) return targetdict
def start(self): ### get volume files volumefiles = glob.glob(self.params['volumes']) if not volumefiles: apDisplay.printError("Could not find volumes, %s" % (self.params['volumes'])) ### make list of all alignments to run cmdlist = [] alignfiles = [] for volfile in volumefiles: alignfile = "align-" + os.path.basename(volfile) alignfiles.append(alignfile) emancmd = "align3d %s %s %s " % (self.params['reference'], volfile, alignfile) if self.params['slow'] is True: emancmd += "slow " if self.params['noshrink'] is True: emancmd += "noshrink " #print emancmd cmdlist.append(emancmd) ### run several alignment commands in parallel apThread.threadCommands(cmdlist) ### average volumes together ref = mrc.read(self.params['reference']) average = numpy.zeros(ref.shape, dtype=numpy.float32) del ref count = 0 for alignfile in alignfiles: if not os.path.isfile(alignfile): apDisplay.printWarning("aligned volume not found: %s" % (alignfile)) aligned = mrc.read(alignfile) count += 1 ### this assume all aligned volume have same box size average += aligned del aligned ### save average average /= count avgfile = os.path.abspath(self.params['average']) mrc.write(average, avgfile) apDisplay.printMsg("Wrote average file: " + avgfile)
def runCTFdisplayTools(imgdata, ctfvalues, opimagedir, fftpath=None, fftfreq=None): ### RUN CTF DISPLAY TOOLS ctfdisplaydict = ctfdisplay.makeCtfImages(imgdata, ctfvalues, fftpath, fftfreq) if ctfdisplaydict is None: return ctfvalues ### save the classic images as well if 'graph1' in ctfvalues: ctfvalues['graph3'] = os.path.basename(ctfvalues['graph1']) if 'graph2' in ctfvalues: ctfvalues['graph4'] = os.path.basename(ctfvalues['graph2']) ### new powerspec file psfile = os.path.join(opimagedir, ctfdisplaydict['powerspecfile']) if not os.path.isfile(ctfdisplaydict['powerspecfile']): apDisplay.printWarning("Powerspec file not created") else: print ctfdisplaydict['powerspecfile'] shutil.move(ctfdisplaydict['powerspecfile'], psfile) ctfvalues['graph1'] = os.path.basename(psfile) ### new 1d plot file plotfile = os.path.join(opimagedir, ctfdisplaydict['plotsfile']) shutil.move(ctfdisplaydict['plotsfile'], plotfile) ctfvalues['graph2'] = os.path.basename(plotfile) ctfvalues['confidence_30_10'] = ctfdisplaydict['conf3010'] ctfvalues['confidence_5_peak'] = ctfdisplaydict['conf5peak'] ctfvalues['overfocus_conf_30_10'] = ctfdisplaydict['overconf3010'] ctfvalues['overfocus_conf_5_peak'] = ctfdisplaydict['overconf5peak'] ctfvalues['resolution_80_percent'] = ctfdisplaydict['res80'] ctfvalues['resolution_50_percent'] = ctfdisplaydict['res50'] if not 'confidence_d' in ctfvalues or ctfvalues['confidence_d'] is None: ctfvalues['confidence_d'] = ctfdisplaydict['conf5peak'] if not 'confidence' in ctfvalues or ctfvalues['confidence'] is None: ctfvalues['confidence'] = ctfdisplaydict['conf3010'] ### override the confidence ctfvalues['confidence'] = max(ctfvalues['confidence'], ctfvalues['confidence_d'], ctfdisplaydict['conf5peak'], ctfdisplaydict['conf3010']) return ctfvalues
def _removeProcessedImages(self): startlen = len(self.imgtree) donecount = 0 reproccount = 0 rejectcount = 0 tiltcount = 0 self.stats['skipcount'] = 0 newimgtree = [] count = 0 t0 = time.time() for imgdata in self.imgtree: count += 1 if count % 10 == 0: sys.stderr.write(".") skip, reason = self.skipTestOnImage(imgdata) imgname = imgdata['filename'] if skip is True: if reason == 'reproc': reproccount += 1 elif reason == 'reject' or reason == 'tilt': self._writeDoneDict(imgname) rejectcount += 1 if skip is True: if self.stats['skipcount'] == 0: sys.stderr.write("skipping processed images\n") elif self.stats['skipcount'] % 80 == 0: sys.stderr.write(".\n") else: sys.stderr.write(".") self.stats['skipcount'] += 1 else: newimgtree.append(imgdata) sys.stderr.write("\n") apDisplay.printMsg("finished skipping in %s" % (apDisplay.timeString(time.time() - t0))) if self.stats['skipcount'] > 0: self.imgtree = newimgtree sys.stderr.write("\n") apDisplay.printWarning("skipped " + str(self.stats['skipcount']) + " of " + str(startlen) + " images") apDisplay.printMsg("[[ " + str(reproccount) + " no reprocess " + " | " + str(rejectcount) + " rejected " + " | " + str(tiltcount) + " wrong tilt " + " | " + str(donecount) + " in donedict ]]")
def checkMPI(self): mpiexe = apParam.getExecPath("mpirun") if mpiexe is None: return None xmippexe = apParam.getExecPath("xmipp_mpi_ml_refine3d") if xmippexe is None: return None lddcmd = "ldd "+xmippexe+" | grep mpi" proc = subprocess.Popen(lddcmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc.wait() lines = proc.stdout.readlines() #print "lines=", lines if lines and len(lines) > 0: return mpiexe else: apDisplay.printWarning("Failed to find mpirun") print "lines=", lines return None
def getResolutionData(self, iternum): fscfile = 'fsc.eotest.%d'%(iternum) fscpath = os.path.join(self.params['rundir'], "iter%03d"%(iternum), fscfile) if not os.path.isfile(fscpath): apDisplay.printWarning("Could not find FSC file: "+fscpath) return None # calculate the resolution: halfres = apRecon.calcRes(fscpath, self.boxsize, self.apix) apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms"%(halfres), "cyan") # save to database resq=appiondata.ApResolutionData() resq['half'] = halfres resq['fscfile'] = fscfile return resq
def getRMeasureData(self, iteration): volumeDensity='threed.'+iteration['num']+'a.mrc' volPath = os.path.join(self.params['rundir'], volumeDensity) if not os.path.exists(volPath): apDisplay.printWarning("R Measure failed, volume density not found: "+volPath) return None resolution = apRecon.runRMeasure(self.params['apix'], volPath) if resolution is None: return None rmesq=appiondata.ApRMeasureData() rmesq['volume']=volumeDensity rmesq['rMeasure']=resolution return rmesq
def processParticle(self, partarray): if self.filetype == "mrc": partfile = os.path.join(self.partdir, "part%06d.mrc" % (self.index)) mrc.write(partarray, partfile) else: partfile = os.path.join(self.partdir, "part%06d.spi" % (self.index)) try: spider.write(partarray, partfile) except: print partarray apDisplay.printWarning( "failed to write spider file for part index %d" % (self.index)) print partarray.shape spider.write(partarray, partfile) self.partdocf.write(os.path.abspath(partfile) + " 1\n")
def setupRunDirectory(self): """ Set the run directory """ if self.params['rundir'] is None: apDisplay.printWarning("run directory not defined, automatically setting it") self.setProcessingDirName() self.setRunDir() if self.params['rundir'] is None: apDisplay.printError("No run directory was set") if self.quiet is False: apDisplay.printMsg("Run directory: "+self.params['rundir']) if not os.path.isdir(self.params['rundir']): apDisplay.printError("run directory must exist for FileScript run") os.chdir(self.params['rundir'])
def checkRefineLogError(self): f = open(self.tasklogfile) text = f.read() f.close() lines = text.split('\n') f.close() tasklog_basename = os.path.basename(self.tasklogfile) # check alarm and error if 'ERROR' in text: self.failed = True apDisplay.printError('There are Errors in %s' % (tasklog_basename), False) if 'WARNING' in text: apDisplay.printWarning('There are Warning in %s during %s' % (tasklog_basename, self.task_noun)) for line in lines: if 'WARNING' in line: apDisplay.printWarning(line)
def getRgbFile(msg=True): """ This file comes with xorg-x11-server-Xorg in Fedora 7,8 missing in Fedora 9 """ filelist = [ "/usr/share/X11/rgb", "/usr/X11R6/lib64/X11/rgb", "/usr/X11R6/lib/X11/rgb", ] xversion = getXversion() if xversion is None or xversion > 1.0109: return " " for rgbfile in filelist: if os.path.isfile(rgbfile+".txt"): return " -co "+rgbfile apDisplay.printWarning("Xvfb: could not find RGB File") return " "
def _getRefImageData(self, reftype): imagedata = super(GatanK2Processing, self).getCorrectedImageData() if not self.use_full_raw_area: refdata = imagedata[reftype] if refdata is None: # Use chosen default image apDisplay.printWarning( 'No %s reference for the image, use default' % reftype) default_image = self.getCorrectedImageData() refdata = default_image[reftype] else: # use most recent CorrectorImageData # TO DO: this should research only ones before the image is taken. scopedata = self.image['scope'] channel = self.image['channel'] refdata = self.c_client.researchCorrectorImageData( reftype, scopedata, self.camerainfo, channel) return refdata
def checkExistingFile(self): savedtomopath = os.path.join(self.params['rundir'], self.params['name'] + ".rec") origtomopath = self.params['file'] apDisplay.printWarning( "A Tomogram by the same filename already exists: '" + savedtomopath + "'") ### a tomogram by the same name already exists savedheader = mrc.readHeaderFromFile(savedtomopath) savedshape = savedheader['shape'] origheader = self.origheader origshape = self.origshape order = self.params['order'] if self.params['full']: newshape = (self.imageshape[0], origshape[order.find('Z')], self.imageshape[1]) else: newshape = (origshape[order.find('X')], origshape[order.find('Y')], origshape[order.find('Z')]) # not using md5 for file comparison because it takes too long. With padding, only min and max are stable if newshape != savedshape or savedheader['amax'] != origheader[ 'amax'] or savedheader['amin'] != origheader['amin']: different = True else: different = False mdnew = None if different: apDisplay.printWarning( "The tomograms are different, cannot overwrite") return True elif apDatabase.isTomoInDB(mdnew, self.params['full'], savedtomopath): ### they are the same and its in the database already apDisplay.printWarning( "Identical Tomogram already exists in the database!") apDisplay.printWarning("Upload Not Allowed") self.params['commit'] = False return True else: ### they are the same, but it is not in the database apDisplay.printWarning("The same tomogram with name '" + savedtomopath + "' already exists, but is not uploaded!") if self.params['commit'] is True: apDisplay.printMsg("Inserting tomogram into database...")
def readData(filename, filetype=None): """ savedata = {} savedata['theta'] = self.data['theta'] savedata['gamma'] = self.data['gamma'] savedata['phi'] = self.data['phi'] savedata['shiftx'] = self.data['shiftx'] savedata['shifty'] = self.data['shifty'] savedata['savetime'] = time.asctime()+" "+time.tzname[time.daylight] savedata['filetype'] = tiltfile.filetypes[self.data['filetypeindex']] savedata['picks1'] = self.getArray1() savedata['picks2'] = self.getArray2() savedata['align1'] = self.getAlignedArray1() savedata['align2'] = self.getAlignedArray2() savedata['rmsd'] = self.getRmsdArray() savedata['filepath'] = os.path.join(self.data['dirname'], self.data['outfile']) savedata['image1name'] = self.panel1.filename savedata['image2name'] = self.panel2.filename """ if not os.path.isfile(filename): apDisplay.printWarning("file " + filename + " does not exist") return None if filetype is None: filetype = guessFileType(filename) if filetype == 'text': savedata = readFromTextFile(filename) elif filetype == 'xml': savedata = readFromXMLFile(filename) elif filetype == 'spider': savedata = readFromSpiderFile(filename) elif filetype == 'pickle': savedata = readFromPickleFile(filename) else: apDisplay.printWarning("unknown file type") return None savedata['filename'] = os.path.basename(filename) savedata['loadtime'] = time.asctime() + " " + time.tzname[time.daylight] savedata['shiftx'] = 0.0 savedata['shifty'] = 0.0 savedata['scale'] = 1.0 return savedata
def getBestCtfValue(imgdata, sortType='res80', method=None, msg=True): """ takes an image and get the best ctfvalues for that image """ ### get all ctf values ctfq = appiondata.ApCtfData() ctfq['image'] = imgdata ctfvalues = ctfq.query() imgname = apDisplay.short(imgdata['filename']) if msg is True: print "Found %d ctf values"%(len(ctfvalues)) ### check if it has values if ctfvalues is None: apDisplay.printWarning("no CTF values found in database for img %s"%(imgname)) return None ### find the best values bestsortvalue = -1 bestctfvalue = None for ctfvalue in ctfvalues: if method is not None: imgmethod = getCtfMethod(ctfvalue) if method != imgmethod: continue sortvalue = getSortValueFromCtfQuery(ctfvalue, sortType) if sortvalue is None: continue if msg is True: print "%.3f -- %s"%(sortvalue, ctfvalue['acerun']['name']) if sortvalue > bestsortvalue: bestsortvalue = sortvalue bestctfvalue = ctfvalue if bestctfvalue is None: apDisplay.printWarning("no best CTF value for image %s"%(imgname)) return None if msg is True: print "*** %.3f"%(bestsortvalue) printCtfData(bestctfvalue) return bestctfvalue
def checkIsDD(self): apDisplay.printWarning('Checking for dd') if self.params['ddstack'] > 0: self.is_dd_stack = True self.is_dd = True else: if self.params['preset'] and '-a' in self.params['preset'] and ( self.params['nframe'] or self.params['driftlimit'] > 0): self.is_dd = True self.is_dd_stack = True elif self.params['mrcnames'] and self.params['mrcnames'].split( ',')[0] and '-a' in self.params['mrcnames'].split( ',')[0] and (self.params['nframe'] or self.params['driftlimit'] > 0): self.is_dd = True self.is_dd_stack = True elif self.params['nframe']: self.is_dd = True self.is_dd_frame = True
def getRMeasurePath(): unames = os.uname() if unames[-1].find('64') >= 0: exename = 'rmeasure64.exe' else: exename = 'rmeasure32.exe' rmeasexe = subprocess.Popen("which "+exename, shell=True, stdout=subprocess.PIPE).stdout.read().strip() if not os.path.isfile(rmeasexe): rmeasexe = os.path.join(apParam.getAppionDirectory(), 'bin', exename) if not os.path.isfile(rmeasexe): exename = "rmeasure.exe" rmeasexe = subprocess.Popen("which "+exename, shell=True, stdout=subprocess.PIPE).stdout.read().strip() if not os.path.isfile(rmeasexe): exename = "rmeasure" rmeasexe = subprocess.Popen("which "+exename, shell=True, stdout=subprocess.PIPE).stdout.read().strip() if not os.path.isfile(rmeasexe): apDisplay.printWarning(exename+" was not found at: "+apParam.getAppionDirectory()) return None return rmeasexe
def checkConflicts(self): # Check that the program needed exists gpuexelist = ['dosefgpu_driftcorr'] exename = 'dosefgpu_driftcorr' gpuexe = subprocess.Popen( "which " + exename, shell=True, stdout=subprocess.PIPE).stdout.read().strip() if not os.path.isfile(gpuexe): apDisplay.printError('Correction program "%s" not available' % exename) # We don't have gpu locking if self.params['parallel']: apDisplay.printWarning( 'Make sure that you use different gpuid for each parallel process' ) if self.params['stackid'] and not self.params['ddstack']: apDisplay.printError( 'Specify stack alone does not work. Use makeDDRawFrameStack.py instead' )
def highPassFilter(imgarray, apix=1.0, bin=1, radius=0.0, localbin=8, msg=True): """ high pass filter image to radius resolution """ if radius is None or radius < 1 or imgarray.shape[0] < 256: if msg is True: apDisplay.printMsg("skipping high pass filter") return(imgarray) try: bimgarray = binImg(imgarray, localbin) sigma=float(radius/apix/float(bin*localbin)) filtimg = ndimage.gaussian_filter(bimgarray, sigma=sigma) expandimg = scaleImage(filtimg, localbin) expandimg = frame_constant(expandimg, imgarray.shape) filtimg = imgarray - expandimg except: apDisplay.printWarning("High Pass Filter failed") return imgarray return filtimg
def insertClusterRun(self): ### create a clustering run object clusterrunq = appiondata.ApClusteringRunData() clusterrunq['runname'] = self.params['runname'] clusterrunq['pixelsize'] = self.params['apix'] clusterrunq['boxsize'] = self.params['boxsize'] clusterrunq['description'] = self.params['description'] clusterrunq['num_particles'] = self.params['num_particles'] clusterrunq['alignstack'] = self.analysisdata['alignstack'] clusterrunq['analysisrun'] = self.analysisdata apDisplay.printMsg("inserting clustering parameters into database") if self.params['commit'] is True: clusterrunq.insert() else: apDisplay.printWarning("not committing results to DB") self.clusterrun = clusterrunq return
def weightedLeastSquares(X, Y, W): """ solve using the normal equations with no manipulation """ t0 = time.time() ### check the input if checkMatrixSizes(X, Y) is False: return None ### solve it XTW = numpy.transpose(X) * W XTWX = numpy.dot(XTW, X) if numpy.linalg.det(XTWX) == 0: apDisplay.printWarning("Singular matrix in calculation") return None XTWXinv = numpy.linalg.inv(XTWX) beta = numpy.dot(numpy.dot(XTWXinv, XTW), Y) #apDisplay.printMsg("weightedLeastSquares completed in %s" # %(apDisplay.timeString(time.time()-t0))) return beta
def convertDefociToConvention(self, ctfdata): ctfdb.printCtfData(ctfdata) initdefocusratio = ctfdata['defocus2']/ctfdata['defocus1'] # program specific corrections? self.angle = ctfdata['angle_astigmatism'] #angle = round(self.angle/2.5,0)*2.5 #by convention: abs(ctfdata['defocus1']) < abs(ctfdata['defocus2']) if abs(ctfdata['defocus1']) > abs(ctfdata['defocus2']): # incorrect, need to shift angle by 90 degrees apDisplay.printWarning("|def1| > |def2|, flipping defocus axes") self.defocus1 = ctfdata['defocus2'] self.defocus2 = ctfdata['defocus1'] self.angle += 90 else: # correct, ratio > 1 self.defocus1 = ctfdata['defocus1'] self.defocus2 = ctfdata['defocus2'] if self.defocus1 < 0 and self.defocus2 < 0: apDisplay.printWarning("Negative defocus values, taking absolute value") self.defocus1 = abs(self.defocus1) self.defocus2 = abs(self.defocus2) self.defdiff = self.defocus1 - self.defocus2 #elliptical ratio is ratio of zero locations NOT defocii self.defocusratio = self.defocus2/self.defocus1 self.ellipratio = ctftools.defocusRatioToEllipseRatio(self.defocus1, self.defocus2, self.initfreq, self.cs, self.volts, self.ampcontrast) # get angle within range -90 < angle <= 90 while self.angle > 90: self.angle -= 180 while self.angle < -90: self.angle += 180 apDisplay.printColor("Final params: def1: %.2e | def2: %.2e | angle: %.1f | defratio %.2f"% (self.defocus1, self.defocus2, self.angle, self.defocusratio), "cyan") perdiff = abs(self.defocus1-self.defocus2)/abs(self.defocus1+self.defocus2) apDisplay.printMsg("Defocus Astig Percent Diff %.2f -- %.3e, %.3e" %(perdiff*100,self.defocus1,self.defocus2)) return
def start(self): self.setFileName() scale = float(self.params['oldapix']) / self.params['newapix'] mrcname = os.path.join(self.params['rundir'], self.params['name'] + ".mrc") origmodel = self.params['file'] if os.path.isfile(mrcname): apDisplay.printError("File exists") if (abs(self.params['oldapix'] - self.params['newapix']) > 1.0e-2 or abs(self.params['oldbox'] - self.params['newbox']) > 1.0e-1): ### rescale old model to a new size apDisplay.printWarning("rescaling original model to a new size") scale = float(self.params['oldapix']) / self.params['newapix'] apDisplay.printMsg("rescaling model by scale factor of %.4f" % (scale)) apVolume.rescaleVolume(origmodel, mrcname, self.params['oldapix'], self.params['newapix'], self.params['newbox']) else: ### simple upload, just copy file to models folder apDisplay.printMsg("copying original model to a new location: " + mrcname) shutil.copyfile(origmodel, mrcname) if self.params['viper2eman'] is True: apVolume.viper2eman(mrcname, mrcname, apix=self.params['newapix']) ### render chimera images of model contour = self.params['contour'] if self.params['mass'] is not None: apChimera.setVolumeMass(mrcname, self.params['newapix'], self.params['mass']) contour = 1.0 apChimera.renderSnapshots(mrcname, contour=contour, zoom=self.params['zoom'], sym=self.params['symdata']['eman_name']) self.insertModel(mrcname)
def makeFilesForETomoSampleRecon(processdir, stackdir, aligndir, templatedir, seriesname, thickness, pixelsize, yspacing, has_rotation=False): ''' Make or link local files required by etomo to redo sampling, creating tomopitch model, and reconstruct the volume. ''' # etomo status file createETomoBoundaryModelEDF(processdir, templatedir, seriesname, thickness, pixelsize) # required by remaking sample tomograms inside etomo if has_rotation: apDisplay.printWarning( 'eTomo wants to regenerate global alignment from non-rotated local alignment. This alignment with rotation will not work right' ) else: # prexf file is generated from database values if this function is called from tomomaker. It is better not to change it if exists. prexf = seriesname + ".prexf" alignprexf = os.path.join(aligndir, prexf) localprexf = os.path.join(processdir, prexf) apFile.safeCopy(alignprexf, localprexf) writeETomoNewstComTemplate(processdir, seriesname) rawtltname = '%s.rawtlt' % (seriesname) shutil.copy(os.path.join(stackdir, rawtltname), os.path.join(processdir, rawtltname)) # required by tomopitch model making tomopitchname = 'tomopitch.com' imodcomdir = os.path.join(os.environ['IMOD_DIR'], 'com') tomopitchpath = os.path.join(processdir, tomopitchname) shutil.copy(os.path.join(imodcomdir, tomopitchname), tomopitchpath) apFile.replaceUniqueLinePatternInTxtFile(tomopitchpath, 'SpacingInY', 'SpacingInY\t%.1f\n' % yspacing) # required by "Final Aligned Stack" step stackname = '%s.st' % (seriesname) apFile.safeSymLink(os.path.join(stackdir, stackname), os.path.join(processdir, stackname))