def createChimeraVolumeSnapshot(self, volume, iteration, reference_number=1):
""" create chimera snapshot images of volume """
if self.params["snapfilter"]:
resolution = self.params["snapfilter"]
else:
newfscfile = os.path.join(
self.resultspath,
"recon_%s_it%.3d_vol%.3d.fsc" % (self.params["timestamp"], iteration, reference_number),
)
try:
resolution = apRecon.getResolutionFromGenericFSCFile(
newfscfile, self.runparams["boxsize"], self.runparams["apix"]
)
except:
apDisplay.printWarning("Failed to get resolution from generic FSC file: " + newfscfile)
resolution = 30
apDisplay.printWarning("Running Chimera Snapshot with resolution: %d " % resolution)
apChimera.filterAndChimera(
volume,
resolution,
self.runparams["apix"],
self.runparams["boxsize"],
"snapshot",
self.params["contour"],
self.params["zoom"],
sym="c1",
mass=self.params["mass"],
)
def checkConflicts(self):
if self.params['stackid'] is None:
apDisplay.printError("stack id was not defined")
if self.params['rad'] is None:
apDisplay.printError("no particle radius set")
if self.params['modelid'] is None:
apDisplay.printError("model id was not defined")
if self.params['lastring'] is None:
apDisplay.printError("a last ring radius was not provided")
if self.params['runname'] is None:
apDisplay.printError("run name was not defined")
if self.params['incr'] is None:
apDisplay.printError("angular increments are not specified")
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
numstackp = apFile.numImagesInStack(stackfile)
if self.params['numpart'] > numstackp:
apDisplay.printError("trying to use more particles "+str(numstackp)
+" than available "+str(numstackp))
elif self.params['numpart'] == 0:
apDisplay.printWarning("using all "+str(numstackp)+" particles")
self.params['numpart'] = numstackp
boxsize = apStack.getStackBoxsize(self.params['stackid'])
if self.params['lastring'] > boxsize/2-2:
apDisplay.printError("last ring radius is too big for boxsize "
+str(self.params['lastring'])+" > "+str(boxsize/2-2))
if self.params['lastring']+self.params['xysearch'] > boxsize/2-2:
apDisplay.printError("last ring plus xysearch radius is too big for boxsize "
+str(self.params['lastring']+self.params['xysearch'])+" > "+str(boxsize/2-2))
if (self.params['xysearch'] % self.params['xystep']) > 0:
apDisplay.printError("translational search (xy-search) must be divisible by search step (xy-step)")
def _initializeDoneDict(self):
"""
reads or creates a done dictionary
"""
self.donedictfile = os.path.join(self.params['rundir'] , self.functionname+".donedict")
if os.path.isfile(self.donedictfile) and self.params['continue'] == True:
### unpickle previously done dictionary
apDisplay.printMsg("Reading old done dictionary: "+os.path.basename(self.donedictfile))
f = open(self.donedictfile,'r')
self.donedict = cPickle.load(f)
f.close()
if not 'commit' in self.donedict or self.donedict['commit'] == self.params['commit']:
### all is well
apDisplay.printMsg("Found "+str(len(self.donedict))+" done dictionary entries")
return
elif self.donedict['commit'] is True and self.params['commit'] is not True:
### die
apDisplay.printError("Commit flag was enabled and is now disabled, create a new runname")
else:
### set up fresh dictionary
apDisplay.printWarning("'--commit' flag was changed, creating new done dictionary")
### set up fresh dictionary
self.donedict = {}
self.donedict['commit'] = self.params['commit']
apDisplay.printMsg("Creating new done dictionary: "+os.path.basename(self.donedictfile))
### write donedict to file
f = open(self.donedictfile, 'w', 0666)
cPickle.dump(self.donedict, f)
f.close()
return
def runChimeraScript(chimscript, xvfb=False):
if not chimscript or not os.path.isfile(chimscript):
print chimscript
apDisplay.printError("Could not find file: apChimSnapshot.py")
#apDisplay.printColor("Trying to use chimera for model imaging","cyan")
if xvfb is True:
port = apParam.resetVirtualFrameBuffer()
time.sleep(1)
if 'CHIMERA' in os.environ and os.path.isdir(os.environ['CHIMERA']):
chimpath = os.environ['CHIMERA']
os.environ['CHIMERA'] = chimpath
os.environ['CHIMERAPATH'] = os.path.join(chimpath,"share")
os.environ['LD_LIBRARY_PATH'] = os.path.join(chimpath,"lib")+":"+os.environ['LD_LIBRARY_PATH']
chimexe = os.path.join(chimpath,"bin/chimera")
if not os.path.isfile(chimexe):
apDisplay.printWarning("Could not find chimera at: "+chimexe)
else:
chimpath = None
chimexe = "chimera"
#apDisplay.printWarning("'CHIMERA' environmental variable is unset")
rendercmd = (chimexe+" --debug python:"+chimscript)
logf = open("chimeraRun.log", "a")
apDisplay.printColor("running Chimera:\n "+rendercmd, "cyan")
if xvfb is True:
print "import -verbose -display :%d -window root screencapture.png"%(port)
proc = subprocess.Popen(rendercmd, shell=True, stdout=logf, stderr=logf)
proc.wait()
logf.close()
if xvfb is True:
apParam.killVirtualFrameBuffer(port)
return
def start(self):
"""
this is the main component of the script
where all the processing is done
"""
### initialize some variables
self.runq = None
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
apDisplay.printMsg("Pixel size: %.5f"%(self.apix))
self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
apDisplay.printMsg("Box size: %d"%(self.boxsize))
self.checkResults()
self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
self.numiter = self.getNumberOfIterations()
for i in range(self.numiter):
iternum = i+1
apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green")
self.uploadIteration(iternum)
reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir'])
if reconrunid:
apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid'])
apRecon.setGoodBadParticlesFromReconId(reconrunid)
else:
apDisplay.printWarning("Could not find recon run id")
def start(self):
self.setFileName()
scale = float(self.params['oldapix'])/self.params['newapix']
mrcname = os.path.join(self.params['rundir'], self.params['name']+".mrc")
origmodel = self.params['file']
if os.path.isfile(mrcname):
apDisplay.printError("File exists")
if (abs(self.params['oldapix'] - self.params['newapix']) > 1.0e-2 or
abs(self.params['oldbox'] - self.params['newbox']) > 1.0e-1):
### rescale old model to a new size
apDisplay.printWarning("rescaling original model to a new size")
scale = float(self.params['oldapix'])/self.params['newapix']
apDisplay.printMsg("rescaling model by scale factor of %.4f"%(scale))
apVolume.rescaleVolume(origmodel, mrcname,
self.params['oldapix'], self.params['newapix'], self.params['newbox'])
else:
### simple upload, just copy file to models folder
apDisplay.printMsg("copying original model to a new location: "+mrcname)
shutil.copyfile(origmodel, mrcname)
if self.params['viper2eman'] is True:
apVolume.viper2eman(mrcname, mrcname, apix=self.params['newapix'])
### render chimera images of model
contour = self.params['contour']
if self.params['mass'] is not None:
apChimera.setVolumeMass(mrcname, self.params['newapix'], self.params['mass'])
contour = 1.0
apChimera.renderSnapshots(mrcname, contour=contour,
zoom=self.params['zoom'], sym=self.params['symdata']['eman_name'])
self.insertModel(mrcname)
def fixAmpContrast(self, defocus, raddata, PSDarray, lowerbound, upperbound):
if self.ctfvalues['amplitude_contrast'] > 0 and self.ctfvalues['amplitude_contrast'] < self.maxAmpCon:
return defocus
bad = True
newdefocus = defocus
count = 0
while count < 20:
count += 1
amplitudecontrast = sinefit.refineAmplitudeContrast(raddata[lowerbound:upperbound]*1e10, newdefocus,
PSDarray[lowerbound:upperbound], self.ctfvalues['cs'], self.wavelength, msg=self.debug)
if amplitudecontrast is None:
apDisplay.printWarning("FAILED to fix amplitude contrast")
return defocus
elif amplitudecontrast < 0:
apDisplay.printColor("Amp Cont: %.3f too small, decrease defocus %.3e"%
(amplitudecontrast, newdefocus), "blue")
scaleFactor = 0.999 - abs(amplitudecontrast)/5.
newdefocus = newdefocus*scaleFactor
elif amplitudecontrast > self.maxAmpCon:
apDisplay.printColor("Amp Cont: %.3f too large, increase defocus %.3e"%
(amplitudecontrast, newdefocus), "cyan")
scaleFactor = 1.001 + abs(amplitudecontrast - self.maxAmpCon)/5.
newdefocus = newdefocus*scaleFactor
else:
apDisplay.printColor("Amp Cont: %.3f in range!!! defocus %.3e"%
(amplitudecontrast, newdefocus), "green")
avgres = self.getResolution(newdefocus, raddata, PSDarray, lowerbound)
self.ctfvalues['amplitude_contrast'] = amplitudecontrast
return newdefocus
avgres = self.getResolution(newdefocus, raddata, PSDarray, lowerbound)
if avgres < self.bestres:
defocus = newdefocus
apDisplay.printWarning("FAILED to fix amplitude contrast")
return defocus
def getGenerationFiles(self):
if self.files is None:
files = glob.glob("class_averages_generation_*.hdf")
if len(files) == 0:
apDisplay.printWarning("Could not find class_averages_generation_*.hdf files.")
self.files = sorted(files, key=lambda a: self.generationFromFile(a))
return self.files
def insertFSCData(self, fscfile, refineIterData):
"""
current functions read the inverse pixel number and correlation value. This function, therefore, converts
generic FSC parameters into pixel / correlation format
"""
if not os.path.isfile(fscfile):
apDisplay.printWarning("Could not open FSC file: " + fscfile)
### skip commented out lines and read FSC data
f = open(fscfile, "r")
fscfileinfo = f.readlines()
f.close()
for i, info in enumerate(fscfileinfo):
if info[0] == "#":
pass
else:
fscfileinfo = fscfileinfo[i:]
break
### insert FSC data into database with the convention: pixel number, FSC
numinserts = 0
for j, info in enumerate(fscfileinfo):
fscq = appiondata.ApFSCData()
fscq["refineIter"] = refineIterData
fscq["pix"] = j + 1
fscq["value"] = float(info.split()[1])
numinserts += 1
if self.params["commit"] is True:
fscq.insert()
if self.params["commit"] is True:
apDisplay.printMsg("inserted " + str(numinserts) + " rows of FSC data into database")
return
def setVolumeMass(volumefile, apix=1.0, mass=1.0, rna=0.0):
"""
set the contour of 1.0 to the desired mass (in kDa) of the
macromolecule based on its density
use RNA to set the percentage of RNA in the structure
"""
if isValidVolume(volumefile) is False:
apDisplay.printError("Volume file is not valid")
procbin = apParam.getExecPath("proc2d")
emandir = os.path.dirname(procbin)
volumebin = os.path.join(emandir, "volume")
if not os.path.isfile(volumebin):
apDisplay.printWarning("failed to find volume program")
return False
command = "%s %s %.3f set=%.3f"%(
volumebin, volumefile, apix, mass
)
t0 = time.time()
proc = subprocess.Popen(command, shell=True)
proc.wait()
if time.time()-t0 < 0.01:
apDisplay.printWarning("failed to scale by mass in "+apDisplay.timeString(time.time()-t0))
return False
apDisplay.printMsg("finished scaling by mass in "+apDisplay.timeString(time.time()-t0))
return True
def checkSelOrDocFileRootDirectory(self, sel_doc_file, old_rootdir, new_rootdir):
''' necessary when files are transferred from one file system to another, e.g., the root directory on Garibaldi is different from that on Guppy'''
try:
f = open(sel_doc_file, "r")
except:
apDisplay.printWarning("Unable to open file: %s" % sel_doc_file)
return
lines = f.readlines()
newlines = []
f.close()
### replace old root directory with new root directory
count = 0
for line in lines:
if re.search(old_rootdir, line):
newline = re.sub(old_rootdir, new_rootdir, line)
count += 1
else:
newline = line
newlines.append(newline)
if count > 0:
apDisplay.printMsg("changing root directory from %s to %s in %s" % (old_rootdir, new_rootdir, sel_doc_file))
f = open(sel_doc_file, "w")
f.writelines(newlines)
f.close()
return
def start(self):
### determine which iterations to upload; last iter is defaulted to infinity
uploadIterations = self.verifyUploadIterations()
### upload each iteration
for iteration in uploadIterations:
for j in range(self.runparams['NumberOfReferences']):
### general error checking, these are the minimum files that are needed
vol = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.mrc" % (self.params['timestamp'], iteration, j+1))
particledatafile = os.path.join(self.resultspath, "particle_data_%s_it%.3d_vol%.3d.txt" % (self.params['timestamp'], iteration, j+1))
if not os.path.isfile(vol):
apDisplay.printError("you must have an mrc volume file in the 'external_package_results' directory")
if not os.path.isfile(particledatafile):
apDisplay.printError("you must have a particle data file in the 'external_package_results' directory")
### make chimera snapshot of volume
try:
self.createChimeraVolumeSnapshot(vol, iteration, j+1)
except:
apDisplay.printWarning("could not create Chimera volume snapshots")
### instantiate database objects
self.insertRefinementRunData(iteration, j+1)
self.insertRefinementIterationData(iteration, j+1)
### calculate Euler jumps
self.calculateEulerJumpsAndGoodBadParticles(uploadIterations)
def getBoxSize(filename, msg=False):
"""
return boxsize of stack in pixels
"""
if not os.path.isfile(filename):
if msg is True:
apDisplay.printWarning("file does not exist")
return (1,1,1)
proc = subprocess.Popen("iminfo %s"%(filename), shell=True, stdout=subprocess.PIPE)
proc.wait()
lines = ""
for line in proc.stdout:
sline = line.strip()
lines += line
m = re.match("^Image\(s\) are ([0-9]+)x([0-9]+)x([0-9]+)", sline)
if m and m.groups() and len(m.groups()) > 1:
xdim = int(m.groups()[0])
ydim = int(m.groups()[1])
zdim = int(m.groups()[2])
return (xdim,ydim,zdim)
m = re.match("^0\.\s+([0-9]+)x([0-9]+)\s+", sline)
if m and m.groups() and len(m.groups()) > 1:
xdim = int(m.groups()[0])
ydim = int(m.groups()[1])
return (xdim,ydim,1)
m = re.match("^0\.\s+([0-9]+)x([0-9]+)x([0-9]+)\s+", sline)
if m and m.groups() and len(m.groups()) > 1:
xdim = int(m.groups()[0])
ydim = int(m.groups()[1])
zdim = int(m.groups()[2])
return (xdim,ydim,zdim)
if msg is True:
apDisplay.printWarning("failed to get boxsize: "+lines)
return (1,1,1)
def readPartDocFile(self, reflist):
partlist = []
docfile = "part"+self.params['timestamp']+".doc"
if not os.path.isfile(docfile) or apFile.fileSize(docfile) < 50:
apDisplay.printWarning("Could not find doc file "+docfile+" to read particle angles")
lastdocfile = "iter%03d/part%s_it%06d.doc"%(self.lastiter, self.params['timestamp'], self.lastiter)
if not os.path.isfile(lastdocfile) or apFile.fileSize(lastdocfile) < 50:
apDisplay.printError("Could not find backup doc file")
apDisplay.printColor("Found a backup copy of docfile", "green")
shutil.copy(lastdocfile, docfile)
f = open(docfile, "r")
mininplane = 360.0
for line in f:
if line[:2] == ' ;':
continue
spidict = operations.spiderInLine(line)
origpartdict = self.spidict2partdict(spidict)
partdict = self.adjustPartDict(origpartdict, reflist)
if partdict['inplane'] < mininplane:
mininplane = partdict['inplane']
partlist.append(partdict)
apDisplay.printMsg("minimum inplane: "+str(mininplane))
for partdict in partlist:
partdict['inplane'] = partdict['inplane']-mininplane
apDisplay.printMsg("read rotation and shift parameters for "+str(len(partlist))+" particles")
if len(partlist) < 1:
apDisplay.printError("Did not find any particles in doc file: "+docfile)
return partlist
def importPicks(self, picks1, picks2, tight=False, msg=True):
t0 = time.time()
#print picks1
#print self.currentpicks1
curpicks1 = numpy.asarray(self.currentpicks1)
curpicks2 = numpy.asarray(self.currentpicks2)
#print curpicks1
# get picks
apTiltTransform.setPointsFromArrays(curpicks1, curpicks2, self.data)
pixdiam = self.data['pixdiam']
if tight is True:
pixdiam /= 4.0
#print self.data, pixdiam
list1, list2 = apTiltTransform.alignPicks2(picks1, picks2, self.data, limit=pixdiam, msg=msg)
if list1.shape[0] == 0 or list2.shape[0] == 0:
apDisplay.printWarning("No new picks were found")
# merge picks
newpicks1, newpicks2 = apTiltTransform.betterMergePicks(curpicks1, list1, curpicks2, list2, msg=msg)
newparts = newpicks1.shape[0] - curpicks1.shape[0]
# copy over picks
self.currentpicks1 = newpicks1
self.currentpicks2 = newpicks2
if msg is True:
apDisplay.printMsg("Inserted "+str(newparts)+" new particles in "+apDisplay.timeString(time.time()-t0))
return True
def readDictFromXml(filename):
if not os.path.isfile(filename):
apDisplay.printWarning("Failed to open file "+filename+" for reading")
return None
dom = xml.dom.minidom.parse(filename)
xmldict = nodeToDict(dom.childNodes[0])
return xmldict
def _startLoop(self, info):
"""
appionLoop OVERRIDE
initilizes several parameters for a new image
and checks if it is okay to start processing image
"""
if info is None:
self.stats['lastimageskipped'] = True
self.stats['skipcount'] += 1
return False
name = info['filename']
# check to see if image of the same name is already in leginon
imgq = leginon.leginondata.AcquisitionImageData(session=self.session, filename=name)
results = imgq.query(readimages=False)
if results:
apDisplay.printWarning("File %s.mrc exists at the destination" % name)
apDisplay.printWarning("Skip Uploading")
self.stats['lastimageskipped'] = True
self.stats['skipcount'] += 1
return False
#calc images left
self.stats['imagesleft'] = self.stats['imagecount'] - self.stats['count']
#only if an image was processed last
if(self.stats['lastcount'] != self.stats['count']):
apDisplay.printColor( "\nStarting image "+str(self.stats['count'])\
+" ( skip:"+str(self.stats['skipcount'])+", remain:"\
+str(self.stats['imagesleft'])+" ) file: "\
+apDisplay.short(name), "green")
self.stats['lastcount'] = self.stats['count']
self._checkMemLeak()
# check to see if image has already been processed
if self._alreadyProcessed(info):
return False
self.stats['waittime'] = 0
return True
def getResolutionData(self, avgpath, iternum):
evenvol = os.path.join(avgpath, "avglist3_%dp1.mrc" % (self.params["rescut"]))
oddvol = os.path.join(avgpath, "avglist3_%dp2.mrc" % (self.params["rescut"]))
evenhed = os.path.join(avgpath, "avglist3_%dp1.hed" % (self.params["rescut"]))
oddhed = os.path.join(avgpath, "avglist3_%dp2.hed" % (self.params["rescut"]))
emancmd1 = "proc3d %s %s" % (evenvol, evenhed)
apEMAN.executeEmanCmd(emancmd1, verbose=True)
emancmd2 = "proc3d %s %s" % (oddvol, oddhed)
apEMAN.executeEmanCmd(emancmd2, verbose=True)
fscfile = "fsc.eotest.%d" % (iternum)
fscpath = os.path.join(avgpath, fscfile)
emancmd3 = "proc3d %s %s fsc=%s" % (evenhed, oddhed, fscpath)
apEMAN.executeEmanCmd(emancmd3, verbose=True)
if not os.path.isfile(fscpath):
apDisplay.printWarning("Could not find FSC file: " + fscpath)
return None
f = open(fscpath, "r")
xy = f.readlines()
lines = len(xy)
boxsize = lines * 2.0
f.close()
# calculate the resolution:
halfres = apRecon.calcRes(fscpath, boxsize, self.params["step"])
apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms" % (halfres), "cyan")
# save to database
resq = appiondata.ApResolutionData()
resq["half"] = halfres
resq["fscfile"] = fscpath
return resq
def processParticles(self, imgdata, partdatas, shiftdata):
self.shortname = apDisplay.short(imgdata['filename'])
### if only selected points along helix,
### fill in points with helical step
if self.params['helicalstep']:
apix = apDatabase.getPixelSize(imgdata)
partdatas = self.fillWithHelicalStep(partdatas, apix)
### run batchboxer
self.boxedpartdatas, self.imgstackfile, self.partmeantree = self.boxParticlesFromImage(imgdata, partdatas, shiftdata)
if self.boxedpartdatas is None:
self.stats['lastpeaks'] = 0
apDisplay.printWarning("no particles were boxed from "+self.shortname+"\n")
self.badprocess = True
return None
self.stats['lastpeaks'] = len(self.boxedpartdatas)
apDisplay.printMsg("do not break function now otherwise it will corrupt stack")
#time.sleep(1.0)
### merge image particles into big stack
totalpart = self.mergeImageStackIntoBigStack(self.imgstackfile, imgdata)
### create a stack average every so often
if self.stats['lastpeaks'] > 0:
totalPartices = self.existingParticleNumber+self.stats['peaksum']+self.stats['lastpeaks']
logpeaks = math.log(totalPartices)
if logpeaks > self.logpeaks:
self.logpeaks = math.ceil(logpeaks)
numpeaks = math.ceil(math.exp(self.logpeaks))
apDisplay.printMsg("writing averaging stack, next average at %d particles"%(numpeaks))
mrc.write(self.summedParticles/float(totalPartices), "average.mrc")
return totalpart
def insertFSCData(self, iternum, refineIterData):
fscfile = 'fsc.eotest.%d'%(iternum)
fscpath = os.path.join(self.params['rundir'], "iter%03d"%(iternum), fscfile)
if not os.path.isfile(fscpath):
apDisplay.printWarning("Could not find FSC file: "+fscpath)
return None
f = open(fscpath, 'r')
apDisplay.printMsg("inserting FSC Data into database")
numinserts = 0
for line in f:
fscq = appiondata.ApFSCData()
fscq['refineIter'] = refineIterData
sline = line.strip()
bits = sline.split('\t')
fscq['pix'] = int(bits[0])
fscq['value'] = float(bits[1])
numinserts+=1
if self.params['commit'] is True:
fscq.insert()
apDisplay.printMsg("inserted "+str(numinserts)+" rows of FSC data into database")
f.close()
def findLastCompletedIteration(self):
#recondir = os.path.join(self.params['rundir'], "recon")
recondir = self.params['rundir']
iternum = 0
stop = False
while stop is False:
## check if iteration is complete
iternum += 1
class1Volume = "recon_it%03d_half1_model.star"%(iternum)
class2Volume = "recon_it%03d_half2_model.star"%(iternum)
class1Volumepath = os.path.join(recondir,class1Volume)
class2Volumepath = os.path.join(recondir,class2Volume)
if not os.path.isfile(class1Volumepath) or not os.path.isfile(class2Volumepath):
apDisplay.printWarning("Model.star file %s or %s is missing"%(class1Volumepath, class2Volumepath))
stop = True
break
### set last working iteration
numiter = iternum-1
if numiter < 1:
apDisplay.printError("No iterations were found")
apDisplay.printColor("Found %d complete iterations"%(numiter), "green")
return numiter
def readBatchUploadInfo(self):
# in this example, the batch script file should be separated by tab
# see example in function readUploadInfo for format
batchfilename = self.params['batchscript']
if not os.path.exists(batchfilename):
apDisplay.printError('Batch file %s not exist' % batchfilename)
return []
batchfile = open(batchfilename,'r')
lines = batchfile.readlines()
batchfile.close()
batchinfo = []
count = 0
for line in lines:
count += 1
#remove white space at ends
sline = line.strip()
if ' ' in sline:
apDisplay.printWarning("There is a space in the batch file on line %d"%(count))
#remove white space at ends
cols = sline.split('\t')
if len(cols) > 1:
batchinfo.append(cols)
else:
apDisplay.printWarning("Skipping line %d"%(count))
return batchinfo
def convertFSCFileForIteration(self, iteration):
''' RElion FSC info is in recon_it002_model.star and the final recon_model.star file, convert this to 3DEM format
For the final iteration, we need to use recon_model.star rather than the iteration specific file to get the actual values.
'''
lastiter = self.findLastCompletedIteration()
if iteration == lastiter:
fscfile = os.path.join(self.projmatchpath, "recon_model.star"%(iteration))
else:
fscfile = os.path.join(self.projmatchpath, "recon_it%.3d_half1_model.star"%(iteration))
f = starFile.StarFile( fscfile )
try:
f.read()
except e:
if iteration == lastiter:
apDisplay.printWarning("%s file could not be opened, data will NOT be inserted into the database. Relion did NOT run to completion." % fscfile)
else:
apDisplay.printWarning("%s file could not be opened, data will NOT be inserted into the database" % fscfile)
return False
dataBlock = f.getDataBlock("data_model_class_1")
loopDict = dataBlock.getLoopDict()
newfscfile = open(os.path.join(self.resultspath, "recon_%s_it%.3d_vol001.fsc" % (self.params['timestamp'], iteration)), "w")
newfscfile.write("### column (1) inverse Angstroms, column (2) Fourier Shell Correlation (FSC)")
for valueSet in loopDict:
resolution = float(valueSet["_rlnResolution"]) # already in inverse angstrom
fsc = float(valueSet["_rlnGoldStandardFsc"]) # should go from a value of 1 to 0
newfscfile.write("%.6f\t%.6f\n" % (resolution, fsc))
newfscfile.close()
return True
def sortFolder(self):
numsort = 0
apDisplay.printMsg("Sorting files into clean folders")
### move files for all particle iterations
files = []
for i in range(self.lastiter+1):
iterdir = "iter%03d"%(i)
apParam.createDirectory(iterdir, warning=False)
wildcard = "part*_it*%03d_*.*"%(i)
files.extend(glob.glob(wildcard))
wildcard = "part*_it*%03d.*"%(i)
files.extend(glob.glob(wildcard))
for filename in files:
if os.path.isfile(filename):
numsort += 1
shutil.move(filename,iterdir)
if numsort < 3:
apDisplay.printWarning("Problem in iteration file sorting, are they already sorted?")
apDisplay.printMsg("Sorted "+str(numsort)+" iteration files")
### move files for all reference iterations
refsort = 0
refdir = "refalign"
apParam.createDirectory(refdir, warning=False)
wildcard = "ref*_it*.*"
files = glob.glob(wildcard)
for filename in files:
refsort += 1
shutil.move(filename, refdir)
#if refsort < 5:
# apDisplay.printError("Problem in reference file sorting")
apDisplay.printMsg("Sorted "+str(refsort)+" reference files")
return
def writeTiltFile(tiltfile, seriesname, imagedict, azimuth, refimg ):
'''
Create the geometry object (tilt angles, dir of tilt axis, origins of ROI) ---
1. read the params from the database
2. write params to text file
'''
try:
f = open(tiltfile,'w')
except:
apDisplay.printWarning("Failed to create %s for writing the protomo2 tilt geometry parameter file" % (tiltfile, ))
raise
f.write('\n')
f.write ("TILT SERIES %s\n" % seriesname)
f.write ("\n")
f.write (" AXIS\n")
f.write ("\n")
f.write (" TILT AZIMUTH %g\n" % azimuth)
f.write ("\n")
keys = imagedict.keys()
keys.sort()
for n in keys:
f.write (" IMAGE %-5d FILE %s ORIGIN [ %8.3f %8.3f ] TILT ANGLE %8.3f ROTATION %8.3f\n" % (n, imagedict[n]['filename'], imagedict[n]['x'], imagedict[n]['y'], imagedict[n]['tilt'], imagedict[n]['rotation']))
f.write ("\n")
f.write (" REFERENCE IMAGE %d\n" % refimg)
f.write ("\n")
f.write ("\n")
f.write (" END\n")
f.close()
def start(self):
print self.params
if self.params['clusterId'] is not None:
self.clusterdata = appiondata.ApClusteringStackData.direct_query(self.params['clusterId'])
self.useClusterForTemplateStack()
elif self.params['alignId'] is not None:
self.aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignId'])
self.useClusterForTemplateStack()
else:
apDisplay.printMsg("Using local file: '"+str(self.params['templatestack'])+"' to upload template")
# copy templates to final location
if str(self.params['templatestack'])[-4:] == ".img" or str(self.params['templatestack'])[-4:] == ".hed":
self.params['templatestack'] = self.params['templatestack'][:-4]
if str(self.params['runname'])[-4:] == ".img" or str(self.params['runname'])[-4:0] == ".hed":
self.params['runname'] = self.params['runname'][:-4]
shutil.copyfile(str(self.params['templatestack'])+".img", os.path.join(self.params['rundir'], str(self.params['runname'])+".img"))
shutil.copyfile(str(self.params['templatestack'])+".hed", os.path.join(self.params['rundir'], str(self.params['runname'])+".hed"))
self.numimages = apFile.numImagesInStack(os.path.join(self.params['rundir'], str(self.params['runname'])+".hed"))
# insert templates to database
if self.params['commit'] is True:
self.uploadTemplateStack(insert=True)
else:
apDisplay.printWarning("not committing results to DB")
def writeTileFile2(tiltfile, seriesname, imagelist, origins, tilts, azimuth, refimg):
try:
f = open(tiltfile,'w')
except:
apDisplay.printWarning("Failed to create %s for writing the protomo2 tilt geometry parameter file" % (tiltfile, ))
raise
f.write('\n')
f.write ("TILT SERIES %s\n" % seriesname)
f.write ("\n")
f.write (" AXIS\n")
f.write ("\n")
f.write (" TILT AZIMUTH %g\n" % azimuth)
f.write ("\n")
for n in range(len(imagelist)):
imagename=os.path.splitext(os.path.basename(imagelist[n]))[0] #strip off path and extension from mrc file
f.write (" IMAGE %-5d FILE %s ORIGIN [ %8.3f %8.3f ] TILT ANGLE %8.3f ROTATION %8.3f\n" % (n+1, imagename, origins[n]['x'], origins[n]['y'], tilts[n], 0))
f.write ("\n")
f.write (" REFERENCE IMAGE %d\n" % refimg)
f.write ("\n")
f.write ("\n")
f.write (" END\n")
f.close()
def loopCommitToDatabase(self, imgdata):
"""
Over-writes the particleLoop commit and uses the appionLoop commit
"""
tiltdata = apTiltPair.getTiltPair(imgdata)
if tiltdata is None:
apDisplay.printWarning("Tilt data not found; not commiting data")
return False
### insert the runid
self.commitRunToDatabase(imgdata['session'], True)
if self.assess is not None:
#note runid is overrided to be 'run1'
apDatabase.insertImgAssessmentStatus(imgdata, self.params['runname'], self.assess)
apDatabase.insertImgAssessmentStatus(tiltdata, self.params['runname'], self.assess)
if len(self.peaktree1) < 3 or len(self.peaktree1) < 3:
apDisplay.printWarning("Not enough particle picks; not commiting transform or particle data")
return False
### insert the transform
transdata = apTiltPair.insertTiltTransform(imgdata, tiltdata, self.tiltparams, self.params)
### insert the particles
self.insertParticlePeakPairs(imgdata, tiltdata, transdata)
def start(self):
sessionname = self.params['sessionname']
runname = self.params['runname']
preset = self.params['preset']
sessionq = leginondata.SessionData(name=sessionname)
presetq=leginondata.PresetData(name=preset)
imgquery = leginondata.AcquisitionImageData()
imgquery['preset'] = presetq
imgquery['session'] = sessionq
imgtree = imgquery.query(readimages=False)
partq = appiondata.ApContourData()
sessiond = sessionq.query()
selectionid = apParticle.getSelectionIdFromName(runname, sessionname)
if not selectionid:
apDisplay.printWarning('No Object Tracing Run found in database, Skipping.......')
return
selectionrundata = apParticle.getSelectionRunDataFromID(selectionid)
file = open('contourpickerData-' + sessionname + '.txt','w')
file.write('session_id ' + runname + '\n')
file.write('usr_id ' + os.getlogin() + '\n')
file.write('experiment_name ' + sessionname + '\n')
file.write('experiment_description ' + sessiond[0]['comment'].strip() + '\n')
file.write('nimages ' + str(len(imgtree)) + '\n')
for imgdata in imgtree:
file.write('START_IMAGE' + '\n')
partq['image'] = imgdata
partq['selectionrun'] = selectionrundata
partd = partq.query()
if len(partd)>0:
file.write('image_refID ' + str(partd[0]['image'].dbid) + '\n')
file.write('image_name ' + imgdata['filename'] + '\n')
if len(partd)>0:
file.write('time_roi ' + str(partd[0].timestamp) + '\n')
#file.write('time_roi ' + partd[0]['DEF_timestamp'] + '\n')
file.write('dfac = 1\n')
maxversion = 0
numparticles = 0
for part in partd:
if int(part['version'])>maxversion:
maxversion = int(part['version'])
for part in partd:
if int(part['version'])==maxversion:
numparticles+=1
file.write('version_id ' + str(maxversion) + '\n')
file.write('ncontours ' + str(numparticles) + '\n')
pointq = appiondata.ApContourPointData()
for part in partd:
if int(part['version'])==maxversion:
# file.write('contour_number ' + )
file.write('method_used ' + part['method'] + ' ')
pointq['contour'] = part
pointd = pointq.query()
for point in pointd:
file.write(str(point['x']) + ',' + str(point['y']) + ';')
file.write('\n')
file.write('END_IMAGE' + '\n')
def start(self):
### universal particle counter
self.partnum = 1
### final stack file
self.combinefile = os.path.join( self.params['rundir'], self.params['stackfilename'] )
if os.path.isfile(self.combinefile):
apDisplay.printError("A stack with name "+self.params['stackfilename']+" and path "
+self.params['rundir']+" already exists.")
### loop through stacks
for stackstr in self.params['stackids']:
stackid = int(stackstr)
### get stack data
stackdata = apStack.getOnlyStackData(stackid)
### append particle to stack file
self.appendToStack(stackdata)
if self.params['commit'] is True:
### insert stack data
apDisplay.printColor("inserting new stack particles from stackid="+str(stackid), "cyan")
self.commitStack(stackid)
else:
apDisplay.printWarning("not committing data to database")
apStack.averageStack(stack=self.combinefile)
def averageSubStack(partlist, stackfile, bin=1):
if len(partlist) > 300:
partlist = partlist[:300]
boxsize = apImagicFile.getBoxsize(stackfile)
if len(partlist) == 0:
binboxsize = boxsize/bin
blank = numpy.ones((binboxsize, binboxsize), dtype=numpy.float32)
return blank
if not os.path.isfile(stackfile):
apDisplay.printWarning("could not find stack, "+stackfile)
return False
partdatalist = apImagicFile.readParticleListFromStack(stackfile, partlist, boxsize, msg=False)
partdataarray = numpy.asarray(partdatalist)
finaldata = partdataarray.mean(0)
if bin > 1:
finaldata = apImage.binImg(finaldata, bin)
return finaldata
def _initializeDoneDict(self):
"""
reads or creates a done dictionary
"""
self.donedictfile = os.path.join(self.params['rundir'],
self.functionname + ".donedict")
if os.path.isfile(
self.donedictfile) and self.params['continue'] == True:
### unpickle previously done dictionary
apDisplay.printMsg("Reading old done dictionary: " +
os.path.basename(self.donedictfile))
f = open(self.donedictfile, 'r')
self.donedict = cPickle.load(f)
f.close()
try:
if self.donedict['commit'] == self.params['commit']:
### all is well
apDisplay.printMsg("Found " + str(len(self.donedict)) +
" done dictionary entries")
return
elif self.donedict['commit'] is True and self.params[
'commit'] is not True:
### die
apDisplay.printError(
"Commit flag was enabled and is now disabled, create a new runname"
)
else:
### set up fresh dictionary
apDisplay.printWarning(
"'--commit' flag was changed, creating new done dictionary"
)
except KeyError:
apDisplay.printMsg("Found " + str(len(self.donedict)) +
" done dictionary entries")
### set up fresh dictionary
self.donedict = {}
self.donedict['commit'] = self.params['commit']
apDisplay.printMsg("Creating new done dictionary: " +
os.path.basename(self.donedictfile))
### write donedict to file
f = open(self.donedictfile, 'w', 0666)
cPickle.dump(self.donedict, f)
f.close()
return
def insertCoranRun(self, insert=False):
# create a AlignAnalysisRun object
analysisq = appiondata.ApAlignAnalysisRunData()
analysisq['runname'] = self.params['runname']
analysisq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
# ... path makes the run unique:
uniquerun = analysisq.query(results=1)
if uniquerun and insert is True:
apDisplay.printError("Run name '"+self.params['runname']+"' for align stack id="+\
str(self.params['alignstackid'])+"\nis already in the database")
coranq = appiondata.ApCoranRunData()
coranq['num_factors'] = self.params['numfactors']
coranq['mask_diam'] = 2.0 * self.params['maskrad']
coranq['run_seconds'] = self.runtime
# finish AlignAnalysisRun object
analysisq['description'] = self.params['description']
analysisq['alignstack'] = self.alignstackdata
analysisq['hidden'] = False
analysisq['coranrun'] = coranq
apDisplay.printMsg(
"inserting Align Analysis Run parameters into database")
if insert is True:
analysisq.insert()
### eigen data
for i in range(self.params['numfactors']):
factnum = i + 1
eigenq = appiondata.ApCoranEigenImageData()
eigenq['coranRun'] = coranq
eigenq['factor_num'] = factnum
path = os.path.join(self.params['rundir'], "coran")
eigenq['path'] = appiondata.ApPathData(path=os.path.abspath(path))
imgname = ("eigenimg%02d.png" % (factnum))
eigenq['image_name'] = imgname
if not os.path.isfile(os.path.join(path, imgname)):
apDisplay.printWarning(imgname + " does not exist")
continue
if insert is True:
eigenq['percent_contrib'] = self.contriblist[i]
eigenq.insert()
return
def checkConflicts(self):
super(MakeDDParticleMovieLoop, self).checkConflicts()
if self.params['bin'] != 1:
apDisplay.printError(
'binning is not yet implemented, please use bin=1')
#if self.params['denoise'] and not self.params['ddstack']:
# apDisplay.printError('denoise only works with ddstack')
if self.params['denoise'] and self.params['framestep'] > 1:
apDisplay.printWarning(
'Interval of frames must be one when denoising')
apDisplay.printWarning('Forcing it to 1....')
self.params['framestep'] = 1
if not self.params['nframe']:
# force nframe to a large member so that checkIsDD will consider it as dd data
# This number will limit the total processing frames if smaller than
# the actual number of frames in the images
self.params['nframe'] = 1000
def moveStack(filename1, filename2, warn=True):
### replace one imagic stack with another
rootname1 = os.path.splitext(filename1)[0]
rootname2 = os.path.splitext(filename2)[0]
for ext in (".hed", ".img"):
inf = rootname1 + ext
outf = rootname2 + ext
if os.path.isfile(inf):
if warn is True:
apDisplay.printWarning("replacing stack file '%s' with '%s' " %
(outf, inf))
time.sleep(1)
try:
shutil.move(inf, outf)
except:
apDisplay.printWarning('%s could not be replaced with %s' %
(outf, inf))
def totalLeastSquares(X, Y, W=None, epsilon=1e-5, maxiter=500):
"""
iterative refine weights of observations, de-emphasizing bad fitting points
"""
t0 = time.time()
### check the input
if checkMatrixSizes(X, Y) is False:
return None
### setup weights if necessary
if W is None:
#W = numpy.random.random((m)) # weights for the observations
#W = W*float(M)/W.sum()
W = numpy.ones(Y.shape) # even weights for the observations
### solve it
err0 = None
sys.stderr.write("running total least squares")
for i in range(maxiter):
sys.stderr.write(".")
beta = weightedLeastSquares(X, Y, W)
if beta is None:
if i < 2:
return None
beta = weightedLeastSquaresSVD(X, Y, W)
## calculate the absolute mean error
err = numpy.absolute(numpy.dot(X, beta) - Y).ravel()
#print "totalLeastSquares iter %d error: %.4f"%(i, err.mean())
## fit to a normal distribution
normerr = (err - err.min()) / err.std()
## calculate new weights based on
W = numpy.exp(-1 * normerr**2)
if W.sum() == 0:
apDisplay.printWarning("Failed to set weights")
return beta
## see if we can stop
if err0 is not None:
change = numpy.absolute(err - err0).mean()
if change < epsilon:
break
err0 = err
apDisplay.printMsg("totalLeastSquares completed in %s in %d iterations" %
(apDisplay.timeString(time.time() - t0), i))
return beta
def getTargets(self, imgdata, scratchdir='', handlefiles='direct'):
targetdict={}
#copy flatfields
brightrefpath=imgdata['bright']['session']['image path']
brightrefname=imgdata['bright']['filename']+'.mrc'
brightref=os.path.join(brightrefpath,brightrefname)
darkrefpath=imgdata['dark']['session']['image path']
darkrefname=imgdata['dark']['filename']+'.mrc'
darkref=os.path.join(darkrefpath,darkrefname)
#################################### do away with override flatfield option
framesdirname=imgdata['filename']+'.frames'
apDisplay.printMsg('Copying frames %s' % (framesdirname))
framespath=imgdata['session']['frame path']
framespathname=os.path.join(framespath,framesdirname)
if handlefiles == 'direct':
targetdict['brightref']=brightref
targetdict['darkref']=darkref
targetdict['framespathname']=framespathname
targetdict['outpath']=self.params['rundir']
elif handlefiles == 'copy':
shutil.copy(brightref,scratchdir)
shutil.copy(darkref,scratchdir)
targetdict['brightref']=os.path.join(scratchdir,brightrefname)
targetdict['darkref']=os.path.join(scratchdir,darkrefname)
try:
shutil.copytree(framespathname,os.path.join(scratchdir, framesdirname))
except:
apDisplay.printWarning('there was a problem copying the frames for %s' % (imgdata['filename']))
targetdict['framespathname']=os.path.join(scratchdir,framesdirname)
targetdict['outpath']=os.path.join(scratchdir,imgdata['filename'])
elif handlefiles == 'link':
os.symlink(brightref,os.path.join(scratchdir,brightrefname))
os.symlink(darkref,os.path.join(scratchdir,darkrefname))
os.symlink(framespathname,os.path.join(scratchdir, framesdirname))
targetdict['brightref']=os.path.join(scratchdir,brightrefname)
targetdict['darkref']=os.path.join(scratchdir,darkrefname)
targetdict['framespathname']=os.path.join(scratchdir,framesdirname)
targetdict['outpath']=os.path.join(scratchdir,imgdata['filename'])
return targetdict
def start(self):
### get volume files
volumefiles = glob.glob(self.params['volumes'])
if not volumefiles:
apDisplay.printError("Could not find volumes, %s" %
(self.params['volumes']))
### make list of all alignments to run
cmdlist = []
alignfiles = []
for volfile in volumefiles:
alignfile = "align-" + os.path.basename(volfile)
alignfiles.append(alignfile)
emancmd = "align3d %s %s %s " % (self.params['reference'], volfile,
alignfile)
if self.params['slow'] is True:
emancmd += "slow "
if self.params['noshrink'] is True:
emancmd += "noshrink "
#print emancmd
cmdlist.append(emancmd)
### run several alignment commands in parallel
apThread.threadCommands(cmdlist)
### average volumes together
ref = mrc.read(self.params['reference'])
average = numpy.zeros(ref.shape, dtype=numpy.float32)
del ref
count = 0
for alignfile in alignfiles:
if not os.path.isfile(alignfile):
apDisplay.printWarning("aligned volume not found: %s" %
(alignfile))
aligned = mrc.read(alignfile)
count += 1
### this assume all aligned volume have same box size
average += aligned
del aligned
### save average
average /= count
avgfile = os.path.abspath(self.params['average'])
mrc.write(average, avgfile)
apDisplay.printMsg("Wrote average file: " + avgfile)
def runCTFdisplayTools(imgdata,
ctfvalues,
opimagedir,
fftpath=None,
fftfreq=None):
### RUN CTF DISPLAY TOOLS
ctfdisplaydict = ctfdisplay.makeCtfImages(imgdata, ctfvalues, fftpath,
fftfreq)
if ctfdisplaydict is None:
return ctfvalues
### save the classic images as well
if 'graph1' in ctfvalues:
ctfvalues['graph3'] = os.path.basename(ctfvalues['graph1'])
if 'graph2' in ctfvalues:
ctfvalues['graph4'] = os.path.basename(ctfvalues['graph2'])
### new powerspec file
psfile = os.path.join(opimagedir, ctfdisplaydict['powerspecfile'])
if not os.path.isfile(ctfdisplaydict['powerspecfile']):
apDisplay.printWarning("Powerspec file not created")
else:
print ctfdisplaydict['powerspecfile']
shutil.move(ctfdisplaydict['powerspecfile'], psfile)
ctfvalues['graph1'] = os.path.basename(psfile)
### new 1d plot file
plotfile = os.path.join(opimagedir, ctfdisplaydict['plotsfile'])
shutil.move(ctfdisplaydict['plotsfile'], plotfile)
ctfvalues['graph2'] = os.path.basename(plotfile)
ctfvalues['confidence_30_10'] = ctfdisplaydict['conf3010']
ctfvalues['confidence_5_peak'] = ctfdisplaydict['conf5peak']
ctfvalues['overfocus_conf_30_10'] = ctfdisplaydict['overconf3010']
ctfvalues['overfocus_conf_5_peak'] = ctfdisplaydict['overconf5peak']
ctfvalues['resolution_80_percent'] = ctfdisplaydict['res80']
ctfvalues['resolution_50_percent'] = ctfdisplaydict['res50']
if not 'confidence_d' in ctfvalues or ctfvalues['confidence_d'] is None:
ctfvalues['confidence_d'] = ctfdisplaydict['conf5peak']
if not 'confidence' in ctfvalues or ctfvalues['confidence'] is None:
ctfvalues['confidence'] = ctfdisplaydict['conf3010']
### override the confidence
ctfvalues['confidence'] = max(ctfvalues['confidence'],
ctfvalues['confidence_d'],
ctfdisplaydict['conf5peak'],
ctfdisplaydict['conf3010'])
return ctfvalues
def _removeProcessedImages(self):
startlen = len(self.imgtree)
donecount = 0
reproccount = 0
rejectcount = 0
tiltcount = 0
self.stats['skipcount'] = 0
newimgtree = []
count = 0
t0 = time.time()
for imgdata in self.imgtree:
count += 1
if count % 10 == 0:
sys.stderr.write(".")
skip, reason = self.skipTestOnImage(imgdata)
imgname = imgdata['filename']
if skip is True:
if reason == 'reproc':
reproccount += 1
elif reason == 'reject' or reason == 'tilt':
self._writeDoneDict(imgname)
rejectcount += 1
if skip is True:
if self.stats['skipcount'] == 0:
sys.stderr.write("skipping processed images\n")
elif self.stats['skipcount'] % 80 == 0:
sys.stderr.write(".\n")
else:
sys.stderr.write(".")
self.stats['skipcount'] += 1
else:
newimgtree.append(imgdata)
sys.stderr.write("\n")
apDisplay.printMsg("finished skipping in %s" %
(apDisplay.timeString(time.time() - t0)))
if self.stats['skipcount'] > 0:
self.imgtree = newimgtree
sys.stderr.write("\n")
apDisplay.printWarning("skipped " + str(self.stats['skipcount']) +
" of " + str(startlen) + " images")
apDisplay.printMsg("[[ " + str(reproccount) + " no reprocess " +
" | " + str(rejectcount) + " rejected " +
" | " + str(tiltcount) + " wrong tilt " +
" | " + str(donecount) + " in donedict ]]")
def checkMPI(self):
mpiexe = apParam.getExecPath("mpirun")
if mpiexe is None:
return None
xmippexe = apParam.getExecPath("xmipp_mpi_ml_refine3d")
if xmippexe is None:
return None
lddcmd = "ldd "+xmippexe+" | grep mpi"
proc = subprocess.Popen(lddcmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc.wait()
lines = proc.stdout.readlines()
#print "lines=", lines
if lines and len(lines) > 0:
return mpiexe
else:
apDisplay.printWarning("Failed to find mpirun")
print "lines=", lines
return None
def getResolutionData(self, iternum):
fscfile = 'fsc.eotest.%d'%(iternum)
fscpath = os.path.join(self.params['rundir'], "iter%03d"%(iternum), fscfile)
if not os.path.isfile(fscpath):
apDisplay.printWarning("Could not find FSC file: "+fscpath)
return None
# calculate the resolution:
halfres = apRecon.calcRes(fscpath, self.boxsize, self.apix)
apDisplay.printColor("FSC 0.5 Resolution of %.3f Angstroms"%(halfres), "cyan")
# save to database
resq=appiondata.ApResolutionData()
resq['half'] = halfres
resq['fscfile'] = fscfile
return resq
def getRMeasureData(self, iteration):
volumeDensity='threed.'+iteration['num']+'a.mrc'
volPath = os.path.join(self.params['rundir'], volumeDensity)
if not os.path.exists(volPath):
apDisplay.printWarning("R Measure failed, volume density not found: "+volPath)
return None
resolution = apRecon.runRMeasure(self.params['apix'], volPath)
if resolution is None:
return None
rmesq=appiondata.ApRMeasureData()
rmesq['volume']=volumeDensity
rmesq['rMeasure']=resolution
return rmesq
def processParticle(self, partarray):
if self.filetype == "mrc":
partfile = os.path.join(self.partdir,
"part%06d.mrc" % (self.index))
mrc.write(partarray, partfile)
else:
partfile = os.path.join(self.partdir,
"part%06d.spi" % (self.index))
try:
spider.write(partarray, partfile)
except:
print partarray
apDisplay.printWarning(
"failed to write spider file for part index %d" %
(self.index))
print partarray.shape
spider.write(partarray, partfile)
self.partdocf.write(os.path.abspath(partfile) + " 1\n")
def setupRunDirectory(self):
"""
Set the run directory
"""
if self.params['rundir'] is None:
apDisplay.printWarning("run directory not defined, automatically setting it")
self.setProcessingDirName()
self.setRunDir()
if self.params['rundir'] is None:
apDisplay.printError("No run directory was set")
if self.quiet is False:
apDisplay.printMsg("Run directory: "+self.params['rundir'])
if not os.path.isdir(self.params['rundir']):
apDisplay.printError("run directory must exist for FileScript run")
os.chdir(self.params['rundir'])
def checkRefineLogError(self):
f = open(self.tasklogfile)
text = f.read()
f.close()
lines = text.split('\n')
f.close()
tasklog_basename = os.path.basename(self.tasklogfile)
# check alarm and error
if 'ERROR' in text:
self.failed = True
apDisplay.printError('There are Errors in %s' % (tasklog_basename),
False)
if 'WARNING' in text:
apDisplay.printWarning('There are Warning in %s during %s' %
(tasklog_basename, self.task_noun))
for line in lines:
if 'WARNING' in line:
apDisplay.printWarning(line)
def getRgbFile(msg=True):
"""
This file comes with xorg-x11-server-Xorg in Fedora 7,8
missing in Fedora 9
"""
filelist = [
"/usr/share/X11/rgb",
"/usr/X11R6/lib64/X11/rgb",
"/usr/X11R6/lib/X11/rgb",
]
xversion = getXversion()
if xversion is None or xversion > 1.0109:
return " "
for rgbfile in filelist:
if os.path.isfile(rgbfile+".txt"):
return " -co "+rgbfile
apDisplay.printWarning("Xvfb: could not find RGB File")
return " "
def _getRefImageData(self, reftype):
imagedata = super(GatanK2Processing, self).getCorrectedImageData()
if not self.use_full_raw_area:
refdata = imagedata[reftype]
if refdata is None:
# Use chosen default image
apDisplay.printWarning(
'No %s reference for the image, use default' % reftype)
default_image = self.getCorrectedImageData()
refdata = default_image[reftype]
else:
# use most recent CorrectorImageData
# TO DO: this should research only ones before the image is taken.
scopedata = self.image['scope']
channel = self.image['channel']
refdata = self.c_client.researchCorrectorImageData(
reftype, scopedata, self.camerainfo, channel)
return refdata
def checkExistingFile(self):
savedtomopath = os.path.join(self.params['rundir'],
self.params['name'] + ".rec")
origtomopath = self.params['file']
apDisplay.printWarning(
"A Tomogram by the same filename already exists: '" +
savedtomopath + "'")
### a tomogram by the same name already exists
savedheader = mrc.readHeaderFromFile(savedtomopath)
savedshape = savedheader['shape']
origheader = self.origheader
origshape = self.origshape
order = self.params['order']
if self.params['full']:
newshape = (self.imageshape[0], origshape[order.find('Z')],
self.imageshape[1])
else:
newshape = (origshape[order.find('X')], origshape[order.find('Y')],
origshape[order.find('Z')])
# not using md5 for file comparison because it takes too long. With padding, only min and max are stable
if newshape != savedshape or savedheader['amax'] != origheader[
'amax'] or savedheader['amin'] != origheader['amin']:
different = True
else:
different = False
mdnew = None
if different:
apDisplay.printWarning(
"The tomograms are different, cannot overwrite")
return True
elif apDatabase.isTomoInDB(mdnew, self.params['full'], savedtomopath):
### they are the same and its in the database already
apDisplay.printWarning(
"Identical Tomogram already exists in the database!")
apDisplay.printWarning("Upload Not Allowed")
self.params['commit'] = False
return True
else:
### they are the same, but it is not in the database
apDisplay.printWarning("The same tomogram with name '" +
savedtomopath +
"' already exists, but is not uploaded!")
if self.params['commit'] is True:
apDisplay.printMsg("Inserting tomogram into database...")
def readData(filename, filetype=None):
"""
savedata = {}
savedata['theta'] = self.data['theta']
savedata['gamma'] = self.data['gamma']
savedata['phi'] = self.data['phi']
savedata['shiftx'] = self.data['shiftx']
savedata['shifty'] = self.data['shifty']
savedata['savetime'] = time.asctime()+" "+time.tzname[time.daylight]
savedata['filetype'] = tiltfile.filetypes[self.data['filetypeindex']]
savedata['picks1'] = self.getArray1()
savedata['picks2'] = self.getArray2()
savedata['align1'] = self.getAlignedArray1()
savedata['align2'] = self.getAlignedArray2()
savedata['rmsd'] = self.getRmsdArray()
savedata['filepath'] = os.path.join(self.data['dirname'], self.data['outfile'])
savedata['image1name'] = self.panel1.filename
savedata['image2name'] = self.panel2.filename
"""
if not os.path.isfile(filename):
apDisplay.printWarning("file " + filename + " does not exist")
return None
if filetype is None:
filetype = guessFileType(filename)
if filetype == 'text':
savedata = readFromTextFile(filename)
elif filetype == 'xml':
savedata = readFromXMLFile(filename)
elif filetype == 'spider':
savedata = readFromSpiderFile(filename)
elif filetype == 'pickle':
savedata = readFromPickleFile(filename)
else:
apDisplay.printWarning("unknown file type")
return None
savedata['filename'] = os.path.basename(filename)
savedata['loadtime'] = time.asctime() + " " + time.tzname[time.daylight]
savedata['shiftx'] = 0.0
savedata['shifty'] = 0.0
savedata['scale'] = 1.0
return savedata
def getBestCtfValue(imgdata, sortType='res80', method=None, msg=True):
"""
takes an image and get the best ctfvalues for that image
"""
### get all ctf values
ctfq = appiondata.ApCtfData()
ctfq['image'] = imgdata
ctfvalues = ctfq.query()
imgname = apDisplay.short(imgdata['filename'])
if msg is True:
print "Found %d ctf values"%(len(ctfvalues))
### check if it has values
if ctfvalues is None:
apDisplay.printWarning("no CTF values found in database for img %s"%(imgname))
return None
### find the best values
bestsortvalue = -1
bestctfvalue = None
for ctfvalue in ctfvalues:
if method is not None:
imgmethod = getCtfMethod(ctfvalue)
if method != imgmethod:
continue
sortvalue = getSortValueFromCtfQuery(ctfvalue, sortType)
if sortvalue is None:
continue
if msg is True:
print "%.3f -- %s"%(sortvalue, ctfvalue['acerun']['name'])
if sortvalue > bestsortvalue:
bestsortvalue = sortvalue
bestctfvalue = ctfvalue
if bestctfvalue is None:
apDisplay.printWarning("no best CTF value for image %s"%(imgname))
return None
if msg is True:
print "*** %.3f"%(bestsortvalue)
printCtfData(bestctfvalue)
return bestctfvalue
def checkIsDD(self):
apDisplay.printWarning('Checking for dd')
if self.params['ddstack'] > 0:
self.is_dd_stack = True
self.is_dd = True
else:
if self.params['preset'] and '-a' in self.params['preset'] and (
self.params['nframe'] or self.params['driftlimit'] > 0):
self.is_dd = True
self.is_dd_stack = True
elif self.params['mrcnames'] and self.params['mrcnames'].split(
',')[0] and '-a' in self.params['mrcnames'].split(
',')[0] and (self.params['nframe']
or self.params['driftlimit'] > 0):
self.is_dd = True
self.is_dd_stack = True
elif self.params['nframe']:
self.is_dd = True
self.is_dd_frame = True
def getRMeasurePath():
unames = os.uname()
if unames[-1].find('64') >= 0:
exename = 'rmeasure64.exe'
else:
exename = 'rmeasure32.exe'
rmeasexe = subprocess.Popen("which "+exename, shell=True, stdout=subprocess.PIPE).stdout.read().strip()
if not os.path.isfile(rmeasexe):
rmeasexe = os.path.join(apParam.getAppionDirectory(), 'bin', exename)
if not os.path.isfile(rmeasexe):
exename = "rmeasure.exe"
rmeasexe = subprocess.Popen("which "+exename, shell=True, stdout=subprocess.PIPE).stdout.read().strip()
if not os.path.isfile(rmeasexe):
exename = "rmeasure"
rmeasexe = subprocess.Popen("which "+exename, shell=True, stdout=subprocess.PIPE).stdout.read().strip()
if not os.path.isfile(rmeasexe):
apDisplay.printWarning(exename+" was not found at: "+apParam.getAppionDirectory())
return None
return rmeasexe
def checkConflicts(self):
# Check that the program needed exists
gpuexelist = ['dosefgpu_driftcorr']
exename = 'dosefgpu_driftcorr'
gpuexe = subprocess.Popen(
"which " + exename, shell=True,
stdout=subprocess.PIPE).stdout.read().strip()
if not os.path.isfile(gpuexe):
apDisplay.printError('Correction program "%s" not available' %
exename)
# We don't have gpu locking
if self.params['parallel']:
apDisplay.printWarning(
'Make sure that you use different gpuid for each parallel process'
)
if self.params['stackid'] and not self.params['ddstack']:
apDisplay.printError(
'Specify stack alone does not work. Use makeDDRawFrameStack.py instead'
)
def highPassFilter(imgarray, apix=1.0, bin=1, radius=0.0, localbin=8, msg=True):
"""
high pass filter image to radius resolution
"""
if radius is None or radius < 1 or imgarray.shape[0] < 256:
if msg is True:
apDisplay.printMsg("skipping high pass filter")
return(imgarray)
try:
bimgarray = binImg(imgarray, localbin)
sigma=float(radius/apix/float(bin*localbin))
filtimg = ndimage.gaussian_filter(bimgarray, sigma=sigma)
expandimg = scaleImage(filtimg, localbin)
expandimg = frame_constant(expandimg, imgarray.shape)
filtimg = imgarray - expandimg
except:
apDisplay.printWarning("High Pass Filter failed")
return imgarray
return filtimg
def insertClusterRun(self):
### create a clustering run object
clusterrunq = appiondata.ApClusteringRunData()
clusterrunq['runname'] = self.params['runname']
clusterrunq['pixelsize'] = self.params['apix']
clusterrunq['boxsize'] = self.params['boxsize']
clusterrunq['description'] = self.params['description']
clusterrunq['num_particles'] = self.params['num_particles']
clusterrunq['alignstack'] = self.analysisdata['alignstack']
clusterrunq['analysisrun'] = self.analysisdata
apDisplay.printMsg("inserting clustering parameters into database")
if self.params['commit'] is True:
clusterrunq.insert()
else:
apDisplay.printWarning("not committing results to DB")
self.clusterrun = clusterrunq
return
def weightedLeastSquares(X, Y, W):
"""
solve using the normal equations with no manipulation
"""
t0 = time.time()
### check the input
if checkMatrixSizes(X, Y) is False:
return None
### solve it
XTW = numpy.transpose(X) * W
XTWX = numpy.dot(XTW, X)
if numpy.linalg.det(XTWX) == 0:
apDisplay.printWarning("Singular matrix in calculation")
return None
XTWXinv = numpy.linalg.inv(XTWX)
beta = numpy.dot(numpy.dot(XTWXinv, XTW), Y)
#apDisplay.printMsg("weightedLeastSquares completed in %s"
# %(apDisplay.timeString(time.time()-t0)))
return beta
def convertDefociToConvention(self, ctfdata):
ctfdb.printCtfData(ctfdata)
initdefocusratio = ctfdata['defocus2']/ctfdata['defocus1']
# program specific corrections?
self.angle = ctfdata['angle_astigmatism']
#angle = round(self.angle/2.5,0)*2.5
#by convention: abs(ctfdata['defocus1']) < abs(ctfdata['defocus2'])
if abs(ctfdata['defocus1']) > abs(ctfdata['defocus2']):
# incorrect, need to shift angle by 90 degrees
apDisplay.printWarning("|def1| > |def2|, flipping defocus axes")
self.defocus1 = ctfdata['defocus2']
self.defocus2 = ctfdata['defocus1']
self.angle += 90
else:
# correct, ratio > 1
self.defocus1 = ctfdata['defocus1']
self.defocus2 = ctfdata['defocus2']
if self.defocus1 < 0 and self.defocus2 < 0:
apDisplay.printWarning("Negative defocus values, taking absolute value")
self.defocus1 = abs(self.defocus1)
self.defocus2 = abs(self.defocus2)
self.defdiff = self.defocus1 - self.defocus2
#elliptical ratio is ratio of zero locations NOT defocii
self.defocusratio = self.defocus2/self.defocus1
self.ellipratio = ctftools.defocusRatioToEllipseRatio(self.defocus1, self.defocus2,
self.initfreq, self.cs, self.volts, self.ampcontrast)
# get angle within range -90 < angle <= 90
while self.angle > 90:
self.angle -= 180
while self.angle < -90:
self.angle += 180
apDisplay.printColor("Final params: def1: %.2e | def2: %.2e | angle: %.1f | defratio %.2f"%
(self.defocus1, self.defocus2, self.angle, self.defocusratio), "cyan")
perdiff = abs(self.defocus1-self.defocus2)/abs(self.defocus1+self.defocus2)
apDisplay.printMsg("Defocus Astig Percent Diff %.2f -- %.3e, %.3e"
%(perdiff*100,self.defocus1,self.defocus2))
return
def start(self):
self.setFileName()
scale = float(self.params['oldapix']) / self.params['newapix']
mrcname = os.path.join(self.params['rundir'],
self.params['name'] + ".mrc")
origmodel = self.params['file']
if os.path.isfile(mrcname):
apDisplay.printError("File exists")
if (abs(self.params['oldapix'] - self.params['newapix']) > 1.0e-2 or
abs(self.params['oldbox'] - self.params['newbox']) > 1.0e-1):
### rescale old model to a new size
apDisplay.printWarning("rescaling original model to a new size")
scale = float(self.params['oldapix']) / self.params['newapix']
apDisplay.printMsg("rescaling model by scale factor of %.4f" %
(scale))
apVolume.rescaleVolume(origmodel, mrcname, self.params['oldapix'],
self.params['newapix'],
self.params['newbox'])
else:
### simple upload, just copy file to models folder
apDisplay.printMsg("copying original model to a new location: " +
mrcname)
shutil.copyfile(origmodel, mrcname)
if self.params['viper2eman'] is True:
apVolume.viper2eman(mrcname, mrcname, apix=self.params['newapix'])
### render chimera images of model
contour = self.params['contour']
if self.params['mass'] is not None:
apChimera.setVolumeMass(mrcname, self.params['newapix'],
self.params['mass'])
contour = 1.0
apChimera.renderSnapshots(mrcname,
contour=contour,
zoom=self.params['zoom'],
sym=self.params['symdata']['eman_name'])
self.insertModel(mrcname)
def makeFilesForETomoSampleRecon(processdir,
stackdir,
aligndir,
templatedir,
seriesname,
thickness,
pixelsize,
yspacing,
has_rotation=False):
'''
Make or link local files required by etomo to redo sampling, creating tomopitch model, and reconstruct the volume.
'''
# etomo status file
createETomoBoundaryModelEDF(processdir, templatedir, seriesname, thickness,
pixelsize)
# required by remaking sample tomograms inside etomo
if has_rotation:
apDisplay.printWarning(
'eTomo wants to regenerate global alignment from non-rotated local alignment. This alignment with rotation will not work right'
)
else:
# prexf file is generated from database values if this function is called from tomomaker. It is better not to change it if exists.
prexf = seriesname + ".prexf"
alignprexf = os.path.join(aligndir, prexf)
localprexf = os.path.join(processdir, prexf)
apFile.safeCopy(alignprexf, localprexf)
writeETomoNewstComTemplate(processdir, seriesname)
rawtltname = '%s.rawtlt' % (seriesname)
shutil.copy(os.path.join(stackdir, rawtltname),
os.path.join(processdir, rawtltname))
# required by tomopitch model making
tomopitchname = 'tomopitch.com'
imodcomdir = os.path.join(os.environ['IMOD_DIR'], 'com')
tomopitchpath = os.path.join(processdir, tomopitchname)
shutil.copy(os.path.join(imodcomdir, tomopitchname), tomopitchpath)
apFile.replaceUniqueLinePatternInTxtFile(tomopitchpath, 'SpacingInY',
'SpacingInY\t%.1f\n' % yspacing)
# required by "Final Aligned Stack" step
stackname = '%s.st' % (seriesname)
apFile.safeSymLink(os.path.join(stackdir, stackname),
os.path.join(processdir, stackname))
