def loopProcessImage(self, imgdata):
"""
Over-writes the particleLoop processImage and uses the appionLoop processImage
"""
#GET THE TILT PAIR
tiltdata = apTiltPair.getTiltPair(imgdata)
if tiltdata is None:
return
### process images
procimgpath = os.path.join(self.params['rundir'],
imgdata['filename'] + '.dwn.mrc')
if not os.path.isfile(procimgpath):
apDisplay.printMsg("processing mrc for first image")
apFindEM.processAndSaveImage(imgdata, params=self.params)
procimg1 = apImage.mrcToArray(procimgpath, msg=False)
procimgpath = os.path.join(self.params['rundir'],
tiltdata['filename'] + '.dwn.mrc')
if not os.path.isfile(procimgpath):
apDisplay.printMsg("processing mrc for second image")
apFindEM.processAndSaveImage(tiltdata, params=self.params)
procimg2 = apImage.mrcToArray(procimgpath, msg=False)
#RUN THE ALIGNER GUI
result = self.runTiltAligner(imgdata, tiltdata)
if self.badprocess is True:
return
numpeaks = len(self.peaktree1)
#apDisplay.printMsg("Found "+str(numpeaks)+" particles for "+apDisplay.shortenImageName(imgdata['filename']))
self.stats['lastpeaks'] = numpeaks
### mask the images
procimg1, procimg2 = apTiltTransform.maskOverlapRegion(
procimg1, procimg2, self.data)
#CREATE PEAK JPEG
if self.threadJpeg is True:
threading.Thread(target=apPeaks.createTiltedPeakJpeg,
args=(imgdata, tiltdata, self.peaktree1,
self.peaktree2, self.params, procimg1,
procimg2)).start()
else:
apPeaks.createTiltedPeakJpeg(imgdata, tiltdata, self.peaktree1,
self.peaktree2, self.params, procimg1,
procimg2)
#EXTRA DONE DICT CALL
self._writeDoneDict(tiltdata['filename'])
def runTestShift(img1name, img2name, imgpath, tiltdiff, coord): img1path = os.path.join(imgpath, img1name) img2path = os.path.join(imgpath, img2name) print img1path print img2path img1 = apImage.binImg(apImage.mrcToArray(img1path),4) img2 = apImage.binImg(apImage.mrcToArray(img2path),4) apImage.arrayToMrc(img1,"img1a-raw.mrc") apImage.arrayToMrc(img2,"img2a-raw.mrc") origin, newpart, snr = apTiltShift.getTiltedCoordinates(img1, img2, tiltdiff, coord) apImage.arrayToJpegPlusPeak(img1, "img1a-guess.jpg", (origin[1], origin[0])) apImage.arrayToJpegPlusPeak(img2, "img2a-guess.jpg", (newpart[1], newpart[0]))
def loopProcessImage(self, imgdata):
"""
Over-writes the particleLoop processImage and uses the appionLoop processImage
"""
#GET THE TILT PAIR
tiltdata = apTiltPair.getTiltPair(imgdata)
if tiltdata is None:
return
### process images
procimgpath = os.path.join(self.params['rundir'], imgdata['filename']+'.dwn.mrc')
if not os.path.isfile(procimgpath):
apDisplay.printMsg("processing mrc for first image")
apFindEM.processAndSaveImage(imgdata, params=self.params)
procimg1 = apImage.mrcToArray(procimgpath, msg=False)
procimgpath = os.path.join(self.params['rundir'], tiltdata['filename']+'.dwn.mrc')
if not os.path.isfile(procimgpath):
apDisplay.printMsg("processing mrc for second image")
apFindEM.processAndSaveImage(tiltdata, params=self.params)
procimg2 = apImage.mrcToArray(procimgpath, msg=False)
#RUN THE ALIGNER GUI
result = self.runTiltAligner(imgdata, tiltdata)
if self.badprocess is True:
return
numpeaks = len(self.peaktree1)
#apDisplay.printMsg("Found "+str(numpeaks)+" particles for "+apDisplay.shortenImageName(imgdata['filename']))
self.stats['lastpeaks'] = numpeaks
### mask the images
procimg1, procimg2 = apTiltTransform.maskOverlapRegion(procimg1, procimg2, self.data)
#CREATE PEAK JPEG
if self.threadJpeg is True:
threading.Thread(target=apPeaks.createTiltedPeakJpeg, args=(imgdata, tiltdata, self.peaktree1,
self.peaktree2, self.params, procimg1, procimg2)).start()
else:
apPeaks.createTiltedPeakJpeg(imgdata, tiltdata, self.peaktree1, self.peaktree2, self.params,
procimg1, procimg2)
#EXTRA DONE DICT CALL
self._writeDoneDict(tiltdata['filename'])
def loopProcessImage(self, imgdata):
"""
setup like this to override things
"""
self.filtimgpath = os.path.join(self.params['rundir'], imgdata['filename']+'.dwn.mrc')
if os.path.isfile(self.filtimgpath) and self.params['continue'] is True:
apDisplay.printMsg("reading filtered image from mrc file")
self.filtarray = apImage.mrcToArray(self.filtimgpath, msg=False)
peaktree = self.processImage(imgdata, self.filtarray)
return peaktree
def openImageFile(self, filename):
self.filename = filename
if filename[-4:] == '.spi':
array = apImage.spiderToArray(filename, msg=False)
return array
elif filename[-4:] == '.mrc':
array = apImage.mrcToArray(filename, msg=False)
return array
else:
image = Image.open(filename)
array = apImage.imageToArray(image, msg=False)
array = array.astype(numpy.float32)
return array
return None
def openImageFile(self, filename): self.filename = filename if filename[-4:] == '.spi': array = apImage.spiderToArray(filename, msg=False) return array elif filename[-4:] == '.mrc': array = apImage.mrcToArray(filename, msg=False) return array else: image = Image.open(filename) array = apImage.imageToArray(image, msg=False) array = array.astype(numpy.float32) return array return None
def loopProcessImage(self, imgdata):
"""
setup like this to override things
"""
self.filtimgpath = os.path.join(self.params['rundir'],
imgdata['filename'] + '.dwn.mrc')
if os.path.isfile(
self.filtimgpath) and self.params['continue'] is True:
apDisplay.printMsg("reading filtered image from mrc file")
self.filtarray = apImage.mrcToArray(self.filtimgpath, msg=False)
peaktree = self.processImage(imgdata, self.filtarray)
return peaktree
def getTemplates(params):
"""
Inputs:
params['templateIds'], a list of template ids
params['apix'], desired pixel size
params['rundir'], output directory
image processing params
Processing:
Copies, scales, and filters templates
Outputs:
params['templatelist'], a list of template file basenames
"""
apDisplay.printMsg("getting templates")
if not params['templateIds']:
apDisplay.printError("No template ids were specified")
params['templatelist'] = [] #list of scaled files
for i,templateid in enumerate(params['templateIds']):
index = i+1
#print templateid
templateid = int(templateid)
if templateid < 0:
continue
#QUERY DB FOR TEMPLATE INFO
templatedata = appiondata.ApTemplateImageData.direct_query(abs(templateid))
if not (templatedata):
apDisplay.printError("Template Id "+str(templateid)+" was not found in database.")
#COPY THE FILE OVER
origtemplatepath = os.path.join(templatedata['path']['path'], templatedata['templatename'])
if not os.path.isfile(origtemplatepath):
apDisplay.printError("Template file not found: "+origtemplatepath)
apDisplay.printMsg("getting template: "+origtemplatepath)
copytemplatepath = os.path.join(params['rundir'], "origTemplate"+str(index)+".mrc")
scaletemplatepath = os.path.join(params['rundir'], "scaledTemplate"+str(index)+".mrc")
filtertemplatepath = os.path.join(params['rundir'], "filterTemplate"+str(index)+".mrc")
#masktemplatepath = os.path.join(params['rundir'], "maskTemplate"+str(index)+".mrc")
shutil.copyfile(origtemplatepath, copytemplatepath)
#RESCALE THE TEMPLATE
templatearray = apImage.mrcToArray(copytemplatepath)
#scale to correct apix
scalefactor = templatedata['apix'] / params['apix']
if abs(scalefactor - 1.0) > 0.01:
apDisplay.printMsg("rescaling template "+str(index)+": "+str(templatedata['apix'])+"->"+str(params['apix']))
templatearray = scaleTemplate(templatearray, scalefactor)
apImage.arrayToMrc(templatearray, scaletemplatepath, msg=False)
#bin and filter
templatearray = apImage.preProcessImage(templatearray, params=params, highpass=0, planeReg=False, invert=False)
#write to file
apImage.arrayToMrc(templatearray, filtertemplatepath, msg=False)
### MASK THE TEMPLATE AND SAVE
#mask the template, visual purposes only
#maskrad = params['diam']/params['apix']/params['bin']/2.0
#maskarray =
#apImage.arrayToMrc(templatearray, masktemplatepath, msg=False)
#ADD TO TEMPLATE LIST
params['templatelist'].append(os.path.basename(filtertemplatepath))
### ADD MIRROR IF REQUESTED
if 'templatemirrors' in params and params['templatemirrors'] is True:
mirrortemplatepath = os.path.join(params['rundir'], "mirrorTemplate"+str(index)+".mrc")
mirrorarray = numpy.fliplr(templatearray)
apImage.arrayToMrc(mirrorarray, mirrortemplatepath, msg=False)
params['templatelist'].append(os.path.basename(mirrortemplatepath))
#FINISH LOOP OVER template ids
#Set the apix
params['templateapix'] = params['apix']
apDisplay.printMsg("scaled & filtered "+str(len(params['templatelist']))+" file(s)")
return params['templatelist']
def processImage(self, imgdata):
self.ctfvalues = {}
bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True)
apix = apDatabase.getPixelSize(imgdata)
if (not (self.params['onepass'] and self.params['zeropass'])):
maskhighpass = False
ace2inputpath = os.path.join(imgdata['session']['image path'],imgdata['filename']+".mrc")
else:
maskhighpass = True
filterimg = apImage.maskHighPassFilter(imgdata['image'],apix,1,self.params['zeropass'],self.params['onepass'])
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
mrc.write(filterimg,ace2inputpath)
# make sure that the image is a square
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
if dimx != dimy:
dims = [dimx,dimy]
dims.sort()
apDisplay.printMsg("resizing image: %ix%i to %ix%i" % (dimx,dimy,dims[0],dims[0]))
mrcarray = apImage.mrcToArray(ace2inputpath,msg=False)
clippedmrc = apImage.frame_cut(mrcarray,[dims[0],dims[0]])
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
apImage.arrayToMrc(clippedmrc,ace2inputpath,msg=False)
### pad out image to speed up FFT calculations for non-standard image sizes
print "checking prime factor"
if primefactor.isGoodStack(dimx) is False:
goodsize = primefactor.getNextEvenPrime(dimx)
factor = float(goodsize) / float(dimx)
apDisplay.printMsg("padding image: %ix%i to %ix%i" % (dimx,dimy,dimx*factor,dimy*factor))
mrcarray = apImage.mrcToArray(ace2inputpath,msg=False)
# paddedmrc = imagefun.pad(mrcarray, None, factor)
paddedmrc = apImage.frame_constant(mrcarray, (dimx*factor,dimy*factor), cval=mrcarray.mean())
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
apImage.arrayToMrc(paddedmrc,ace2inputpath,msg=False)
inputparams = {
'input': ace2inputpath,
'cs': self.params['cs'],
'kv': imgdata['scope']['high tension']/1000.0,
'apix': apix,
'binby': self.params['bin'],
}
### make standard input for ACE 2
apDisplay.printMsg("Ace2 executable: "+self.ace2exe)
commandline = ( self.ace2exe
+ " -i " + str(inputparams['input'])
+ " -b " + str(inputparams['binby'])
+ " -c " + str(inputparams['cs'])
+ " -k " + str(inputparams['kv'])
+ " -a " + str(inputparams['apix'])
+ " -e " + str(self.params['edge_b'])+","+str(self.params['edge_t'])
+ " -r " + str(self.params['rotblur'])
+ "\n" )
### run ace2
apDisplay.printMsg("running ace2 at "+time.asctime())
apDisplay.printColor(commandline, "purple")
t0 = time.time()
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf)
ace2proc.wait()
### check if ace2 worked
basename = os.path.basename(ace2inputpath)
imagelog = basename+".ctf.txt"
if not os.path.isfile(imagelog) and self.stats['count'] <= 1:
### ace2 always crashes on first image??? .fft_wisdom file??
time.sleep(1)
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf)
ace2proc.wait()
if self.params['verbose'] is False:
aceoutf.close()
aceerrf.close()
if not os.path.isfile(imagelog):
lddcmd = "ldd "+self.ace2exe
lddproc = subprocess.Popen(lddcmd, shell=True)
lddproc.wait()
apDisplay.printError("ace2 did not run")
apDisplay.printMsg("ace2 completed in " + apDisplay.timeString(time.time()-t0))
### parse log file
self.ctfvalues = {
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
}
logf = open(imagelog, "r")
apDisplay.printMsg("reading log file %s"%(imagelog))
for line in logf:
sline = line.strip()
if re.search("^Final Defocus: ", sline):
### old ACE2
apDisplay.printError("This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now")
#parts = sline.split()
#self.ctfvalues['defocus1'] = float(parts[2])
#self.ctfvalues['defocus2'] = float(parts[3])
### convert to degrees
#self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4]))
elif re.search("^Final Defocus \(m,m,deg\):", sline):
### new ACE2
apDisplay.printMsg("Reading new ACE2 defocus")
parts = sline.split()
#print parts
self.ctfvalues['defocus1'] = float(parts[3])
self.ctfvalues['defocus2'] = float(parts[4])
# ace2 defines negative angle from +x toward +y
self.ctfvalues['angle_astigmatism'] = -float(parts[5])
elif re.search("^Amplitude Contrast:",sline):
parts = sline.split()
self.ctfvalues['amplitude_contrast'] = float(parts[2])
elif re.search("^Confidence:",sline):
parts = sline.split()
self.ctfvalues['confidence'] = float(parts[1])
self.ctfvalues['confidence_d'] = float(parts[1])
logf.close()
### summary stats
apDisplay.printMsg("============")
avgdf = (self.ctfvalues['defocus1']+self.ctfvalues['defocus2'])/2.0
ampconst = 100.0*self.ctfvalues['amplitude_contrast']
pererror = 100.0 * (self.ctfvalues['defocus1']-self.ctfvalues['defocus2']) / avgdf
apDisplay.printMsg("Defocus: %.3f x %.3f um (%.2f percent astigmatism)"%
(self.ctfvalues['defocus1']*1.0e6, self.ctfvalues['defocus2']*1.0e6, pererror ))
apDisplay.printMsg("Angle astigmatism: %.2f degrees"%(self.ctfvalues['angle_astigmatism']))
apDisplay.printMsg("Amplitude contrast: %.2f percent"%(ampconst))
apDisplay.printColor("Final confidence: %.3f"%(self.ctfvalues['confidence']),'cyan')
### double check that the values are reasonable
if avgdf > self.params['maxdefocus'] or avgdf < self.params['mindefocus']:
apDisplay.printWarning("bad defocus estimate, not committing values to database")
self.badprocess = True
if ampconst < 0.0 or ampconst > 80.0:
apDisplay.printWarning("bad amplitude contrast, not committing values to database")
self.badprocess = True
if self.ctfvalues['confidence'] < 0.2:
apDisplay.printWarning("bad confidence value, not committing values to database")
self.badprocess = True
## create power spectra jpeg
mrcfile = imgdata['filename']+".mrc.edge.mrc"
if os.path.isfile(mrcfile):
jpegfile = os.path.join(self.powerspecdir, apDisplay.short(imgdata['filename'])+".jpg")
ps = apImage.mrcToArray(mrcfile,msg=False)
c = numpy.array(ps.shape)/2.0
ps[c[0]-0,c[1]-0] = ps.mean()
ps[c[0]-1,c[1]-0] = ps.mean()
ps[c[0]-0,c[1]-1] = ps.mean()
ps[c[0]-1,c[1]-1] = ps.mean()
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
ps = numpy.log(ps+1.0)
ps = (ps-ps.mean())/ps.std()
cutoff = -2.0*ps.min()
ps = numpy.where(ps > cutoff, cutoff, ps)
cutoff = ps.mean()
ps = numpy.where(ps < cutoff, cutoff, ps)
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
apImage.arrayToJpeg(ps, jpegfile, msg=False)
apFile.removeFile(mrcfile)
self.ctfvalues['graph3'] = jpegfile
otherfiles = glob.glob(imgdata['filename']+".*.txt")
### remove extra debugging files
for filename in otherfiles:
if filename[-9:] == ".norm.txt":
continue
elif filename[-8:] == ".ctf.txt":
continue
else:
apFile.removeFile(filename)
if maskhighpass and os.path.isfile(ace2inputpath):
apFile.removeFile(ace2inputpath)
return
def fakeOutput(imgname, ccmapfile, params):
a = apImage.mrcToArray(imgname)
a = numpy.zeros(a.shape)
apImage.arrayToMrc(a, ccmapfile)
apFile.safeCopy('/home/acheng/Projects/Gfindem/example.box',
getBoxFileName(ccmapfile))
def fakeOutput(imgname,ccmapfile,params):
a = apImage.mrcToArray(imgname)
a = numpy.zeros(a.shape)
apImage.arrayToMrc(a,ccmapfile)
apFile.safeCopy('/home/acheng/Projects/Gfindem/example.box', getBoxFileName(ccmapfile))
def runSpectralFindEM(imgdict, params, thread=False):
"""
runs a separate thread of findem.exe for each template
to get cross-correlation maps
"""
imgname = imgdict['filename']
dwnimgname = os.path.splitext(imgname)[0] + ".dwn.mrc"
os.chdir(params['rundir'])
joblist = []
ccmaplist = []
processAndSaveImage(imgdict, params)
### FindEM crashes when an input image is longer than 76 characters
if len(dwnimgname) > 76:
randlink = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for x in range(10))
randlink += '.mrc'
os.symlink(dwnimgname, randlink)
if len(params['templatelist']) < 1:
apDisplay.printError("templatelist == 0; there are no templates")
for i, templatename in enumerate(params['templatelist']):
classavg = i + 1
#First round: normal findem: template x image
numstr = "%03d" % (100 + classavg)
ccmapfile1 = "cccmaxmap" + numstr + ".mrc"
apFile.removeFile(ccmapfile1)
params["startang" + str(100 + classavg)] = params["startang" +
str(classavg)]
params["endang" + str(100 + classavg)] = params["endang" +
str(classavg)]
params["incrang" + str(100 + classavg)] = params["incrang" +
str(classavg)]
if len(dwnimgname) > 76:
feed = findEMString(100 + classavg, templatename, randlink,
ccmapfile1, params)
else:
feed = findEMString(100 + classavg, templatename, dwnimgname,
ccmapfile1, params)
execFindEM(feed)
#Second round: template x template
numstr = "%03d" % (200 + classavg)
ccmapfile2 = "cccmaxmap" + numstr + ".mrc"
apFile.removeFile(ccmapfile2)
params["startang" + str(200 + classavg)] = params["startang" +
str(classavg)]
params["endang" + str(200 + classavg)] = params["endang" +
str(classavg)]
params["incrang" + str(200 + classavg)] = params["incrang" +
str(classavg)]
feed = findEMString(200 + classavg, templatename, templatename,
ccmapfile2, params)
execFindEM(feed)
#Final round: (template x template) x (template x image) = spectral
numstr = "%03d" % (300 + classavg)
ccmapfile3 = "cccmaxmap" + numstr + ".mrc"
apFile.removeFile(ccmapfile3)
params["startang" + str(300 + classavg)] = 0
params["endang" + str(300 + classavg)] = 10
params["incrang" + str(300 + classavg)] = 20
feed = findEMString(300 + classavg, ccmapfile2, ccmapfile1, ccmapfile3,
params)
execFindEM(feed)
#READ OUTPUT FILE
if not os.path.isfile(ccmapfile3):
apDisplay.printError("findem.exe did not run or crashed.\n" +
"Did you source useappion.sh?")
else:
ccmaxmap = apImage.mrcToArray(ccmapfile3)
ccmaplist.append(ccmaxmap)
return ccmaplist
def processImage(self, imgdata):
"""
time ./ctffind3.exe << eof
micrograph.mrc
montage.pow
2.0, 200.0, 0.07, 60000, 7.0, 2 #! CS[mm], HT[kV], AmpCnst, XMAG, DStep[um], PAve
128, 200.0, 8.0, 5000.0, 40000.0, 5000.0 #! Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep
eof
CARD 1: Input file name for image
CARD 2: Output file name to check result
CARD 3: CS[mm], HT[kV], AmpCnst, XMAG, DStep[um],PAve
CARD 4: Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep, dAst[A]
CTFTILT also asks for TiltA[deg], TiltR[deg] at CARD4
The output image file to check the result of the fitting
shows the filtered average power spectrum of the input
image in one half, and the fitted CTF (squared) in the
other half. The two halves should agree very well for a
successful fit.
CS: Spherical aberration coefficient of the objective in mm
HT: Electron beam voltage in kV
AmpCnst: Amount of amplitude contrast (fraction). For ice
images 0.07, for negative stain about 0.15.
XMAG: Magnification of original image
DStep: Pixel size on scanner in microns, or apix*mag/10000
PAve: Pixel averaging (PAve x PAve) for input image
Box: Tile size. The program divides the image into square
tiles and calculates the average power spectrum. Tiles
with a significantly higher or lower variance are
excluded; these are parts of the image which are unlikely
to contain useful information (beam edge, film number
etc). IMPORTANT: Box must have a even pixel dimensions.
ResMin: Low resolution end of data to be fitted.
ResMaX: High resolution end of data to be fitted.
dFMin: Starting defocus value for grid search in Angstrom.
Positive values represent an underfocus. The program
performs a systematic grid search of defocus values
and astigmatism before fitting a CTF to machine
precision.
dFMax: End defocus value for grid search in Angstrom.
FStep: Step width for grid search in Angstrom.
dAst: An additional parameter, dAst, was added to CARD 4 to restrain
the amount of astigmatism in the CTF fit. This makes the
fitting procedure more robust, especially in cases where
the Thon rings are not easily visible.
TiltA: guessed tilt angle
TiltR: angular range for initial coarse search
"""
#get Defocus in Angstroms
self.ctfvalues = {}
if self.params['nominal'] is not None:
nominal = abs(self.params['nominal']*1e4)
else:
nominal = abs(imgdata['scope']['defocus']*-1.0e10)
ctfvalue = ctfdb.getBestCtfByResolution(imgdata)
if ctfvalue is not None:
"""
## CTFFIND V3.5 (7-March-2012) prefers the smaller of the two values for astigmatic images
I found that say you have an image with 1.1um and 1.5um defocus astigmatism. If you give
CTFFIND the average value of 1.3um for the defocus and 0.4um astig (dast) then it will
try to fit 1.3um and 1.8um, so you need to give it the minimum value (1.1um) for it to
fit 1.1um and 1.5um.
"""
bestdef = min(ctfvalue['defocus1'],ctfvalue['defocus2'])*1.0e10
else:
bestdef = nominal
if ctfvalue is not None and self.params['bestdb'] is True:
bestampcontrast = round(ctfvalue['amplitude_contrast'],3)
beststigdiff = round(abs(ctfvalue['defocus1'] - ctfvalue['defocus2'])*1e10,1)
else:
bestampcontrast = self.params['amp'+self.params['medium']]
beststigdiff = self.params['dast']
if ctfvalue is not None and self.params['bestdb'] is True:
### set res max from resolution_80_percent
gmean = (ctfvalue['resolution_80_percent']*ctfvalue['resolution_50_percent']*self.params['resmax'])**(1/3.)
if gmean < self.params['resmin']:
# replace only if valid Issue #3291, #3547
self.params['resmax'] = round(gmean,2)
apDisplay.printColor("Setting resmax to the geometric mean of resolution values", "purple")
# dstep is the physical detector pixel size
dstep = None
if 'camera' in imgdata and imgdata['camera'] and imgdata['camera']['pixel size']:
dstep = imgdata['camera']['pixel size']['x']
if dstep is None:
dstep = apDatabase.getPixelSize(imgdata)*imgdata['scope']['magnification']/10000.0
dstep /=1e6
dstep = float(dstep)
mpixelsize = apDatabase.getPixelSize(imgdata)*1e-10
if self.params['apix_man'] is not None:
mpixelsize = self.params['apix_man']*1e-10
xmag = dstep / mpixelsize
apDisplay.printMsg("Xmag=%d, dstep=%.2e, mpix=%.2e"%(xmag, dstep, mpixelsize))
inputparams = {
'orig': os.path.join(imgdata['session']['image path'], imgdata['filename']+".mrc"),
'input': apDisplay.short(imgdata['filename'])+".mrc",
'output': apDisplay.short(imgdata['filename'])+"-pow.mrc",
'cs': self.params['cs'],
'kv': imgdata['scope']['high tension']/1000.0,
'ampcnst': bestampcontrast,
'xmag': xmag,
'dstep': dstep*1e6,
'pixavg': self.params['bin'],
'box': self.params['fieldsize'],
'resmin': self.params['resmin'],
'resmax': self.params['resmax'],
'defstep': self.params['defstep'], #round(defocus/32.0, 1),
'dast': beststigdiff,
}
defrange = self.params['defstep'] * self.params['numstep'] ## do 25 steps in either direction
inputparams['defmin']= round(bestdef-defrange, 1) #in meters
if inputparams['defmin'] < 0:
apDisplay.printWarning("Defocus minimum is less than zero")
inputparams['defmin'] = inputparams['defstep']
inputparams['defmax']= round(bestdef+defrange, 1) #in meters
apDisplay.printColor("Defocus search range: %d A to %d A (%.2f to %.2f um)"
%(inputparams['defmin'], inputparams['defmax'],
inputparams['defmin']*1e-4, inputparams['defmax']*1e-4), "cyan")
### secondary lock check right before it starts on the real part
if self.params['parallel'] and os.path.isfile(apDisplay.short(imgdata['filename'])+".mrc"):
# This is a secondary image lock check, checking the first output of the process.
# It alone is not good enough
apDisplay.printWarning('Some other parallel process is working on the same image. Skipping')
return
### create local link to image
if not os.path.exists(inputparams['input']):
os.symlink(inputparams['orig'], inputparams['input'])
### make standard input for ctf estimation
line1cmd = inputparams['input']+"\n"
line2cmd = inputparams['output']+"\n"
line3cmd = (
str(inputparams['cs'])+","
+ str(inputparams['kv'])+","
+ str(inputparams['ampcnst'])+","
+ str(inputparams['xmag'])+","
+ str(inputparams['dstep'])+","
+ str(inputparams['pixavg'])+"\n")
line4cmd = (
str(inputparams['box'])+","
+ str(inputparams['resmin'])+","
+ str(inputparams['resmax'])+","
+ str(inputparams['defmin'])+","
+ str(inputparams['defmax'])+","
+ str(inputparams['defstep'])+","
+ str(inputparams['dast']))
### additional ctftilt parameters
if self.params['ctftilt'] is True:
tiltang = apDatabase.getTiltAngleDeg(imgdata)
line4cmd += (","+str(tiltang)+",10")
line4cmd += "\n"
if os.path.isfile(inputparams['output']):
# program crashes if this file exists
apFile.removeFile(inputparams['output'])
t0 = time.time()
apDisplay.printMsg("running ctf estimation at "+time.asctime())
ctfproglog = os.path.join(self.logdir, os.path.splitext(imgdata['filename'])[0]+"-ctfprog.log")
logf = open(ctfproglog, "w")
ctfprogproc = subprocess.Popen(self.ctfprgmexe, shell=True, stdin=subprocess.PIPE, stdout=logf)
apDisplay.printColor(self.ctfprgmexe, "magenta")
apDisplay.printColor(line1cmd.strip(),"magenta")
apDisplay.printColor(line2cmd.strip(),"magenta")
apDisplay.printColor(line3cmd.strip(),"magenta")
apDisplay.printColor(line4cmd.strip(),"magenta")
ctfprogproc.stdin.write(line1cmd)
ctfprogproc.stdin.write(line2cmd)
ctfprogproc.stdin.write(line3cmd)
ctfprogproc.stdin.write(line4cmd)
ctfprogproc.communicate()
logf.close()
apDisplay.printMsg("ctf estimation completed in "+apDisplay.timeString(time.time()-t0))
#apFile.removeFile(inputparams['input'])
### parse ctf estimation output
self.ctfvalues = {}
logf = open(ctfproglog, "r")
## ctffind & ctftilt have diff # values
numvals = 6
if self.params['ctftilt'] is True:
numvals=8
for line in logf:
sline = line.strip()
if sline[-12:] == "Final Values":
#print sline
if '**********' in sline:
sline = re.sub('**********', ' **********', sline)
bits = sline.split()
if len(bits) != numvals:
apDisplay.printError("wrong number of values in "+str(bits))
for i,bit in enumerate(bits[0:(numvals-2)]):
bits[i] = float(bit)
self.ctfvalues = {
'defocus1': float(bits[0])*1e-10,
'defocus2': float(bits[1])*1e-10,
# WARNING: this is the negative of the direct result
'angle_astigmatism': float(bits[2]),
'amplitude_contrast': inputparams['ampcnst'],
'cross_correlation': float(bits[numvals-3]),
'nominal': nominal*1e-10,
'defocusinit': bestdef*1e-10,
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
'confidence': float(bits[numvals-3]),
'confidence_d': round(math.sqrt(abs(float(bits[numvals-3]))), 5)
}
if self.params['ctftilt'] is True:
self.ctfvalues['tilt_axis_angle']=float(bits[3])
self.ctfvalues['tilt_angle']=float(bits[4])
### write to log file
f = open("ctfvalues.log", "a")
f.write("=== "+imgdata['filename']+" ===\n")
if not self.ctfvalues:
nominaldf = imgdata['scope']['defocus']
else:
nominaldf = self.ctfvalues['nominal']
line1 = ("nominal=%.1e, bestdef=%.1e," %
( nominaldf, self.ctfvalues['defocusinit']))
if self.params['ctftilt'] is True:
self.ctfvalues['origtiltang'] = tiltang
line1+=" tilt=%.1f,"%tiltang
apDisplay.printMsg(line1)
f.write(line1)
line2 = ("def_1=%.1e, def_2=%.1e, astig_angle=%.1f, cross_corr=%.3f,\n" %
( self.ctfvalues['defocus1'], self.ctfvalues['defocus2'], self.ctfvalues['angle_astigmatism'],
self.ctfvalues['cross_correlation'] ))
if self.params['ctftilt'] is True:
line2+= ("tilt_angle=%.1f, tilt_axis_angle=%.1f,\n" %
(self.ctfvalues['tilt_angle'], self.ctfvalues['tilt_axis_angle']))
apDisplay.printMsg(line2)
f.write(line2)
f.close()
#convert powerspectra to JPEG
outputjpgbase = os.path.basename(os.path.splitext(inputparams['output'])[0]+".jpg")
self.lastjpg = outputjpgbase
outputjpg = os.path.join(self.powerspecdir, self.lastjpg)
powspec = apImage.mrcToArray(inputparams['output'])
apImage.arrayToJpeg(powspec, outputjpg)
shutil.move(inputparams['output'], os.path.join(self.powerspecdir, inputparams['output']))
self.ctfvalues['graph1'] = outputjpg
#apFile.removeFile(inputparams['input'])
return
def getTemplates(params):
"""
Inputs:
params['templateIds'], a list of template ids
params['apix'], desired pixel size
params['rundir'], output directory
image processing params
Processing:
Copies, scales, and filters templates
Outputs:
params['templatelist'], a list of template file basenames
"""
apDisplay.printMsg("getting templates")
if not params['templateIds']:
apDisplay.printError("No template ids were specified")
params['templatelist'] = [] #list of scaled files
for i, templateid in enumerate(params['templateIds']):
index = i + 1
#print templateid
templateid = int(templateid)
if templateid < 0:
continue
#QUERY DB FOR TEMPLATE INFO
templatedata = appiondata.ApTemplateImageData.direct_query(
abs(templateid))
if not (templatedata):
apDisplay.printError("Template Id " + str(templateid) +
" was not found in database.")
#COPY THE FILE OVER
origtemplatepath = os.path.join(templatedata['path']['path'],
templatedata['templatename'])
if not os.path.isfile(origtemplatepath):
apDisplay.printError("Template file not found: " +
origtemplatepath)
apDisplay.printMsg("getting template: " + origtemplatepath)
copytemplatepath = os.path.join(params['rundir'],
"origTemplate" + str(index) + ".mrc")
scaletemplatepath = os.path.join(
params['rundir'], "scaledTemplate" + str(index) + ".mrc")
filtertemplatepath = os.path.join(
params['rundir'], "filterTemplate" + str(index) + ".mrc")
#masktemplatepath = os.path.join(params['rundir'], "maskTemplate"+str(index)+".mrc")
shutil.copyfile(origtemplatepath, copytemplatepath)
#RESCALE THE TEMPLATE
templatearray = apImage.mrcToArray(copytemplatepath)
#scale to correct apix
scalefactor = templatedata['apix'] / params['apix']
if abs(scalefactor - 1.0) > 0.01:
apDisplay.printMsg("rescaling template " + str(index) + ": " +
str(templatedata['apix']) + "->" +
str(params['apix']))
templatearray = scaleTemplate(templatearray, scalefactor)
apImage.arrayToMrc(templatearray, scaletemplatepath, msg=False)
#bin and filter
templatearray = apImage.preProcessImage(templatearray,
params=params,
highpass=0,
planeReg=False,
invert=False)
#write to file
apImage.arrayToMrc(templatearray, filtertemplatepath, msg=False)
### MASK THE TEMPLATE AND SAVE
#mask the template, visual purposes only
#maskrad = params['diam']/params['apix']/params['bin']/2.0
#maskarray =
#apImage.arrayToMrc(templatearray, masktemplatepath, msg=False)
#ADD TO TEMPLATE LIST
params['templatelist'].append(os.path.basename(filtertemplatepath))
### ADD MIRROR IF REQUESTED
if 'templatemirrors' in params and params['templatemirrors'] is True:
mirrortemplatepath = os.path.join(
params['rundir'], "mirrorTemplate" + str(index) + ".mrc")
mirrorarray = numpy.fliplr(templatearray)
apImage.arrayToMrc(mirrorarray, mirrortemplatepath, msg=False)
params['templatelist'].append(os.path.basename(mirrortemplatepath))
#FINISH LOOP OVER template ids
#Set the apix
params['templateapix'] = params['apix']
apDisplay.printMsg("scaled & filtered " +
str(len(params['templatelist'])) + " file(s)")
return params['templatelist']
def runFindEM(imgdict, params, thread=False):
"""
runs a separate thread of findem.exe for each template
to get cross-correlation maps
"""
### check image
processAndSaveImage(imgdict, params)
dwnimgname = imgdict['filename']+".dwn.mrc"
if not os.path.isfile(dwnimgname):
apDisplay.printError("cound not find image to process: "+dwnimgname)
### FindEM crashes when an input image is longer than 76 characters
if len(dwnimgname) > 76:
randlink = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(10))
randlink+= '.mrc'
os.symlink(dwnimgname, randlink)
### check template
if len(params['templatelist']) < 1:
apDisplay.printError("templatelist == 0; there are no templates")
joblist = []
ccmapfilelist = []
# For FindEM2
# ### generate circular mask for FindEM2
# apDisplay.printMsg("creating mask file for template matching")
# img = apImage.mrcToArray(params['templatelist'][0])
# circlemask = 1 - imagefun.filled_circle(img.shape,img.shape[0]/2*0.95)
# apImage.arrayToMrc(circlemask,"tmpmask.mrc")
# del img,circlemask
workimg = randlink if len(dwnimgname) > 76 else dwnimgname
### create list of inputs for findem threads
feeds = []
for i,templatename in enumerate(params['templatelist']):
classavg = i + 1
# OUTPUT FILE NAME
numstr = "%03d" % classavg
ccmapfile="cccmaxmap"+numstr+".mrc"
apFile.removeFile(ccmapfile)
feeds.append(findEMString(classavg, templatename, workimg, ccmapfile, params))
#STORE OUTPUT FILE
ccmapfilelist.append(ccmapfile)
### launch findem threads
t0 = time.time()
findemexe = getFindEMPath()
pool = multiprocessing.Pool(processes=params['nproc'])
runner = findemrunner(findemexe,len(params['templatelist']))
for i,feed in enumerate(feeds):
pool.apply_async(runner, (i,feed))
pool.close()
pool.join()
apDisplay.printMsg("\nFindEM finished in "+apDisplay.timeString(time.time()-t0)+"\n")
# For FindEM2
# os.remove("tmpmask.mrc")
### READ OUTPUT FILES
ccmaplist = []
for ccmapfile in ccmapfilelist:
if not os.path.isfile(ccmapfile):
apDisplay.printError("findem.exe did not run or crashed.\n")
ccmaxmap = apImage.mrcToArray(ccmapfile)
ccmaplist.append(ccmaxmap)
return ccmaplist
def runSpectralFindEM(imgdict, params, thread=False):
"""
runs a separate thread of findem.exe for each template
to get cross-correlation maps
"""
imgname = imgdict['filename']
dwnimgname = os.path.splitext(imgname)[0]+".dwn.mrc"
os.chdir(params['rundir'])
joblist = []
ccmaplist = []
processAndSaveImage(imgdict, params)
### FindEM crashes when an input image is longer than 76 characters
if len(dwnimgname) > 76:
randlink = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(10))
randlink+= '.mrc'
os.symlink(dwnimgname, randlink)
if len(params['templatelist']) < 1:
apDisplay.printError("templatelist == 0; there are no templates")
for i,templatename in enumerate(params['templatelist']):
classavg = i + 1
#First round: normal findem: template x image
numstr = "%03d" % (100+classavg)
ccmapfile1 = "cccmaxmap"+numstr+".mrc"
apFile.removeFile(ccmapfile1)
params["startang"+str(100+classavg)] = params["startang"+str(classavg)]
params["endang"+str(100+classavg)] = params["endang"+str(classavg)]
params["incrang"+str(100+classavg)] = params["incrang"+str(classavg)]
if len(dwnimgname) > 76:
feed = findEMString(100+classavg, templatename, randlink, ccmapfile1, params)
else:
feed = findEMString(100+classavg, templatename, dwnimgname, ccmapfile1, params)
sys.exit()
execFindEM(feed)
#Second round: template x template
numstr = "%03d" % (200+classavg)
ccmapfile2 = "cccmaxmap"+numstr+".mrc"
apFile.removeFile(ccmapfile2)
params["startang"+str(200+classavg)] = params["startang"+str(classavg)]
params["endang"+str(200+classavg)] = params["endang"+str(classavg)]
params["incrang"+str(200+classavg)] = params["incrang"+str(classavg)]
feed = findEMString(200+classavg, templatename, templatename, ccmapfile2, params)
execFindEM(feed)
#Final round: (template x template) x (template x image) = spectral
numstr = "%03d" % (300+classavg)
ccmapfile3 = "cccmaxmap"+numstr+".mrc"
apFile.removeFile(ccmapfile3)
params["startang"+str(300+classavg)] = 0
params["endang"+str(300+classavg)] = 10
params["incrang"+str(300+classavg)] = 20
feed = findEMString(300+classavg, ccmapfile2, ccmapfile1, ccmapfile3, params)
execFindEM(feed)
#READ OUTPUT FILE
if not os.path.isfile(ccmapfile3):
apDisplay.printError("findem.exe did not run or crashed.\n"+
"Did you source useappion.sh?")
else:
ccmaxmap = apImage.mrcToArray(ccmapfile3)
ccmaplist.append(ccmaxmap)
return ccmaplist
def processImage(self, imgdata):
self.ctfvalues = {}
bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True)
apix = apDatabase.getPixelSize(imgdata)
if (not (self.params['onepass'] and self.params['zeropass'])):
maskhighpass = False
ace2inputpath = os.path.join(imgdata['session']['image path'],
imgdata['filename'] + ".mrc")
else:
maskhighpass = True
filterimg = apImage.maskHighPassFilter(imgdata['image'], apix, 1,
self.params['zeropass'],
self.params['onepass'])
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
mrc.write(filterimg, ace2inputpath)
# make sure that the image is a square
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
if dimx != dimy:
dims = [dimx, dimy]
dims.sort()
apDisplay.printMsg("resizing image: %ix%i to %ix%i" %
(dimx, dimy, dims[0], dims[0]))
mrcarray = apImage.mrcToArray(ace2inputpath, msg=False)
clippedmrc = apImage.frame_cut(mrcarray, [dims[0], dims[0]])
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
apImage.arrayToMrc(clippedmrc, ace2inputpath, msg=False)
### pad out image to speed up FFT calculations for non-standard image sizes
print "checking prime factor"
if primefactor.isGoodStack(dimx) is False:
goodsize = primefactor.getNextEvenPrime(dimx)
factor = float(goodsize) / float(dimx)
apDisplay.printMsg("padding image: %ix%i to %ix%i" %
(dimx, dimy, dimx * factor, dimy * factor))
mrcarray = apImage.mrcToArray(ace2inputpath, msg=False)
# paddedmrc = imagefun.pad(mrcarray, None, factor)
paddedmrc = apImage.frame_constant(mrcarray,
(dimx * factor, dimy * factor),
cval=mrcarray.mean())
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
apImage.arrayToMrc(paddedmrc, ace2inputpath, msg=False)
inputparams = {
'input': ace2inputpath,
'cs': self.params['cs'],
'kv': imgdata['scope']['high tension'] / 1000.0,
'apix': apix,
'binby': self.params['bin'],
}
### make standard input for ACE 2
apDisplay.printMsg("Ace2 executable: " + self.ace2exe)
commandline = (self.ace2exe + " -i " + str(inputparams['input']) +
" -b " + str(inputparams['binby']) + " -c " +
str(inputparams['cs']) + " -k " +
str(inputparams['kv']) + " -a " +
str(inputparams['apix']) + " -e " +
str(self.params['edge_b']) + "," +
str(self.params['edge_t']) + " -r " +
str(self.params['rotblur']) + "\n")
### run ace2
apDisplay.printMsg("running ace2 at " + time.asctime())
apDisplay.printColor(commandline, "purple")
t0 = time.time()
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline,
shell=True,
stderr=aceerrf,
stdout=aceoutf)
ace2proc.wait()
### check if ace2 worked
basename = os.path.basename(ace2inputpath)
imagelog = basename + ".ctf.txt"
if not os.path.isfile(imagelog) and self.stats['count'] <= 1:
### ace2 always crashes on first image??? .fft_wisdom file??
time.sleep(1)
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline,
shell=True,
stderr=aceerrf,
stdout=aceoutf)
ace2proc.wait()
if self.params['verbose'] is False:
aceoutf.close()
aceerrf.close()
if not os.path.isfile(imagelog):
lddcmd = "ldd " + self.ace2exe
lddproc = subprocess.Popen(lddcmd, shell=True)
lddproc.wait()
apDisplay.printError("ace2 did not run")
apDisplay.printMsg("ace2 completed in " +
apDisplay.timeString(time.time() - t0))
### parse log file
self.ctfvalues = {
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
}
logf = open(imagelog, "r")
apDisplay.printMsg("reading log file %s" % (imagelog))
for line in logf:
sline = line.strip()
if re.search("^Final Defocus: ", sline):
### old ACE2
apDisplay.printError(
"This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now"
)
#parts = sline.split()
#self.ctfvalues['defocus1'] = float(parts[2])
#self.ctfvalues['defocus2'] = float(parts[3])
### convert to degrees
#self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4]))
elif re.search("^Final Defocus \(m,m,deg\):", sline):
### new ACE2
apDisplay.printMsg("Reading new ACE2 defocus")
parts = sline.split()
#print parts
self.ctfvalues['defocus1'] = float(parts[3])
self.ctfvalues['defocus2'] = float(parts[4])
# ace2 defines negative angle from +x toward +y
self.ctfvalues['angle_astigmatism'] = -float(parts[5])
elif re.search("^Amplitude Contrast:", sline):
parts = sline.split()
self.ctfvalues['amplitude_contrast'] = float(parts[2])
elif re.search("^Confidence:", sline):
parts = sline.split()
self.ctfvalues['confidence'] = float(parts[1])
self.ctfvalues['confidence_d'] = float(parts[1])
logf.close()
### summary stats
apDisplay.printMsg("============")
avgdf = (self.ctfvalues['defocus1'] + self.ctfvalues['defocus2']) / 2.0
ampconst = 100.0 * self.ctfvalues['amplitude_contrast']
pererror = 100.0 * (self.ctfvalues['defocus1'] -
self.ctfvalues['defocus2']) / avgdf
apDisplay.printMsg(
"Defocus: %.3f x %.3f um (%.2f percent astigmatism)" %
(self.ctfvalues['defocus1'] * 1.0e6,
self.ctfvalues['defocus2'] * 1.0e6, pererror))
apDisplay.printMsg("Angle astigmatism: %.2f degrees" %
(self.ctfvalues['angle_astigmatism']))
apDisplay.printMsg("Amplitude contrast: %.2f percent" % (ampconst))
apDisplay.printColor(
"Final confidence: %.3f" % (self.ctfvalues['confidence']), 'cyan')
### double check that the values are reasonable
if avgdf > self.params['maxdefocus'] or avgdf < self.params[
'mindefocus']:
apDisplay.printWarning(
"bad defocus estimate, not committing values to database")
self.badprocess = True
if ampconst < 0.0 or ampconst > 80.0:
apDisplay.printWarning(
"bad amplitude contrast, not committing values to database")
self.badprocess = True
if self.ctfvalues['confidence'] < 0.2:
apDisplay.printWarning(
"bad confidence value, not committing values to database")
self.badprocess = True
## create power spectra jpeg
mrcfile = imgdata['filename'] + ".mrc.edge.mrc"
if os.path.isfile(mrcfile):
jpegfile = os.path.join(
self.powerspecdir,
apDisplay.short(imgdata['filename']) + ".jpg")
ps = apImage.mrcToArray(mrcfile, msg=False)
c = numpy.array(ps.shape) / 2.0
ps[c[0] - 0, c[1] - 0] = ps.mean()
ps[c[0] - 1, c[1] - 0] = ps.mean()
ps[c[0] - 0, c[1] - 1] = ps.mean()
ps[c[0] - 1, c[1] - 1] = ps.mean()
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
ps = numpy.log(ps + 1.0)
ps = (ps - ps.mean()) / ps.std()
cutoff = -2.0 * ps.min()
ps = numpy.where(ps > cutoff, cutoff, ps)
cutoff = ps.mean()
ps = numpy.where(ps < cutoff, cutoff, ps)
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
apImage.arrayToJpeg(ps, jpegfile, msg=False)
apFile.removeFile(mrcfile)
self.ctfvalues['graph3'] = jpegfile
otherfiles = glob.glob(imgdata['filename'] + ".*.txt")
### remove extra debugging files
for filename in otherfiles:
if filename[-9:] == ".norm.txt":
continue
elif filename[-8:] == ".ctf.txt":
continue
else:
apFile.removeFile(filename)
if maskhighpass and os.path.isfile(ace2inputpath):
apFile.removeFile(ace2inputpath)
return
def processImage(self, imgdata):
"""
time ./ctffind3.exe << eof
Input image file name [input.mrc] : 15aug13neil2_14jul14d_05sq_012hl_02ed-a.mrc
Output diagnostic filename
[diagnostic_output.mrc] : 15aug13neil2_14jul14d_05sq_012hl_02ed-a-pow.mrc
Pixel size [1.0] : 2.7
Acceleration voltage [300.0] : 300
Spherical aberration [2.7] : 2.7
Amplitude contrast [0.07] : 0.07
Size of power spectrum to compute [512] : 512
Minimum resolution [30.0] : 20
Maximum resolution [5.0] : 5
Minimum defocus [5000.0] :
Maximum defocus [50000.0] :
Defocus search step [500.0] :
Expected (tolerated) astigmatism [100.0] :
Find additional phase shift? [no] :
"""
paramInputOrder = ['input', 'output', 'apix', 'kv', 'cs', 'ampcontrast', 'fieldsize',
'resmin', 'resmax', 'defmin', 'defmax', 'defstep', 'expect_astig', 'phase', 'newline',]
#get Defocus in Angstroms
self.ctfvalues = {}
nominal = abs(imgdata['scope']['defocus']*-1.0e10)
ctfvalue = ctfdb.getBestCtfByResolution(imgdata)
if ctfvalue is not None:
"""
## CTFFIND V3.5 (7-March-2012) prefers the smaller of the two values for astigmatic images
I found that say you have an image with 1.1um and 1.5um defocus astigmatism. If you give
CTFFIND the average value of 1.3um for the defocus and 0.4um astig (dast) then it will
try to fit 1.3um and 1.8um, so you need to give it the minimum value (1.1um) for it to
fit 1.1um and 1.5um.
"""
bestdef = min(ctfvalue['defocus1'],ctfvalue['defocus2'])*1.0e10
else:
bestdef = nominal
if ctfvalue is not None and self.params['bestdb'] is True:
bestampcontrast = round(ctfvalue['amplitude_contrast'],3)
beststigdiff = round(abs(ctfvalue['defocus1'] - ctfvalue['defocus2'])*1e10,1)
else:
bestampcontrast = self.params['ampcontrast']
beststigdiff = self.params['dast']*10000.
if ctfvalue is not None and self.params['bestdb'] is True:
### set res max from resolution_80_percent
gmean = (ctfvalue['resolution_80_percent']*ctfvalue['resolution_50_percent']*self.params['resmax'])**(1/3.)
if gmean < self.params['resmin']*0.9:
# replace only if valid Issue #3291
self.params['resmax'] = round(gmean,2)
apDisplay.printColor("Setting resmax to the geometric mean of resolution values", "purple")
# dstep is the physical detector pixel size
apix = apDatabase.getPixelSize(imgdata)
# inputparams defocii and astig are in Angstroms
inputparams = {
'orig': os.path.join(imgdata['session']['image path'], imgdata['filename']+".mrc"),
'input': apDisplay.short(imgdata['filename'])+".mrc",
'output': apDisplay.short(imgdata['filename'])+"-pow.mrc",
'apix': apix,
'kv': imgdata['scope']['high tension']/1000.0,
'cs': self.params['cs'],
'ampcontrast': bestampcontrast,
'fieldsize': self.params['fieldsize'],
'resmin': self.params['resmin'],
'resmax': self.params['resmax'],
'defstep': self.params['defstep']*10000., #round(defocus/32.0, 1),
'expect_astig': beststigdiff,
'phase': 'no', # this is a secondary amp contrast term for phase plates
'newline': '\n',
}
defrange = self.params['defstep'] * self.params['numstep'] * 1e4 ## do 25 steps in either direction # in angstrum
inputparams['defmin']= round(bestdef-defrange, 1) #in angstrom
if inputparams['defmin'] < 0:
apDisplay.printWarning("Defocus minimum is less than zero")
inputparams['defmin'] = inputparams['defstep']
inputparams['defmax']= round(bestdef+defrange, 1) #in angstrom
apDisplay.printColor("Defocus search range: %d A to %d A (%.2f to %.2f um)"
%(inputparams['defmin'], inputparams['defmax'],
inputparams['defmin']*1e-4, inputparams['defmax']*1e-4), "cyan")
### secondary lock check right before it starts on the real part
if self.params['parallel'] and os.path.isfile(apDisplay.short(imgdata['filename'])+".mrc"):
# This is a secondary image lock check, checking the first output of the process.
# It alone is not good enough
apDisplay.printWarning('Some other parallel process is working on the same image. Skipping')
return
### create local link to image
if not os.path.exists(inputparams['input']):
os.symlink(inputparams['orig'], inputparams['input'])
if os.path.isfile(inputparams['output']):
# program crashes if this file exists
apFile.removeFile(inputparams['output'])
t0 = time.time()
apDisplay.printMsg("running ctf estimation at "+time.asctime())
for paramName in paramInputOrder:
apDisplay.printColor("%s = %s"%(paramName,inputparams[paramName]),"magenta")
print ""
ctfprogproc = subprocess.Popen(self.ctfprgmexe, shell=True, stdin=subprocess.PIPE,)
apDisplay.printColor(self.ctfprgmexe, "magenta")
for paramName in paramInputOrder:
apDisplay.printColor(inputparams[paramName],"magenta")
ctfprogproc.stdin.write(str(inputparams[paramName])+'\n')
ctfprogproc.communicate()
tdiff = time.time()-t0
apDisplay.printMsg("ctf estimation completed in "+apDisplay.timeString(tdiff))
if tdiff < 1.0:
apDisplay.printError("Failed to run CTFFIND4 program...")
### cannot run ctffind_plot_results.sh on CentOS 6
# This script requires gnuplot version >= 4.6, but you have version 4.2
### parse ctf estimation output
self.ctfvalues = {}
ctfproglog = apDisplay.short(imgdata['filename'])+"-pow.txt"
apDisplay.printMsg("reading %s"%(ctfproglog))
logf = open(ctfproglog, "r")
for line in logf:
sline = line.strip()
if sline.startswith('#'):
continue
bits = sline.split()
self.ctfvalues = {
'imagenum': int(float(bits[0])),
'defocus2': float(bits[1])*1e-10,
'defocus1': float(bits[2])*1e-10,
'angle_astigmatism': float(bits[3]),
'extra_phase': float(bits[4]),
'amplitude_contrast': inputparams['ampcontrast'],
'cross_correlation': float(bits[5]),
'ctffind4_resolution': float(bits[6]),
'defocusinit': bestdef*1e-10,
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
'confidence': float(bits[5]),
'confidence_d': round(math.sqrt(abs(float(bits[5]))), 5)
}
print self.ctfvalues
#convert powerspectra to JPEG
outputjpgbase = apDisplay.short(imgdata['filename'])+"-pow.jpg"
self.lastjpg = outputjpgbase
outputjpg = os.path.join(self.powerspecdir, self.lastjpg)
powspec = apImage.mrcToArray(inputparams['output'])
apImage.arrayToJpeg(powspec, outputjpg)
shutil.move(inputparams['output'], os.path.join(self.powerspecdir, inputparams['output']))
self.ctfvalues['graph1'] = outputjpg
#apFile.removeFile(inputparams['input'])
return
def tiltPhaseFlipParticles(self, imgdata, imgstackfile, partdatas):
apDisplay.printMsg("Applying per-particle CTF")
ctfvalue = ctfdb.getBestTiltCtfValueForImage(imgdata)
if ctfvalue is None:
apDisplay.printError("Failed to get ctf parameters")
apix = apDatabase.getPixelSize(imgdata)
ctfimgstackfile = os.path.join(self.params['rundir'], apDisplay.short(imgdata['filename'])+"-ctf.hed")
ampconst = ctfvalue['amplitude_contrast']
### calculate defocus at given position
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
CX = dimx/2
CY = dimy/2
if ctfvalue['tilt_axis_angle'] is not None:
N1 = -1.0 * math.sin( math.radians(ctfvalue['tilt_axis_angle']) )
N2 = math.cos( math.radians(ctfvalue['tilt_axis_angle']) )
else:
N1 = 0.0
N2 = 1.0
PSIZE = apix
### High tension on CM is given in kv instead of v so do not divide by 1000 in that case
if imgdata['scope']['tem']['name'] == "CM":
voltage = imgdata['scope']['high tension']
else:
voltage = (imgdata['scope']['high tension'])/1000
# find cs
cs = self.getCS(ctfvalue)
imagicdata = apImagicFile.readImagic(imgstackfile, msg=False)
ctfpartstack = []
for i in range(len(partdatas)):
partdata = partdatas[i]
prepartarray = imagicdata['images'][i]
prepartmrc = "rawpart.dwn.mrc"
postpartmrc = "ctfpart.dwn.mrc"
apImage.arrayToMrc(prepartarray, prepartmrc, msg = False)
### calculate ctf based on position
NX = partdata['xcoord']
NY = dimy-partdata['ycoord'] # reverse due to boxer flip
DX = CX - NX
DY = CY - NY
DF = (N1*DX + N2*DY) * PSIZE * math.tan( math.radians(ctfvalue['tilt_angle']) )
### defocus is in Angstroms
DFL1 = abs(ctfvalue['defocus1'])*1.0e10 + DF
DFL2 = abs(ctfvalue['defocus2'])*1.0e10 + DF
DF_final = (DFL1+DFL2)/2.0
### convert defocus to microns
defocus = DF_final*-1.0e-4
### check to make sure defocus is a reasonable value for applyctf
self.checkDefocus(defocus, apDisplay.short(imgdata['filename']))
parmstr = ("parm=%f,200,1,%.3f,0,17.4,9,1.53,%i,%.1f,%f"
%(defocus, ampconst, voltage, cs, apix))
emancmd = ("applyctf %s %s %s setparm flipphase" % (prepartmrc, postpartmrc, parmstr))
apEMAN.executeEmanCmd(emancmd, showcmd = False)
ctfpartarray = apImage.mrcToArray(postpartmrc, msg=False)
ctfpartstack.append(ctfpartarray)
apImagicFile.writeImagic(ctfpartstack, ctfimgstackfile)
return ctfimgstackfile
def runFindEM(imgdict, params, thread=False):
"""
runs a separate thread of findem.exe for each template
to get cross-correlation maps
"""
### check image
processAndSaveImage(imgdict, params)
dwnimgname = imgdict['filename'] + ".dwn.mrc"
if not os.path.isfile(dwnimgname):
apDisplay.printError("cound not find image to process: " + dwnimgname)
### FindEM crashes when an input image is longer than 76 characters
if len(dwnimgname) > 76:
randlink = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for x in range(10))
randlink += '.mrc'
os.symlink(dwnimgname, randlink)
### check template
if len(params['templatelist']) < 1:
apDisplay.printError("templatelist == 0; there are no templates")
joblist = []
ccmapfilelist = []
t0 = time.time()
for i, templatename in enumerate(params['templatelist']):
classavg = i + 1
#DETERMINE OUTPUT FILE NAME
#CHANGE THIS TO BE 00%i in future
numstr = "%03d" % classavg
#numstr = str(classavg%10)+"00"
ccmapfile = "cccmaxmap" + numstr + ".mrc"
apFile.removeFile(ccmapfile)
#GET FINDEM RUN COMMANDS
if len(dwnimgname) > 76:
feed = findEMString(classavg, templatename, randlink, ccmapfile,
params)
else:
feed = findEMString(classavg, templatename, dwnimgname, ccmapfile,
params)
#RUN THE PROGRAM
if thread is True:
job = findemjob(feed)
joblist.append(job)
job.start()
else:
execFindEM(feed)
#STORE OUTPUT FILE
ccmapfilelist.append(ccmapfile)
# get findemexe name
findemexepath = getFindEMPath()
findemexename = os.path.basename(findemexepath)
### WAIT FOR THREADS TO COMPLETE
if thread is True:
apDisplay.printMsg("waiting for " + str(len(joblist)) +
" findem threads to complete")
numtimes = 0
for i, job in enumerate(joblist):
while job.isAlive():
sys.stderr.write(".")
time.sleep(1.5)
numtimes += 1
if numtimes == 40:
pidof = "pidof %s" % findemexename
pids = subprocess.Popen(
pidof, shell=True,
stdout=subprocess.PIPE).stdout.read()
pids = pids.split()
for pid in pids:
os.kill(int(float(pid)), 9)
apDisplay.printWarning(
"\nFindEM likely stalled. Re-running the template correlator command\n"
)
runFindEM(imgdict, params, thread)
sys.stderr.write("\n")
apDisplay.printMsg("FindEM finished in " +
apDisplay.timeString(time.time() - t0))
### READ OUTPUT FILES
ccmaplist = []
for ccmapfile in ccmapfilelist:
if not os.path.isfile(ccmapfile):
apDisplay.printError("findem.exe did not run or crashed.\n")
ccmaxmap = apImage.mrcToArray(ccmapfile)
ccmaplist.append(ccmaxmap)
return ccmaplist
def processImage(self, imgdata):
"""
time ./ctffind3.exe << eof
micrograph.mrc
montage.pow
2.0, 200.0, 0.07, 60000, 7.0, 2 #! CS[mm], HT[kV], AmpCnst, XMAG, DStep[um], PAve
128, 200.0, 8.0, 5000.0, 40000.0, 5000.0 #! Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep
eof
CARD 1: Input file name for image
CARD 2: Output file name to check result
CARD 3: CS[mm], HT[kV], AmpCnst, XMAG, DStep[um],PAve
CARD 4: Box, ResMin[A], ResMax[A], dFMin[A], dFMax[A], FStep, dAst[A]
CTFTILT also asks for TiltA[deg], TiltR[deg] at CARD4
The output image file to check the result of the fitting
shows the filtered average power spectrum of the input
image in one half, and the fitted CTF (squared) in the
other half. The two halves should agree very well for a
successful fit.
CS: Spherical aberration coefficient of the objective in mm
HT: Electron beam voltage in kV
AmpCnst: Amount of amplitude contrast (fraction). For ice
images 0.07, for negative stain about 0.15.
XMAG: Magnification of original image
DStep: Pixel size on scanner in microns, or apix*mag/10000
PAve: Pixel averaging (PAve x PAve) for input image
Box: Tile size. The program divides the image into square
tiles and calculates the average power spectrum. Tiles
with a significantly higher or lower variance are
excluded; these are parts of the image which are unlikely
to contain useful information (beam edge, film number
etc). IMPORTANT: Box must have a even pixel dimensions.
ResMin: Low resolution end of data to be fitted.
ResMaX: High resolution end of data to be fitted.
dFMin: Starting defocus value for grid search in Angstrom.
Positive values represent an underfocus. The program
performs a systematic grid search of defocus values
and astigmatism before fitting a CTF to machine
precision.
dFMax: End defocus value for grid search in Angstrom.
FStep: Step width for grid search in Angstrom.
dAst: An additional parameter, dAst, was added to CARD 4 to restrain
the amount of astigmatism in the CTF fit. This makes the
fitting procedure more robust, especially in cases where
the Thon rings are not easily visible.
TiltA: guessed tilt angle
TiltR: angular range for initial coarse search
"""
#get Defocus in Angstroms
self.ctfvalues = {}
nominal = abs(imgdata['scope']['defocus'] * -1.0e10)
ctfvalue = ctfdb.getBestCtfByResolution(imgdata)
if ctfvalue is not None:
bestdef = abs(ctfvalue['defocus1'] +
ctfvalue['defocus2']) / 2.0 * 1.0e10
else:
bestdef = nominal
if ctfvalue is not None and self.params['bestdb'] is True:
bestampcontrast = ctfvalue['amplitude_contrast']
beststigdiff = abs(ctfvalue['defocus1'] -
ctfvalue['defocus2']) * 1e10
else:
bestampcontrast = self.params['amp' + self.params['medium']]
beststigdiff = self.params['dast']
# dstep is the physical detector pixel size
dstep = None
if 'camera' in imgdata and imgdata['camera'] and imgdata['camera'][
'pixel size']:
dstep = imgdata['camera']['pixel size']['x']
if dstep is None:
dstep = apDatabase.getPixelSize(
imgdata) * imgdata['scope']['magnification'] / 10000.0
dstep /= 1e6
dstep = float(dstep)
mpixelsize = apDatabase.getPixelSize(imgdata) * 1e-10
xmag = dstep / mpixelsize
apDisplay.printMsg("Xmag=%d, dstep=%.2e, mpix=%.2e" %
(xmag, dstep, mpixelsize))
inputparams = {
'orig':
os.path.join(imgdata['session']['image path'],
imgdata['filename'] + ".mrc"),
'input':
apDisplay.short(imgdata['filename']) + ".mrc",
'output':
apDisplay.short(imgdata['filename']) + "-pow.mrc",
'cs':
self.params['cs'],
'kv':
imgdata['scope']['high tension'] / 1000.0,
'ampcnst':
bestampcontrast,
'xmag':
xmag,
'dstep':
dstep * 1e6,
'pixavg':
self.params['bin'],
'box':
self.params['fieldsize'],
'resmin':
self.params['resmin'],
'resmax':
self.params['resmax'],
'defstep':
self.params['defstep'], #round(defocus/32.0, 1),
'dast':
beststigdiff,
}
defrange = self.params['defstep'] * self.params[
'numstep'] ## do 25 steps in either direction
inputparams['defmin'] = round(bestdef - defrange, 1) #in meters
if inputparams['defmin'] < 0:
apDisplay.printWarning("Defocus minimum is less than zero")
inputparams['defmin'] = inputparams['defstep']
inputparams['defmax'] = round(bestdef + defrange, 1) #in meters
apDisplay.printColor(
"Defocus search range: %d A to %d A (%.2f to %.2f um)" %
(inputparams['defmin'], inputparams['defmax'],
inputparams['defmin'] * 1e-4, inputparams['defmax'] * 1e-4),
"cyan")
### create local link to image
if not os.path.exists(inputparams['input']):
cmd = "ln -s " + inputparams['orig'] + " " + inputparams[
'input'] + "\n"
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
### make standard input for ctf estimation
line1cmd = inputparams['input'] + "\n"
line2cmd = inputparams['output'] + "\n"
line3cmd = (str(inputparams['cs']) + "," + str(inputparams['kv']) +
"," + str(inputparams['ampcnst']) + "," +
str(inputparams['xmag']) + "," +
str(inputparams['dstep']) + "," +
str(inputparams['pixavg']) + "\n")
line4cmd = (str(inputparams['box']) + "," +
str(inputparams['resmin']) + "," +
str(inputparams['resmax']) + "," +
str(inputparams['defmin']) + "," +
str(inputparams['defmax']) + "," +
str(inputparams['defstep']) + "," +
str(inputparams['dast']))
### additional ctftilt parameters
if self.params['ctftilt'] is True:
tiltang = apDatabase.getTiltAngleDeg(imgdata)
line4cmd += ("," + str(tiltang) + ",10")
line4cmd += "\n"
if os.path.isfile(inputparams['output']):
# program crashes if this file exists
apFile.removeFile(inputparams['output'])
t0 = time.time()
apDisplay.printMsg("running ctf estimation at " + time.asctime())
ctfproglog = os.path.join(
self.logdir,
os.path.splitext(imgdata['filename'])[0] + "-ctfprog.log")
logf = open(ctfproglog, "w")
ctfprogproc = subprocess.Popen(self.ctfprgmexe,
shell=True,
stdin=subprocess.PIPE,
stdout=logf)
apDisplay.printColor(self.ctfprgmexe, "magenta")
apDisplay.printColor(line1cmd.strip(), "magenta")
apDisplay.printColor(line2cmd.strip(), "magenta")
apDisplay.printColor(line3cmd.strip(), "magenta")
apDisplay.printColor(line4cmd.strip(), "magenta")
ctfprogproc.stdin.write(line1cmd)
ctfprogproc.stdin.write(line2cmd)
ctfprogproc.stdin.write(line3cmd)
ctfprogproc.stdin.write(line4cmd)
ctfprogproc.communicate()
logf.close()
apDisplay.printMsg("ctf estimation completed in " +
apDisplay.timeString(time.time() - t0))
#apFile.removeFile(inputparams['input'])
### parse ctf estimation output
self.ctfvalues = {}
logf = open(ctfproglog, "r")
## ctffind & ctftilt have diff # values
numvals = 6
if self.params['ctftilt'] is True:
numvals = 8
for line in logf:
sline = line.strip()
if sline[-12:] == "Final Values":
#print sline
if '**********' in sline:
sline = re.sub('**********', ' **********', sline)
bits = sline.split()
if len(bits) != numvals:
apDisplay.printError("wrong number of values in " +
str(bits))
for i, bit in enumerate(bits[0:(numvals - 2)]):
bits[i] = float(bit)
self.ctfvalues = {
'defocus1':
float(bits[0]) * 1e-10,
'defocus2':
float(bits[1]) * 1e-10,
# WARNING: this is the negative of the direct result
'angle_astigmatism':
float(bits[2]),
'amplitude_contrast':
inputparams['ampcnst'],
'cross_correlation':
float(bits[numvals - 3]),
'nominal':
nominal * 1e-10,
'defocusinit':
bestdef * 1e-10,
'cs':
self.params['cs'],
'volts':
imgdata['scope']['high tension'],
'confidence':
float(bits[numvals - 3]),
'confidence_d':
round(math.sqrt(abs(float(bits[numvals - 3]))), 5)
}
if self.params['ctftilt'] is True:
self.ctfvalues['tilt_axis_angle'] = float(bits[3])
self.ctfvalues['tilt_angle'] = float(bits[4])
### write to log file
f = open("ctfvalues.log", "a")
f.write("=== " + imgdata['filename'] + " ===\n")
line1 = ("nominal=%.1e, bestdef=%.1e," %
(self.ctfvalues['nominal'], self.ctfvalues['defocusinit']))
if self.params['ctftilt'] is True:
self.ctfvalues['origtiltang'] = tiltang
line1 += " tilt=%.1f," % tiltang
apDisplay.printMsg(line1)
f.write(line1)
line2 = (
"def_1=%.1e, def_2=%.1e, astig_angle=%.1f, cross_corr=%.3f,\n" %
(self.ctfvalues['defocus1'], self.ctfvalues['defocus2'],
self.ctfvalues['angle_astigmatism'],
self.ctfvalues['cross_correlation']))
if self.params['ctftilt'] is True:
line2 += ("tilt_angle=%.1f, tilt_axis_angle=%.1f,\n" %
(self.ctfvalues['tilt_angle'],
self.ctfvalues['tilt_axis_angle']))
apDisplay.printMsg(line2)
f.write(line2)
f.close()
#convert powerspectra to JPEG
outputjpgbase = os.path.basename(
os.path.splitext(inputparams['output'])[0] + ".jpg")
self.lastjpg = outputjpgbase
outputjpg = os.path.join(self.powerspecdir, self.lastjpg)
powspec = apImage.mrcToArray(inputparams['output'])
apImage.arrayToJpeg(powspec, outputjpg)
shutil.move(inputparams['output'],
os.path.join(self.powerspecdir, inputparams['output']))
self.ctfvalues['graph1'] = outputjpg
#apFile.removeFile(inputparams['input'])
return
def runFindEM(imgdict, params, thread=False):
"""
runs a separate thread of findem.exe for each template
to get cross-correlation maps
"""
### check image
processAndSaveImage(imgdict, params)
dwnimgname = imgdict['filename']+".dwn.mrc"
if not os.path.isfile(dwnimgname):
apDisplay.printError("cound not find image to process: "+dwnimgname)
### FindEM crashes when an input image is longer than 76 characters
if len(dwnimgname) > 76:
randlink = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(10))
randlink+= '.mrc'
os.symlink(dwnimgname, randlink)
### check template
if len(params['templatelist']) < 1:
apDisplay.printError("templatelist == 0; there are no templates")
joblist = []
ccmapfilelist = []
t0 = time.time()
for i,templatename in enumerate(params['templatelist']):
classavg = i + 1
#DETERMINE OUTPUT FILE NAME
#CHANGE THIS TO BE 00%i in future
numstr = "%03d" % classavg
#numstr = str(classavg%10)+"00"
ccmapfile="cccmaxmap"+numstr+".mrc"
apFile.removeFile(ccmapfile)
#GET FINDEM RUN COMMANDS
if len(dwnimgname) > 76:
feed = findEMString(classavg, templatename, randlink, ccmapfile, params)
else:
feed = findEMString(classavg, templatename, dwnimgname, ccmapfile, params)
#RUN THE PROGRAM
if thread is True:
job = findemjob(feed)
joblist.append(job)
job.start()
else:
execFindEM(feed)
#STORE OUTPUT FILE
ccmapfilelist.append(ccmapfile)
# get findemexe name
findemexepath = getFindEMPath()
findemexename = os.path.basename(findemexepath)
### WAIT FOR THREADS TO COMPLETE
if thread is True:
apDisplay.printMsg("waiting for "+str(len(joblist))+" findem threads to complete")
numtimes = 0
for i,job in enumerate(joblist):
while job.isAlive():
sys.stderr.write(".")
time.sleep(1.5)
numtimes+=1
if numtimes == 40:
pidof = "pidof %s" % findemexename
pids=subprocess.Popen(pidof, shell=True,stdout=subprocess.PIPE).stdout.read()
pids = pids.split()
for pid in pids:
os.kill(int(float(pid)),9)
apDisplay.printWarning("\nFindEM likely stalled. Re-running the template correlator command\n")
runFindEM(imgdict,params,thread)
sys.stderr.write("\n")
apDisplay.printMsg("FindEM finished in "+apDisplay.timeString(time.time()-t0))
### READ OUTPUT FILES
ccmaplist = []
for ccmapfile in ccmapfilelist:
if not os.path.isfile(ccmapfile):
apDisplay.printError("findem.exe did not run or crashed.\n")
ccmaxmap = apImage.mrcToArray(ccmapfile)
ccmaplist.append(ccmaxmap)
return ccmaplist