def processImage(self, imgdata, filtarray):
if abs(self.params['apix'] - self.params['templateapix']) > 0.01:
#rescale templates, apix has changed
apTemplate.getTemplates(self.params)
### RUN FindEM
### save filter image to .dwn.mrc
imgpath = os.path.join(self.params['rundir'], imgdata['filename']+".dwn.mrc")
apImage.arrayToMrc(filtarray, imgpath, msg=False)
### run FindEM
looptdiff = time.time()-self.proct0
self.proct0 = time.time()
proctdiff = time.time()-self.proct0
f = open("template_image_timing.dat", "a")
datstr = "%d\t%.5f\t%.5f\n"%(self.stats['count'], proctdiff, looptdiff)
f.write(datstr)
f.close()
### run Template Correlation program
cclist = self.runTemplateCorrelator(imgdata)
### find peaks in map
peaktree = self.findPeaks(imgdata, cclist)
return peaktree
def processImage(self, imgdata, filtarray):
filtfile = os.path.join(self.params['rundir'],
imgdata['filename'] + ".dwn.mrc")
if not os.path.isfile(filtfile):
apImage.arrayToMrc(filtarray, filtfile, msg=False)
peaktree = self.runManualPicker(imgdata)
return peaktree
def processImage(self, imgdata, filtarray):
if abs(self.params['apix'] - self.params['templateapix']) > 0.01:
#rescale templates, apix has changed
apTemplate.getTemplates(self.params)
### RUN FindEM
### save filter image to .dwn.mrc
imgpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".dwn.mrc")
apImage.arrayToMrc(filtarray, imgpath, msg=False)
### run FindEM
looptdiff = time.time() - self.proct0
self.proct0 = time.time()
proctdiff = time.time() - self.proct0
f = open("template_image_timing.dat", "a")
datstr = "%d\t%.5f\t%.5f\n" % (self.stats['count'], proctdiff,
looptdiff)
f.write(datstr)
f.close()
### run Template Correlation program
cclist = self.runTemplateCorrelator(imgdata)
### find peaks in map
peaktree = self.findPeaks(imgdata, cclist)
return peaktree
def processImage(self, imgdata, filtarray):
if abs(self.params['apix'] - self.params['templateapix']) > 0.01:
#rescale templates, apix has changed
apTemplate.getTemplates(self.params)
### RUN Signature
### save filter image to .dwn.mrc
imgpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".dwn.mrc")
apImage.arrayToMrc(filtarray, imgpath, msg=False)
### run Signature
looptdiff = time.time() - self.proct0
self.proct0 = time.time()
plist = apSignature.runSignature(imgdata, self.params)
proctdiff = time.time() - self.proct0
f = open("template_image_timing.dat", "a")
datstr = "%d\t%.5f\t%.5f\n" % (self.stats['count'], proctdiff,
looptdiff)
f.write(datstr)
f.close()
# convert list of particles to peaktree
peaktree = apSignature.partToPeakTree(plist, self.params['bin'])
return peaktree
def createAlignedReferenceStack(self):
searchstr = "part"+self.params['timestamp']+"_ref0*.xmp"
files = glob.glob(searchstr)
files.sort()
stack = []
reflist = self.readRefDocFile()
for i in range(len(files)):
fname = files[i]
refdict = reflist[i]
if refdict['partnum'] != i+1:
print i, refdict['partnum']
apDisplay.printError("sorting error in reflist, see neil")
refarray = spider.read(fname)
xyshift = (refdict['xshift'], refdict['yshift'])
alignrefarray = apImage.xmippTransform(refarray, rot=refdict['inplane'],
shift=xyshift, mirror=refdict['mirror'])
stack.append(alignrefarray)
stackarray = numpy.asarray(stack, dtype=numpy.float32)
#print stackarray.shape
avgstack = "part"+self.params['timestamp']+"_average.hed"
apFile.removeStack(avgstack, warn=False)
apImagicFile.writeImagic(stackarray, avgstack)
### create a average mrc
avgdata = stackarray.mean(0)
apImage.arrayToMrc(avgdata, "average.mrc")
return
def getShift(imgdata1, imgdata2):
# assumes images are square
print "Finding shift between", apDisplay.short(imgdata1["filename"]), "and", apDisplay.short(imgdata2["filename"])
dimension1 = imgdata1["camera"]["dimension"]["x"]
binning1 = imgdata1["camera"]["binning"]["x"]
dimension2 = imgdata2["camera"]["dimension"]["x"]
binning2 = imgdata2["camera"]["binning"]["x"]
finalsize = 512
# test to make sure images are at same mag
if imgdata1["scope"]["magnification"] != imgdata2["scope"]["magnification"]:
apDisplay.printWarning("Defocus pairs are at different magnifications, so shift can't be calculated.")
return None
# test to see if images capture the same area
if (dimension1 * binning1) != (dimension2 * binning2):
apDisplay.printWarning("Defocus pairs do not capture the same imaging area, so shift can't be calculated.")
return None
# images must not be less than finalsize (currently 512) pixels. This is arbitrary but for good reason
if dimension1 < finalsize or dimension2 < finalsize:
apDisplay.printWarning("Images must be greater than " + finalsize + " pixels to calculate shift.")
return None
# low pass filter 2 images to twice the final pixelsize BEFORE binning
shrinkfactor1 = dimension1 / finalsize
shrinkfactor2 = dimension2 / finalsize
binned1 = apImage.filterImg(imgdata1["image"], 1.0, shrinkfactor1 * 2)
binned2 = apImage.filterImg(imgdata2["image"], 1.0, shrinkfactor2 * 2)
# now bin 2 images
binned1 = apImage.binImg(binned1, shrinkfactor1)
binned2 = apImage.binImg(binned2, shrinkfactor2)
### fix for non-square images, correlation fails on non-square images
mindim = min(binned1.shape)
binned1 = binned1[:mindim, :mindim]
binned2 = binned2[:mindim, :mindim]
### use phase correlation, performs better than cross
pc = correlator.phase_correlate(binned1, binned2)
apImage.arrayToMrc(pc, "phaseCorrelate.mrc")
### find peak, filtering to 10.0 helps
peak = peakfinder.findSubpixelPeak(pc, lpf=10.0)
subpixpeak = peak["subpixel peak"]
shift = correlator.wrap_coord(subpixpeak, pc.shape)
peak["scalefactor"] = dimension2 / float(dimension1)
# print shift[0]*shrinkfactor1, shift[1]*shrinkfactor1
xshift = int(round(shift[0] * shrinkfactor1))
yshift = int(round(shift[1] * shrinkfactor1))
peak["shift"] = numpy.array((xshift, yshift))
apDisplay.printMsg("Determined shifts: %f %f" % (peak["shift"][0], peak["shift"][1]))
# print peak['shift']
# sys.exit(1)
return peak
def processAndSaveImage(imgdata, params): imgpath = os.path.join(params['rundir'], imgdata['filename']+".dwn.mrc") if os.path.isfile(imgpath): return False #downsize and filter leginon image if params['uncorrected']: imgarray = apImage.correctImage(imgdata, params) else: imgarray = imgdata['image'] imgarray = apImage.preProcessImage(imgarray, params=params, msg=False) apImage.arrayToMrc(imgarray, imgpath, msg=False) return True
def processAndSaveImage(imgdata, params):
imgpath = os.path.join(params['rundir'], imgdata['filename'] + ".dwn.mrc")
if os.path.isfile(imgpath):
return False
#downsize and filter leginon image
if params['uncorrected']:
imgarray = apDBImage.correctImage(imgdata)
else:
imgarray = imgdata['image']
imgarray = apImage.preProcessImage(imgarray, params=params, msg=False)
apImage.arrayToMrc(imgarray, imgpath, msg=False)
return True
def runTestShift(img1name, img2name, imgpath, tiltdiff, coord): img1path = os.path.join(imgpath, img1name) img2path = os.path.join(imgpath, img2name) print img1path print img2path img1 = apImage.binImg(apImage.mrcToArray(img1path),4) img2 = apImage.binImg(apImage.mrcToArray(img2path),4) apImage.arrayToMrc(img1,"img1a-raw.mrc") apImage.arrayToMrc(img2,"img2a-raw.mrc") origin, newpart, snr = apTiltShift.getTiltedCoordinates(img1, img2, tiltdiff, coord) apImage.arrayToJpegPlusPeak(img1, "img1a-guess.jpg", (origin[1], origin[0])) apImage.arrayToJpegPlusPeak(img2, "img2a-guess.jpg", (newpart[1], newpart[0]))
def loopProcessImage(self, imgdata):
"""
setup like this to override things
"""
self.filtimgpath = os.path.join(self.params['rundir'], imgdata['filename']+'.dwn.mrc')
if os.path.isfile(self.filtimgpath) and self.params['continue'] is True:
apDisplay.printMsg("reading filtered image from mrc file")
self.filtarray = apImage.mrcToArray(self.filtimgpath, msg=False)
else:
self.filtarray = apImage.preProcessImage(imgdata['image'], apix=self.params['apix'], params=self.params)
apImage.arrayToMrc(self.filtarray, self.filtimgpath)
peaktree = self.processImage(imgdata, self.filtarray)
return peaktree
def createReferenceStack(self):
avgstack = "part"+self.timestamp+"_average.hed"
apFile.removeStack(avgstack, warn=False)
searchstr = "part"+self.timestamp+"_ref0*.xmp"
files = glob.glob(searchstr)
if len(files) == 0:
apDisplay.printError("Xmipp did not run")
files.sort()
stack = []
for i in range(len(files)):
fname = files[i]
refarray = spider.read(fname)
stack.append(refarray)
apImagicFile.writeImagic(stack, avgstack)
### create a average mrc
stackarray = numpy.asarray(stack, dtype=numpy.float32)
avgdata = stackarray.mean(0)
apImage.arrayToMrc(avgdata, "average.mrc")
return
def getOriginalImagePath(self, imgdata):
'''
This function gives back the image path to be used for boxing.
Three possible results:
1. image from the leginon/session/rawdata as recorded in imgdata (typical)
2. -darknorm.dwn.mrc previously created
3. -darknorm.dwn.mrc made from dd frames.
'''
# default path
imgpath = os.path.join(imgdata['session']['image path'], imgdata['filename']+".mrc")
if self.is_dd:
### dark/bright correct image
tmpname = self.shortname+"-darknorm.dwn.mrc"
imgpath = os.path.join(self.params['rundir'], tmpname)
if not self.params['usedownmrc'] or not os.path.isfile(imgpath):
# make downmrc
imgarray = self.getDDImageArray(imgdata)
apImage.arrayToMrc(imgarray, imgpath)
apDisplay.printMsg('Boxing is done on %s' % (imgpath,))
return imgpath
def loopProcessImage(self, imgdata):
"""
setup like this to override things
"""
self.filtimgpath = os.path.join(self.params['rundir'],
imgdata['filename'] + '.dwn.mrc')
if os.path.isfile(
self.filtimgpath) and self.params['continue'] is True:
apDisplay.printMsg("reading filtered image from mrc file")
self.filtarray = apImage.mrcToArray(self.filtimgpath, msg=False)
else:
self.filtarray = apImage.preProcessImage(imgdata['image'],
apix=self.params['apix'],
params=self.params)
apImage.arrayToMrc(self.filtarray, self.filtimgpath)
peaktree = self.processImage(imgdata, self.filtarray)
return peaktree
def processImage(self, imgdata, filtarray):
if abs(self.params['apix'] - self.params['templateapix']) > 0.01:
#rescale templates, apix has changed
apTemplate.getTemplates(self.params)
### RUN Signature
### save filter image to .dwn.mrc
imgpath = os.path.join(self.params['rundir'], imgdata['filename']+".dwn.mrc")
apImage.arrayToMrc(filtarray, imgpath, msg=False)
### run Signature
looptdiff = time.time()-self.proct0
self.proct0 = time.time()
plist = apSignature.runSignature(imgdata, self.params)
proctdiff = time.time()-self.proct0
f = open("template_image_timing.dat", "a")
datstr = "%d\t%.5f\t%.5f\n"%(self.stats['count'], proctdiff, looptdiff)
f.write(datstr)
f.close()
# convert list of particles to peaktree
peaktree = apSignature.partToPeakTree(plist,self.params['bin'])
return peaktree
def tiltPhaseFlipParticles(self, imgdata, imgstackfile, partdatas):
apDisplay.printMsg("Applying per-particle CTF")
ctfvalue = ctfdb.getBestTiltCtfValueForImage(imgdata)
if ctfvalue is None:
apDisplay.printError("Failed to get ctf parameters")
apix = apDatabase.getPixelSize(imgdata)
ctfimgstackfile = os.path.join(self.params['rundir'], apDisplay.short(imgdata['filename'])+"-ctf.hed")
ampconst = ctfvalue['amplitude_contrast']
### calculate defocus at given position
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
CX = dimx/2
CY = dimy/2
if ctfvalue['tilt_axis_angle'] is not None:
N1 = -1.0 * math.sin( math.radians(ctfvalue['tilt_axis_angle']) )
N2 = math.cos( math.radians(ctfvalue['tilt_axis_angle']) )
else:
N1 = 0.0
N2 = 1.0
PSIZE = apix
### High tension on CM is given in kv instead of v so do not divide by 1000 in that case
if imgdata['scope']['tem']['name'] == "CM":
voltage = imgdata['scope']['high tension']
else:
voltage = (imgdata['scope']['high tension'])/1000
# find cs
cs = self.getCS(ctfvalue)
imagicdata = apImagicFile.readImagic(imgstackfile, msg=False)
ctfpartstack = []
for i in range(len(partdatas)):
partdata = partdatas[i]
prepartarray = imagicdata['images'][i]
prepartmrc = "rawpart.dwn.mrc"
postpartmrc = "ctfpart.dwn.mrc"
apImage.arrayToMrc(prepartarray, prepartmrc, msg = False)
### calculate ctf based on position
NX = partdata['xcoord']
NY = dimy-partdata['ycoord'] # reverse due to boxer flip
DX = CX - NX
DY = CY - NY
DF = (N1*DX + N2*DY) * PSIZE * math.tan( math.radians(ctfvalue['tilt_angle']) )
### defocus is in Angstroms
DFL1 = abs(ctfvalue['defocus1'])*1.0e10 + DF
DFL2 = abs(ctfvalue['defocus2'])*1.0e10 + DF
DF_final = (DFL1+DFL2)/2.0
### convert defocus to microns
defocus = DF_final*-1.0e-4
### check to make sure defocus is a reasonable value for applyctf
self.checkDefocus(defocus, apDisplay.short(imgdata['filename']))
parmstr = ("parm=%f,200,1,%.3f,0,17.4,9,1.53,%i,%.1f,%f"
%(defocus, ampconst, voltage, cs, apix))
emancmd = ("applyctf %s %s %s setparm flipphase" % (prepartmrc, postpartmrc, parmstr))
apEMAN.executeEmanCmd(emancmd, showcmd = False)
ctfpartarray = apImage.mrcToArray(postpartmrc, msg=False)
ctfpartstack.append(ctfpartarray)
apImagicFile.writeImagic(ctfpartstack, ctfimgstackfile)
return ctfimgstackfile
def processImage(self, imgdata):
self.ctfvalues = {}
bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True)
apix = apDatabase.getPixelSize(imgdata)
if (not (self.params['onepass'] and self.params['zeropass'])):
maskhighpass = False
ace2inputpath = os.path.join(imgdata['session']['image path'],
imgdata['filename'] + ".mrc")
else:
maskhighpass = True
filterimg = apImage.maskHighPassFilter(imgdata['image'], apix, 1,
self.params['zeropass'],
self.params['onepass'])
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
mrc.write(filterimg, ace2inputpath)
# make sure that the image is a square
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
if dimx != dimy:
dims = [dimx, dimy]
dims.sort()
apDisplay.printMsg("resizing image: %ix%i to %ix%i" %
(dimx, dimy, dims[0], dims[0]))
mrcarray = apImage.mrcToArray(ace2inputpath, msg=False)
clippedmrc = apImage.frame_cut(mrcarray, [dims[0], dims[0]])
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
apImage.arrayToMrc(clippedmrc, ace2inputpath, msg=False)
### pad out image to speed up FFT calculations for non-standard image sizes
print "checking prime factor"
if primefactor.isGoodStack(dimx) is False:
goodsize = primefactor.getNextEvenPrime(dimx)
factor = float(goodsize) / float(dimx)
apDisplay.printMsg("padding image: %ix%i to %ix%i" %
(dimx, dimy, dimx * factor, dimy * factor))
mrcarray = apImage.mrcToArray(ace2inputpath, msg=False)
# paddedmrc = imagefun.pad(mrcarray, None, factor)
paddedmrc = apImage.frame_constant(mrcarray,
(dimx * factor, dimy * factor),
cval=mrcarray.mean())
ace2inputpath = os.path.join(self.params['rundir'],
imgdata['filename'] + ".mrc")
apImage.arrayToMrc(paddedmrc, ace2inputpath, msg=False)
inputparams = {
'input': ace2inputpath,
'cs': self.params['cs'],
'kv': imgdata['scope']['high tension'] / 1000.0,
'apix': apix,
'binby': self.params['bin'],
}
### make standard input for ACE 2
apDisplay.printMsg("Ace2 executable: " + self.ace2exe)
commandline = (self.ace2exe + " -i " + str(inputparams['input']) +
" -b " + str(inputparams['binby']) + " -c " +
str(inputparams['cs']) + " -k " +
str(inputparams['kv']) + " -a " +
str(inputparams['apix']) + " -e " +
str(self.params['edge_b']) + "," +
str(self.params['edge_t']) + " -r " +
str(self.params['rotblur']) + "\n")
### run ace2
apDisplay.printMsg("running ace2 at " + time.asctime())
apDisplay.printColor(commandline, "purple")
t0 = time.time()
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline,
shell=True,
stderr=aceerrf,
stdout=aceoutf)
ace2proc.wait()
### check if ace2 worked
basename = os.path.basename(ace2inputpath)
imagelog = basename + ".ctf.txt"
if not os.path.isfile(imagelog) and self.stats['count'] <= 1:
### ace2 always crashes on first image??? .fft_wisdom file??
time.sleep(1)
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline,
shell=True,
stderr=aceerrf,
stdout=aceoutf)
ace2proc.wait()
if self.params['verbose'] is False:
aceoutf.close()
aceerrf.close()
if not os.path.isfile(imagelog):
lddcmd = "ldd " + self.ace2exe
lddproc = subprocess.Popen(lddcmd, shell=True)
lddproc.wait()
apDisplay.printError("ace2 did not run")
apDisplay.printMsg("ace2 completed in " +
apDisplay.timeString(time.time() - t0))
### parse log file
self.ctfvalues = {
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
}
logf = open(imagelog, "r")
apDisplay.printMsg("reading log file %s" % (imagelog))
for line in logf:
sline = line.strip()
if re.search("^Final Defocus: ", sline):
### old ACE2
apDisplay.printError(
"This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now"
)
#parts = sline.split()
#self.ctfvalues['defocus1'] = float(parts[2])
#self.ctfvalues['defocus2'] = float(parts[3])
### convert to degrees
#self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4]))
elif re.search("^Final Defocus \(m,m,deg\):", sline):
### new ACE2
apDisplay.printMsg("Reading new ACE2 defocus")
parts = sline.split()
#print parts
self.ctfvalues['defocus1'] = float(parts[3])
self.ctfvalues['defocus2'] = float(parts[4])
# ace2 defines negative angle from +x toward +y
self.ctfvalues['angle_astigmatism'] = -float(parts[5])
elif re.search("^Amplitude Contrast:", sline):
parts = sline.split()
self.ctfvalues['amplitude_contrast'] = float(parts[2])
elif re.search("^Confidence:", sline):
parts = sline.split()
self.ctfvalues['confidence'] = float(parts[1])
self.ctfvalues['confidence_d'] = float(parts[1])
logf.close()
### summary stats
apDisplay.printMsg("============")
avgdf = (self.ctfvalues['defocus1'] + self.ctfvalues['defocus2']) / 2.0
ampconst = 100.0 * self.ctfvalues['amplitude_contrast']
pererror = 100.0 * (self.ctfvalues['defocus1'] -
self.ctfvalues['defocus2']) / avgdf
apDisplay.printMsg(
"Defocus: %.3f x %.3f um (%.2f percent astigmatism)" %
(self.ctfvalues['defocus1'] * 1.0e6,
self.ctfvalues['defocus2'] * 1.0e6, pererror))
apDisplay.printMsg("Angle astigmatism: %.2f degrees" %
(self.ctfvalues['angle_astigmatism']))
apDisplay.printMsg("Amplitude contrast: %.2f percent" % (ampconst))
apDisplay.printColor(
"Final confidence: %.3f" % (self.ctfvalues['confidence']), 'cyan')
### double check that the values are reasonable
if avgdf > self.params['maxdefocus'] or avgdf < self.params[
'mindefocus']:
apDisplay.printWarning(
"bad defocus estimate, not committing values to database")
self.badprocess = True
if ampconst < 0.0 or ampconst > 80.0:
apDisplay.printWarning(
"bad amplitude contrast, not committing values to database")
self.badprocess = True
if self.ctfvalues['confidence'] < 0.2:
apDisplay.printWarning(
"bad confidence value, not committing values to database")
self.badprocess = True
## create power spectra jpeg
mrcfile = imgdata['filename'] + ".mrc.edge.mrc"
if os.path.isfile(mrcfile):
jpegfile = os.path.join(
self.powerspecdir,
apDisplay.short(imgdata['filename']) + ".jpg")
ps = apImage.mrcToArray(mrcfile, msg=False)
c = numpy.array(ps.shape) / 2.0
ps[c[0] - 0, c[1] - 0] = ps.mean()
ps[c[0] - 1, c[1] - 0] = ps.mean()
ps[c[0] - 0, c[1] - 1] = ps.mean()
ps[c[0] - 1, c[1] - 1] = ps.mean()
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
ps = numpy.log(ps + 1.0)
ps = (ps - ps.mean()) / ps.std()
cutoff = -2.0 * ps.min()
ps = numpy.where(ps > cutoff, cutoff, ps)
cutoff = ps.mean()
ps = numpy.where(ps < cutoff, cutoff, ps)
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
apImage.arrayToJpeg(ps, jpegfile, msg=False)
apFile.removeFile(mrcfile)
self.ctfvalues['graph3'] = jpegfile
otherfiles = glob.glob(imgdata['filename'] + ".*.txt")
### remove extra debugging files
for filename in otherfiles:
if filename[-9:] == ".norm.txt":
continue
elif filename[-8:] == ".ctf.txt":
continue
else:
apFile.removeFile(filename)
if maskhighpass and os.path.isfile(ace2inputpath):
apFile.removeFile(ace2inputpath)
return
def processImage(self, imgdata):
self.ctfvalues = {}
bestdef = ctfdb.getBestCtfByResolution(imgdata, msg=True)
apix = apDatabase.getPixelSize(imgdata)
if (not (self.params['onepass'] and self.params['zeropass'])):
maskhighpass = False
ace2inputpath = os.path.join(imgdata['session']['image path'],imgdata['filename']+".mrc")
else:
maskhighpass = True
filterimg = apImage.maskHighPassFilter(imgdata['image'],apix,1,self.params['zeropass'],self.params['onepass'])
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
mrc.write(filterimg,ace2inputpath)
# make sure that the image is a square
dimx = imgdata['camera']['dimension']['x']
dimy = imgdata['camera']['dimension']['y']
if dimx != dimy:
dims = [dimx,dimy]
dims.sort()
apDisplay.printMsg("resizing image: %ix%i to %ix%i" % (dimx,dimy,dims[0],dims[0]))
mrcarray = apImage.mrcToArray(ace2inputpath,msg=False)
clippedmrc = apImage.frame_cut(mrcarray,[dims[0],dims[0]])
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
apImage.arrayToMrc(clippedmrc,ace2inputpath,msg=False)
### pad out image to speed up FFT calculations for non-standard image sizes
print "checking prime factor"
if primefactor.isGoodStack(dimx) is False:
goodsize = primefactor.getNextEvenPrime(dimx)
factor = float(goodsize) / float(dimx)
apDisplay.printMsg("padding image: %ix%i to %ix%i" % (dimx,dimy,dimx*factor,dimy*factor))
mrcarray = apImage.mrcToArray(ace2inputpath,msg=False)
# paddedmrc = imagefun.pad(mrcarray, None, factor)
paddedmrc = apImage.frame_constant(mrcarray, (dimx*factor,dimy*factor), cval=mrcarray.mean())
ace2inputpath = os.path.join(self.params['rundir'],imgdata['filename']+".mrc")
apImage.arrayToMrc(paddedmrc,ace2inputpath,msg=False)
inputparams = {
'input': ace2inputpath,
'cs': self.params['cs'],
'kv': imgdata['scope']['high tension']/1000.0,
'apix': apix,
'binby': self.params['bin'],
}
### make standard input for ACE 2
apDisplay.printMsg("Ace2 executable: "+self.ace2exe)
commandline = ( self.ace2exe
+ " -i " + str(inputparams['input'])
+ " -b " + str(inputparams['binby'])
+ " -c " + str(inputparams['cs'])
+ " -k " + str(inputparams['kv'])
+ " -a " + str(inputparams['apix'])
+ " -e " + str(self.params['edge_b'])+","+str(self.params['edge_t'])
+ " -r " + str(self.params['rotblur'])
+ "\n" )
### run ace2
apDisplay.printMsg("running ace2 at "+time.asctime())
apDisplay.printColor(commandline, "purple")
t0 = time.time()
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf)
ace2proc.wait()
### check if ace2 worked
basename = os.path.basename(ace2inputpath)
imagelog = basename+".ctf.txt"
if not os.path.isfile(imagelog) and self.stats['count'] <= 1:
### ace2 always crashes on first image??? .fft_wisdom file??
time.sleep(1)
if self.params['verbose'] is True:
ace2proc = subprocess.Popen(commandline, shell=True)
else:
aceoutf = open("ace2.out", "a")
aceerrf = open("ace2.err", "a")
ace2proc = subprocess.Popen(commandline, shell=True, stderr=aceerrf, stdout=aceoutf)
ace2proc.wait()
if self.params['verbose'] is False:
aceoutf.close()
aceerrf.close()
if not os.path.isfile(imagelog):
lddcmd = "ldd "+self.ace2exe
lddproc = subprocess.Popen(lddcmd, shell=True)
lddproc.wait()
apDisplay.printError("ace2 did not run")
apDisplay.printMsg("ace2 completed in " + apDisplay.timeString(time.time()-t0))
### parse log file
self.ctfvalues = {
'cs': self.params['cs'],
'volts': imgdata['scope']['high tension'],
}
logf = open(imagelog, "r")
apDisplay.printMsg("reading log file %s"%(imagelog))
for line in logf:
sline = line.strip()
if re.search("^Final Defocus: ", sline):
### old ACE2
apDisplay.printError("This old version of ACE2 has a bug in the astigmastism, please upgrade ACE2 now")
#parts = sline.split()
#self.ctfvalues['defocus1'] = float(parts[2])
#self.ctfvalues['defocus2'] = float(parts[3])
### convert to degrees
#self.ctfvalues['angle_astigmatism'] = math.degrees(float(parts[4]))
elif re.search("^Final Defocus \(m,m,deg\):", sline):
### new ACE2
apDisplay.printMsg("Reading new ACE2 defocus")
parts = sline.split()
#print parts
self.ctfvalues['defocus1'] = float(parts[3])
self.ctfvalues['defocus2'] = float(parts[4])
# ace2 defines negative angle from +x toward +y
self.ctfvalues['angle_astigmatism'] = -float(parts[5])
elif re.search("^Amplitude Contrast:",sline):
parts = sline.split()
self.ctfvalues['amplitude_contrast'] = float(parts[2])
elif re.search("^Confidence:",sline):
parts = sline.split()
self.ctfvalues['confidence'] = float(parts[1])
self.ctfvalues['confidence_d'] = float(parts[1])
logf.close()
### summary stats
apDisplay.printMsg("============")
avgdf = (self.ctfvalues['defocus1']+self.ctfvalues['defocus2'])/2.0
ampconst = 100.0*self.ctfvalues['amplitude_contrast']
pererror = 100.0 * (self.ctfvalues['defocus1']-self.ctfvalues['defocus2']) / avgdf
apDisplay.printMsg("Defocus: %.3f x %.3f um (%.2f percent astigmatism)"%
(self.ctfvalues['defocus1']*1.0e6, self.ctfvalues['defocus2']*1.0e6, pererror ))
apDisplay.printMsg("Angle astigmatism: %.2f degrees"%(self.ctfvalues['angle_astigmatism']))
apDisplay.printMsg("Amplitude contrast: %.2f percent"%(ampconst))
apDisplay.printColor("Final confidence: %.3f"%(self.ctfvalues['confidence']),'cyan')
### double check that the values are reasonable
if avgdf > self.params['maxdefocus'] or avgdf < self.params['mindefocus']:
apDisplay.printWarning("bad defocus estimate, not committing values to database")
self.badprocess = True
if ampconst < 0.0 or ampconst > 80.0:
apDisplay.printWarning("bad amplitude contrast, not committing values to database")
self.badprocess = True
if self.ctfvalues['confidence'] < 0.2:
apDisplay.printWarning("bad confidence value, not committing values to database")
self.badprocess = True
## create power spectra jpeg
mrcfile = imgdata['filename']+".mrc.edge.mrc"
if os.path.isfile(mrcfile):
jpegfile = os.path.join(self.powerspecdir, apDisplay.short(imgdata['filename'])+".jpg")
ps = apImage.mrcToArray(mrcfile,msg=False)
c = numpy.array(ps.shape)/2.0
ps[c[0]-0,c[1]-0] = ps.mean()
ps[c[0]-1,c[1]-0] = ps.mean()
ps[c[0]-0,c[1]-1] = ps.mean()
ps[c[0]-1,c[1]-1] = ps.mean()
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
ps = numpy.log(ps+1.0)
ps = (ps-ps.mean())/ps.std()
cutoff = -2.0*ps.min()
ps = numpy.where(ps > cutoff, cutoff, ps)
cutoff = ps.mean()
ps = numpy.where(ps < cutoff, cutoff, ps)
#print "%.3f -- %.3f -- %.3f"%(ps.min(), ps.mean(), ps.max())
apImage.arrayToJpeg(ps, jpegfile, msg=False)
apFile.removeFile(mrcfile)
self.ctfvalues['graph3'] = jpegfile
otherfiles = glob.glob(imgdata['filename']+".*.txt")
### remove extra debugging files
for filename in otherfiles:
if filename[-9:] == ".norm.txt":
continue
elif filename[-8:] == ".ctf.txt":
continue
else:
apFile.removeFile(filename)
if maskhighpass and os.path.isfile(ace2inputpath):
apFile.removeFile(ace2inputpath)
return
def fakeOutput(imgname, ccmapfile, params):
a = apImage.mrcToArray(imgname)
a = numpy.zeros(a.shape)
apImage.arrayToMrc(a, ccmapfile)
apFile.safeCopy('/home/acheng/Projects/Gfindem/example.box',
getBoxFileName(ccmapfile))
def fakeOutput(imgname,ccmapfile,params):
a = apImage.mrcToArray(imgname)
a = numpy.zeros(a.shape)
apImage.arrayToMrc(a,ccmapfile)
apFile.safeCopy('/home/acheng/Projects/Gfindem/example.box', getBoxFileName(ccmapfile))
def getTemplates(params):
"""
Inputs:
params['templateIds'], a list of template ids
params['apix'], desired pixel size
params['rundir'], output directory
image processing params
Processing:
Copies, scales, and filters templates
Outputs:
params['templatelist'], a list of template file basenames
"""
apDisplay.printMsg("getting templates")
if not params['templateIds']:
apDisplay.printError("No template ids were specified")
params['templatelist'] = [] #list of scaled files
for i, templateid in enumerate(params['templateIds']):
index = i + 1
#print templateid
templateid = int(templateid)
if templateid < 0:
continue
#QUERY DB FOR TEMPLATE INFO
templatedata = appiondata.ApTemplateImageData.direct_query(
abs(templateid))
if not (templatedata):
apDisplay.printError("Template Id " + str(templateid) +
" was not found in database.")
#COPY THE FILE OVER
origtemplatepath = os.path.join(templatedata['path']['path'],
templatedata['templatename'])
if not os.path.isfile(origtemplatepath):
apDisplay.printError("Template file not found: " +
origtemplatepath)
apDisplay.printMsg("getting template: " + origtemplatepath)
copytemplatepath = os.path.join(params['rundir'],
"origTemplate" + str(index) + ".mrc")
scaletemplatepath = os.path.join(
params['rundir'], "scaledTemplate" + str(index) + ".mrc")
filtertemplatepath = os.path.join(
params['rundir'], "filterTemplate" + str(index) + ".mrc")
#masktemplatepath = os.path.join(params['rundir'], "maskTemplate"+str(index)+".mrc")
shutil.copyfile(origtemplatepath, copytemplatepath)
#RESCALE THE TEMPLATE
templatearray = apImage.mrcToArray(copytemplatepath)
#scale to correct apix
scalefactor = templatedata['apix'] / params['apix']
if abs(scalefactor - 1.0) > 0.01:
apDisplay.printMsg("rescaling template " + str(index) + ": " +
str(templatedata['apix']) + "->" +
str(params['apix']))
templatearray = scaleTemplate(templatearray, scalefactor)
apImage.arrayToMrc(templatearray, scaletemplatepath, msg=False)
#bin and filter
templatearray = apImage.preProcessImage(templatearray,
params=params,
highpass=0,
planeReg=False,
invert=False)
#write to file
apImage.arrayToMrc(templatearray, filtertemplatepath, msg=False)
### MASK THE TEMPLATE AND SAVE
#mask the template, visual purposes only
#maskrad = params['diam']/params['apix']/params['bin']/2.0
#maskarray =
#apImage.arrayToMrc(templatearray, masktemplatepath, msg=False)
#ADD TO TEMPLATE LIST
params['templatelist'].append(os.path.basename(filtertemplatepath))
### ADD MIRROR IF REQUESTED
if 'templatemirrors' in params and params['templatemirrors'] is True:
mirrortemplatepath = os.path.join(
params['rundir'], "mirrorTemplate" + str(index) + ".mrc")
mirrorarray = numpy.fliplr(templatearray)
apImage.arrayToMrc(mirrorarray, mirrortemplatepath, msg=False)
params['templatelist'].append(os.path.basename(mirrortemplatepath))
#FINISH LOOP OVER template ids
#Set the apix
params['templateapix'] = params['apix']
apDisplay.printMsg("scaled & filtered " +
str(len(params['templatelist'])) + " file(s)")
return params['templatelist']
def runAce(matlab, imgdata, params, showprev=True):
imgname = imgdata['filename']
if showprev is True:
bestctfvalue, bestconf = ctfdb.getBestCtfValueForImage(imgdata)
if bestctfvalue:
print ( "Prev best: '"+bestctfvalue['acerun']['name']+"', conf="+
apDisplay.colorProb(bestconf)+", defocus="+str(round(-1.0*abs(bestctfvalue['defocus1']*1.0e6),2))+
" microns" )
if params['uncorrected']:
tmpname='temporaryCorrectedImage.mrc'
imgarray = apDBImage.correctImage(imgdata)
imgpath = os.path.join(params['rundir'],tmpname)
apImage.arrayToMrc(imgarray, imgpath)
print "processing", imgpath
else:
imgpath = os.path.join(imgdata['session']['image path'], imgname+'.mrc')
nominal = None
if params['nominal'] is not None:
nominal=params['nominal']
elif params['newnominal'] is True:
nominal = ctfdb.getBestDefocusForImage(imgdata, msg=True)
if nominal is None:
nominal = imgdata['scope']['defocus']
if nominal is None or nominal > 0 or nominal < -15e-6:
apDisplay.printWarning("Nominal should be of the form nominal=-1.2e-6"+\
" for -1.2 microns NOT:"+str(nominal))
#Neil's Hack
#if 'autosample' in params and params['autosample']:
# x = abs(nominal*1.0e6)
# val = 1.585 + 0.057587 * x - 0.044106 * x**2 + 0.010877 * x**3
# resamplefr_override = round(val,3)
# print "resamplefr_override=",resamplefr_override
# pymat.eval(matlab, "resamplefr="+str(resamplefr_override)+";")
pymat.eval(matlab,("dforig = %e;" % nominal))
if params['stig'] == 0:
plist = (imgpath, params['outtextfile'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/")
acecmd = makeMatlabCmd("ctfparams = ace(",");",plist)
else:
plist = (imgname, imgpath, params['outtextfile'], params['opimagedir'], \
params['matdir'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/", params['resamplefr'])
acecmd = makeMatlabCmd("ctfparams = measureAstigmatism(",");",plist)
#print acecmd
pymat.eval(matlab,acecmd)
matfile = os.path.join(params['matdir'], imgname+".mrc.mat")
if params['stig']==0:
savematcmd = "save('"+matfile+"','ctfparams','scopeparams', 'dforig');"
pymat.eval(matlab,savematcmd)
ctfvalue = pymat.get(matlab, 'ctfparams')
ctfvalue=ctfvalue[0]
ctfdb.printResults(params, nominal, ctfvalue)
return ctfvalue
def getTemplates(params):
"""
Inputs:
params['templateIds'], a list of template ids
params['apix'], desired pixel size
params['rundir'], output directory
image processing params
Processing:
Copies, scales, and filters templates
Outputs:
params['templatelist'], a list of template file basenames
"""
apDisplay.printMsg("getting templates")
if not params['templateIds']:
apDisplay.printError("No template ids were specified")
params['templatelist'] = [] #list of scaled files
for i,templateid in enumerate(params['templateIds']):
index = i+1
#print templateid
templateid = int(templateid)
if templateid < 0:
continue
#QUERY DB FOR TEMPLATE INFO
templatedata = appiondata.ApTemplateImageData.direct_query(abs(templateid))
if not (templatedata):
apDisplay.printError("Template Id "+str(templateid)+" was not found in database.")
#COPY THE FILE OVER
origtemplatepath = os.path.join(templatedata['path']['path'], templatedata['templatename'])
if not os.path.isfile(origtemplatepath):
apDisplay.printError("Template file not found: "+origtemplatepath)
apDisplay.printMsg("getting template: "+origtemplatepath)
copytemplatepath = os.path.join(params['rundir'], "origTemplate"+str(index)+".mrc")
scaletemplatepath = os.path.join(params['rundir'], "scaledTemplate"+str(index)+".mrc")
filtertemplatepath = os.path.join(params['rundir'], "filterTemplate"+str(index)+".mrc")
#masktemplatepath = os.path.join(params['rundir'], "maskTemplate"+str(index)+".mrc")
shutil.copyfile(origtemplatepath, copytemplatepath)
#RESCALE THE TEMPLATE
templatearray = apImage.mrcToArray(copytemplatepath)
#scale to correct apix
scalefactor = templatedata['apix'] / params['apix']
if abs(scalefactor - 1.0) > 0.01:
apDisplay.printMsg("rescaling template "+str(index)+": "+str(templatedata['apix'])+"->"+str(params['apix']))
templatearray = scaleTemplate(templatearray, scalefactor)
apImage.arrayToMrc(templatearray, scaletemplatepath, msg=False)
#bin and filter
templatearray = apImage.preProcessImage(templatearray, params=params, highpass=0, planeReg=False, invert=False)
#write to file
apImage.arrayToMrc(templatearray, filtertemplatepath, msg=False)
### MASK THE TEMPLATE AND SAVE
#mask the template, visual purposes only
#maskrad = params['diam']/params['apix']/params['bin']/2.0
#maskarray =
#apImage.arrayToMrc(templatearray, masktemplatepath, msg=False)
#ADD TO TEMPLATE LIST
params['templatelist'].append(os.path.basename(filtertemplatepath))
### ADD MIRROR IF REQUESTED
if 'templatemirrors' in params and params['templatemirrors'] is True:
mirrortemplatepath = os.path.join(params['rundir'], "mirrorTemplate"+str(index)+".mrc")
mirrorarray = numpy.fliplr(templatearray)
apImage.arrayToMrc(mirrorarray, mirrortemplatepath, msg=False)
params['templatelist'].append(os.path.basename(mirrortemplatepath))
#FINISH LOOP OVER template ids
#Set the apix
params['templateapix'] = params['apix']
apDisplay.printMsg("scaled & filtered "+str(len(params['templatelist']))+" file(s)")
return params['templatelist']
def getShift(imgdata1, imgdata2):
#assumes images are square
print "Finding shift between", apDisplay.short(
imgdata1['filename']), "and", apDisplay.short(imgdata2['filename'])
dimension1 = imgdata1['camera']['dimension']['x']
binning1 = imgdata1['camera']['binning']['x']
dimension2 = imgdata2['camera']['dimension']['x']
binning2 = imgdata2['camera']['binning']['x']
finalsize = 512
#test to make sure images are at same mag
if imgdata1['scope']['magnification'] != imgdata2['scope']['magnification']:
apDisplay.printWarning(
"Defocus pairs are at different magnifications, so shift can't be calculated."
)
return None
#test to see if images capture the same area
if (dimension1 * binning1) != (dimension2 * binning2):
apDisplay.printWarning(
"Defocus pairs do not capture the same imaging area, so shift can't be calculated."
)
return None
#images must not be less than finalsize (currently 512) pixels. This is arbitrary but for good reason
if dimension1 < finalsize or dimension2 < finalsize:
apDisplay.printWarning("Images must be greater than " + finalsize +
" pixels to calculate shift.")
return None
#low pass filter 2 images to twice the final pixelsize BEFORE binning
shrinkfactor1 = dimension1 / finalsize
shrinkfactor2 = dimension2 / finalsize
binned1 = apImage.filterImg(imgdata1['image'], 1.0, shrinkfactor1 * 2)
binned2 = apImage.filterImg(imgdata2['image'], 1.0, shrinkfactor2 * 2)
#now bin 2 images
binned1 = apImage.binImg(binned1, shrinkfactor1)
binned2 = apImage.binImg(binned2, shrinkfactor2)
### fix for non-square images, correlation fails on non-square images
mindim = min(binned1.shape)
binned1 = binned1[:mindim, :mindim]
binned2 = binned2[:mindim, :mindim]
### use phase correlation, performs better than cross
pc = correlator.phase_correlate(binned1, binned2)
apImage.arrayToMrc(pc, "phaseCorrelate.mrc")
### find peak, filtering to 10.0 helps
peak = peakfinder.findSubpixelPeak(pc, lpf=10.0)
subpixpeak = peak['subpixel peak']
shift = correlator.wrap_coord(subpixpeak, pc.shape)
peak['scalefactor'] = dimension2 / float(dimension1)
#print shift[0]*shrinkfactor1, shift[1]*shrinkfactor1
xshift = int(round(shift[0] * shrinkfactor1))
yshift = int(round(shift[1] * shrinkfactor1))
peak['shift'] = numpy.array((xshift, yshift))
apDisplay.printMsg("Determined shifts: %f %f" %
(peak['shift'][0], peak['shift'][1]))
#print peak['shift']
#sys.exit(1)
return peak
def processImage(self, imgdata, filtarray): filtfile = os.path.join(self.params['rundir'], imgdata['filename']+".dwn.mrc") if not os.path.isfile(filtfile): apImage.arrayToMrc(filtarray, filtfile, msg=False) peaktree = self.runManualPicker(imgdata) return peaktree
def runAce(matlab, imgdata, params, showprev=True):
imgname = imgdata['filename']
if showprev is True:
bestctfvalue = ctfdb.getBestCtfByResolution(imgdata)
if bestctfvalue:
bestconf = ctfdb.calculateConfidenceScore(bestctfvalue)
print ( "Prev best: '"+bestctfvalue['acerun']['name']+"', conf="+
apDisplay.colorProb(bestconf)+", defocus="+str(round(-1.0*abs(bestctfvalue['defocus1']*1.0e6),2))+
" microns" )
if params['uncorrected']:
tmpname='temporaryCorrectedImage.mrc'
imgarray = apDBImage.correctImage(imgdata)
imgpath = os.path.join(params['rundir'],tmpname)
apImage.arrayToMrc(imgarray, imgpath)
print "processing", imgpath
else:
imgpath = os.path.join(imgdata['session']['image path'], imgname+'.mrc')
nominal = None
if params['nominal'] is not None:
nominal=params['nominal']
elif params['newnominal'] is True:
bestctfvalue = ctfdb.getBestCtfByResolution(imgdata)
nominal = bestctfvalue['defocus1']
if nominal is None:
nominal = imgdata['scope']['defocus']
if nominal is None or nominal > 0 or nominal < -15e-6:
apDisplay.printWarning("Nominal should be of the form nominal=-1.2e-6"+\
" for -1.2 microns NOT:"+str(nominal))
#Neil's Hack
#if 'autosample' in params and params['autosample']:
# x = abs(nominal*1.0e6)
# val = 1.585 + 0.057587 * x - 0.044106 * x**2 + 0.010877 * x**3
# resamplefr_override = round(val,3)
# print "resamplefr_override=",resamplefr_override
# pymat.eval(matlab, "resamplefr="+str(resamplefr_override)+";")
pymat.eval(matlab,("dforig = %e;" % nominal))
if params['stig'] == 0:
plist = (imgpath, params['outtextfile'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/")
acecmd = makeMatlabCmd("ctfparams = ace(",");",plist)
else:
plist = (imgname, imgpath, params['outtextfile'], params['opimagedir'], \
params['matdir'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/", params['resamplefr'])
acecmd = makeMatlabCmd("ctfparams = measureAstigmatism(",");",plist)
#print acecmd
pymat.eval(matlab,acecmd)
matfile = os.path.join(params['matdir'], imgname+".mrc.mat")
if params['stig']==0:
savematcmd = "save('"+matfile+"','ctfparams','scopeparams', 'dforig');"
pymat.eval(matlab,savematcmd)
ctfvalue = pymat.get(matlab, 'ctfparams')
ctfvalue=ctfvalue[0]
printResults(params, nominal, ctfvalue)
return ctfvalue
