def start(self):
#get aligned stack id
aligndata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
xSizeVoxel = aligndata['boxsize']
#get averaged image
avgmrc = os.path.join(aligndata['path']['path'], aligndata["avgmrcfile"])
avg = mrc.read(avgmrc)
tmpSpiderFile="average.xmp"
spider.write(avg, tmpSpiderFile)
self.runFindCenter(tmpSpiderFile,xSizeVoxel)
#get aligned stack
alignStack = os.path.join(aligndata['path']['path'], aligndata["imagicfile"])
tempFileNameforSpectra=self.runMakeSpectra(alignStack)
#kerdensom will work with spectra output
self.runKerdenSOM(tempFileNameforSpectra)
self.createMontageInMemory()
self.insertRotKerDenSOM()
#apFile.removeFile(outdata)
apFile.removeFilePattern("*.cod")
apFile.removeFilePattern("*.err")
apFile.removeFilePattern("*.his")
apFile.removeFilePattern("*.inf")
apFile.removeFilePattern("*.vs")
apFile.removeFilePattern("*.xmp")
apFile.removeFile(tempFileNameforSpectra)
def start(self):
#get aligned stack id
aligndata = appiondata.ApAlignStackData.direct_query(
self.params['alignstackid'])
xSizeVoxel = aligndata['boxsize']
#get averaged image
avgmrc = os.path.join(aligndata['path']['path'],
aligndata["avgmrcfile"])
avg = mrc.read(avgmrc)
tmpSpiderFile = "average.xmp"
spider.write(avg, tmpSpiderFile)
self.runFindCenter(tmpSpiderFile, xSizeVoxel)
#get aligned stack
alignStack = os.path.join(aligndata['path']['path'],
aligndata["imagicfile"])
tempFileNameforSpectra = self.runMakeSpectra(alignStack)
#kerdensom will work with spectra output
self.runKerdenSOM(tempFileNameforSpectra)
self.createMontageInMemory()
self.insertRotKerDenSOM()
#apFile.removeFile(outdata)
apFile.removeFilePattern("*.cod")
apFile.removeFilePattern("*.err")
apFile.removeFilePattern("*.his")
apFile.removeFilePattern("*.inf")
apFile.removeFilePattern("*.vs")
apFile.removeFilePattern("*.xmp")
apFile.removeFile(tempFileNameforSpectra)
def emanSpiderToStack(partlist):
apFile.removeStack("emanspi.hed", warn=False)
for part in partlist:
spider.write(part, "temp.spi")
emancmd = "proc2d temp.spi emanspi.hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apFile.removeFile("temp.spi")
return
def emanSpiderToStack(partlist):
apFile.removeStack("emanspi.hed", warn=False)
for part in partlist:
spider.write(part, "temp.spi")
emancmd = "proc2d temp.spi emanspi.hed"
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=False)
apFile.removeFile("temp.spi")
return
def arrayToSpider(numer, filename, msg=True):
"""
takes a numpy and writes a SPIDER imag
"""
#numer = numpy.asarray(numer, dtype=numpy.float32)
if msg is True:
apDisplay.printMsg("writing SPIDER image: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
spider.write(numer, filename)
return
def arrayToSpider(numer, filename, msg=True):
"""
takes a numpy and writes a SPIDER imag
"""
#numer = numpy.asarray(numer, dtype=numpy.float32)
if msg is True:
apDisplay.printMsg("writing SPIDER image: "+apDisplay.short(filename)+\
" size:"+str(numer.shape)+" dtype:"+str(numer.dtype))
spider.write(numer, filename)
return
def processParticle(self, partarray):
if self.filetype == "mrc":
partfile = os.path.join(self.partdir, "part%06d.mrc"%(self.index))
mrc.write(partarray, partfile)
else:
partfile = os.path.join(self.partdir, "part%06d.spi"%(self.index))
try:
spider.write(partarray, partfile)
except:
print partarray
apDisplay.printWarning("failed to write spider file for part index %d"%(self.index))
print partarray.shape
spider.write(partarray, partfile)
self.partdocf.write(os.path.abspath(partfile)+" 1\n")
def fermiLowPassFilter(imgarray, pixrad=2.0, dataext="spi", nproc=None):
if dataext[0] == '.': dataext = dataext[1:]
if nproc is None:
nproc = apParam.getNumProcessors(msg=False)
### save array to spider file
spider.write(imgarray, "rawimg."+dataext)
### run the filter
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, nproc=nproc)
### filter request: infile, outfile, filter-type, inv-radius, temperature
mySpider.toSpiderQuiet("FQ", "rawimg", "filtimg", "5", str(1.0/pixrad), "0.04")
mySpider.close()
### read array from spider file
filtarray = spider.read("filtimg."+dataext)
### clean up
apFile.removeFile("rawimg."+dataext)
apFile.removeFile("filtimg."+dataext)
return filtarray
def processParticle(self, partarray):
if self.filetype == "mrc":
partfile = os.path.join(self.partdir,
"part%06d.mrc" % (self.index))
mrc.write(partarray, partfile)
else:
partfile = os.path.join(self.partdir,
"part%06d.spi" % (self.index))
try:
spider.write(partarray, partfile)
except:
print partarray
apDisplay.printWarning(
"failed to write spider file for part index %d" %
(self.index))
print partarray.shape
spider.write(partarray, partfile)
self.partdocf.write(os.path.abspath(partfile) + " 1\n")
def fermiLowPassFilter(imgarray, pixrad=2.0, dataext="spi", nproc=None):
if dataext[0] == '.': dataext = dataext[1:]
if nproc is None:
nproc = apParam.getNumProcessors(msg=False)
### save array to spider file
spider.write(imgarray, "rawimg." + dataext)
### run the filter
mySpider = spyder.SpiderSession(dataext=dataext, logo=False, nproc=nproc)
### filter request: infile, outfile, filter-type, inv-radius, temperature
mySpider.toSpiderQuiet("FQ", "rawimg", "filtimg", "5", str(1.0 / pixrad),
"0.04")
mySpider.close()
### read array from spider file
filtarray = spider.read("filtimg." + dataext)
### clean up
apFile.removeFile("rawimg." + dataext)
apFile.removeFile("filtimg." + dataext)
return filtarray
def processImage(self, imgdata):
self.ctfvalues = {}
### convert image to spider
spiderimage = apDisplay.short(imgdata['filename']) + ".spi"
spider.write(imgdata['image'], spiderimage)
### high tension in kiloVolts
kvolts = imgdata['scope']['high tension'] / 1000.0
### spherical aberration in millimeters
cs = apInstrument.getCsValueFromSession(self.getSessionData())
### pixel size in Angstroms
apix = apDatabase.getPixelSize(imgdata)
### write image name
inputstr = ""
inputstr += ("\n")
inputstr += ("# image filename in spider format\n")
inputstr += ("image=%s\n" % (spiderimage))
### common parameters that never change
inputstr += ("\n")
inputstr += ("# common parameters that never change\n")
inputstr += ("N_horizontal=%d\n" % (self.params['fieldsize']))
#inputstr += ("particle_horizontal=%d\n"%(128))
inputstr += ("show_optimization=yes\n")
inputstr += ("micrograph_averaging=yes\n")
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents the typically peak at the center of the PSD to interfere with CTF estimation.
The default value is 0.05, but for micrographs with a very fine sampling this may be lowered towards 0.0
"""
#minres = apix/minfreq => minfreq = apix/minres
minfreq = apix / self.params['resmin']
inputstr += ("min_freq=%.3f\n" % (minfreq))
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents high-resolution terms where only noise exists to interfere with CTF estimation.
The default value is 0.35, but it should be increased for micrographs with signals extending beyond this value
"""
#maxres = apix/maxfreq => maxfreq = apix/maxres
maxfreq = apix / self.params['resmax']
inputstr += ("max_freq=%.3f\n" % (maxfreq))
### CTF input parameters from database
inputstr += ("\n")
inputstr += ("# CTF input parameters from database\n")
inputstr += ("voltage=%d\n" % (kvolts))
inputstr += ("spherical_aberration=%.1f\n" % (cs))
inputstr += ("sampling_rate=%.4f\n" % (apix))
### best defocus in negative Angstroms
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata, msg=True)
nominal = (ctfvalue['defocus1'] + ctfvalue['defocus2']) / 2.0
inputstr += ("defocusU=%d\n" % (-abs(ctfvalue['defocus1']) * 1.0e10))
inputstr += ("defocusV=%d\n" % (-abs(ctfvalue['defocus2']) * 1.0e10))
inputstr += ("Q0=%d\n" % (-ctfvalue['amplitude_contrast']))
inputstr += ("\n")
### open and write to input parameter file
paramfile = apDisplay.short(imgdata['filename']) + "-CTF.prm"
f = open(paramfile, "w")
print inputstr
f.write(inputstr)
f.close()
#[min_freq=<f=0.05>]
#[max_freq=<f=0.35>]
xmippcmd = "xmipp_ctf_estimate_from_micrograph -i %s" % (paramfile)
#xmippcmd = "echo %s"%(paramfile)
t0 = time.time()
apDisplay.printMsg("running ctf estimation at " + time.asctime())
proc = subprocess.Popen(xmippcmd, shell=True, stdout=subprocess.PIPE)
#waittime = 2
#while proc.poll() is None:
# sys.stderr.write(".")
# time.sleep(waittime)
(stdout, stderr) = proc.communicate()
#print (stdout, stderr)
apDisplay.printMsg("ctf estimation completed in " +
apDisplay.timeString(time.time() - t0))
### read commandline output to get fit score
lines = stdout.split('\n')
lastline = ""
for line in lines[-50:]:
if "--->" in line:
lastline = line
if not "--->" in lastline:
apDisplay.printWarning("Xmipp CTF failed")
self.badprocess = True
return
bits = lastline.split('--->')
confidence = float(bits[1])
score = round(math.sqrt(1 - confidence), 5)
apDisplay.printColor(
"Confidence value is %.5f (score %.5f)" % (confidence, score),
"cyan")
f = open("confidence.log", "a")
f.write("Confidence value is %.5f for image %s (score %.3f)\n" %
(confidence, apDisplay.short(imgdata['filename']), score))
f.close()
### delete image in spider format no longer needed
apFile.removeFile(spiderimage)
### read output parameter file
outparamfile = apDisplay.short(
imgdata['filename']) + "_Periodogramavg.ctfparam"
f = open(outparamfile, "r")
for line in f:
sline = line.strip()
bits = sline.split('=')
if sline.startswith("defocusU"):
defocus1 = float(bits[1].strip())
elif sline.startswith("defocusV"):
defocus2 = float(bits[1].strip())
elif sline.startswith("azimuthal_angle"):
angle_astigmatism = float(bits[1].strip())
elif sline.startswith("Q0"):
amplitude_contrast = abs(float(bits[1].strip()))
print defocus1, defocus2, angle_astigmatism, amplitude_contrast
#defocusU= -18418.6
#defocusV= -24272.1
#azimuthal_angle= 79.7936
#Q0= -0.346951 #negative of ACE amplitude_contrast
print "AMP CONTRAST: %.4f -- %.4f" % (amplitude_contrast,
ctfvalue['amplitude_contrast'])
self.ctfvalues = {
'defocus1': defocus1 * 1e-10,
'defocus2': defocus2 * 1e-10,
'angle_astigmatism': angle_astigmatism,
'amplitude_contrast': amplitude_contrast,
'nominal': nominal,
'defocusinit': nominal,
'confidence_d': score,
'cs': self.params['cs'],
'volts': kvolts * 1000.0,
}
return
def processImage(self, imgdata):
self.ctfvalues = {}
### convert image to spider
spiderimage = apDisplay.short(imgdata['filename'])+".spi"
spider.write(imgdata['image'], spiderimage)
### high tension in kiloVolts
kvolts = imgdata['scope']['high tension']/1000.0
### spherical aberration in millimeters
cs = apInstrument.getCsValueFromSession(self.getSessionData())
### pixel size in Angstroms
apix = apDatabase.getPixelSize(imgdata)
### write image name
inputstr = ""
inputstr += ("\n")
inputstr += ("# image filename in spider format\n")
inputstr += ("image=%s\n"%(spiderimage))
### common parameters that never change
inputstr += ("\n")
inputstr += ("# common parameters that never change\n")
inputstr += ("N_horizontal=%d\n"%(self.params['fieldsize']))
#inputstr += ("particle_horizontal=%d\n"%(128))
inputstr += ("show_optimization=yes\n")
inputstr += ("micrograph_averaging=yes\n")
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents the typically peak at the center of the PSD to interfere with CTF estimation.
The default value is 0.05, but for micrographs with a very fine sampling this may be lowered towards 0.0
"""
#minres = apix/minfreq => minfreq = apix/minres
minfreq = apix/self.params['resmin']
inputstr += ("min_freq=%.3f\n"%(minfreq))
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents high-resolution terms where only noise exists to interfere with CTF estimation.
The default value is 0.35, but it should be increased for micrographs with signals extending beyond this value
"""
#maxres = apix/maxfreq => maxfreq = apix/maxres
maxfreq = apix/self.params['resmax']
inputstr += ("max_freq=%.3f\n"%(maxfreq))
### CTF input parameters from database
inputstr += ("\n")
inputstr += ("# CTF input parameters from database\n")
inputstr += ("voltage=%d\n"%(kvolts))
inputstr += ("spherical_aberration=%.1f\n"%(cs))
inputstr += ("sampling_rate=%.4f\n"%(apix))
### best defocus in negative Angstroms
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata, msg=True)
if ctfvalue is None:
apDisplay.printWarning("Xmipp CTF as implemented in Appion requires an initial CTF estimate to process"
+"\nPlease run another CTF program first and try again on this image")
self.ctfvalues = None
return
nominal = (ctfvalue['defocus1']+ctfvalue['defocus2'])/2.0
inputstr += ("defocusU=%d\n"%(-abs(ctfvalue['defocus1'])*1.0e10))
inputstr += ("defocusV=%d\n"%(-abs(ctfvalue['defocus2'])*1.0e10))
inputstr += ("Q0=%d\n"%(-ctfvalue['amplitude_contrast']))
inputstr += ("\n")
### open and write to input parameter file
paramfile = apDisplay.short(imgdata['filename'])+"-CTF.prm"
f = open(paramfile, "w")
print inputstr
f.write(inputstr)
f.close()
#[min_freq=<f=0.05>]
#[max_freq=<f=0.35>]
xmippcmd = "xmipp_ctf_estimate_from_micrograph -i %s"%(paramfile)
#xmippcmd = "echo %s"%(paramfile)
t0 = time.time()
apDisplay.printMsg("running ctf estimation at "+time.asctime())
proc = subprocess.Popen(xmippcmd, shell=True, stdout=subprocess.PIPE)
#waittime = 2
#while proc.poll() is None:
# sys.stderr.write(".")
# time.sleep(waittime)
(stdout, stderr) = proc.communicate()
#print (stdout, stderr)
apDisplay.printMsg("ctf estimation completed in "+apDisplay.timeString(time.time()-t0))
### read commandline output to get fit score
lines = stdout.split('\n')
lastline = ""
for line in lines[-50:]:
if "--->" in line:
lastline = line
if not "--->" in lastline:
apDisplay.printWarning("Xmipp CTF failed")
self.badprocess = True
return
bits = lastline.split('--->')
confidence = float(bits[1])
score = round(math.sqrt(1-confidence), 5)
apDisplay.printColor("Confidence value is %.5f (score %.5f)"%(confidence, score), "cyan")
f = open("confidence.log", "a")
f.write("Confidence value is %.5f for image %s (score %.3f)\n"
%(confidence, apDisplay.short(imgdata['filename']), score))
f.close()
### delete image in spider format no longer needed
apFile.removeFile(spiderimage)
### read output parameter file
outparamfile = apDisplay.short(imgdata['filename'])+"_Periodogramavg.ctfparam"
f = open(outparamfile, "r")
for line in f:
sline = line.strip()
bits = sline.split('=')
if sline.startswith("defocusU"):
defocus1 = float(bits[1].strip())
elif sline.startswith("defocusV"):
defocus2 = float(bits[1].strip())
elif sline.startswith("azimuthal_angle"):
angle_astigmatism = float(bits[1].strip())
elif sline.startswith("Q0"):
amplitude_contrast = abs(float(bits[1].strip()))
print defocus1, defocus2, angle_astigmatism, amplitude_contrast
#defocusU= -18418.6
#defocusV= -24272.1
#azimuthal_angle= 79.7936
#Q0= -0.346951 #negative of ACE amplitude_contrast
print "AMP CONTRAST: %.4f -- %.4f"%(amplitude_contrast, ctfvalue['amplitude_contrast'])
self.ctfvalues = {
'defocus1': defocus1*1e-10,
'defocus2': defocus2*1e-10,
'angle_astigmatism': angle_astigmatism,
'amplitude_contrast': amplitude_contrast,
'nominal': nominal,
'defocusinit': nominal,
'confidence_d': score,
'cs': self.params['cs'],
'volts': kvolts*1000.0,
}
return
