def getCtfValueConfidenceForImage(self, imgdata, msg=False):
method = self.params['ctfmethod']
ctfrunid = self.params['ctfrunid']
if ctfrunid is None:
return ctfdb.getBestCtfValueForImage(imgdata,msg=msg,method=method)
else:
return ctfdb.getCtfValueForImage(imgdata, ctfrunid, msg=msg, method=method)
def runAceCorrect(imgdict,params):
imgname = imgdict['filename']
imgpath = os.path.join(imgdict['session']['image path'], imgname+'.mrc')
voltage = (imgdict['scope']['high tension'])
apix = apDatabase.getPixelSize(imgdict)
ctfvalues, conf = ctfdb.getBestCtfValueForImage(imgdict)
ctdimname = imgname
ctdimpath = os.path.join(params['rundir'],ctdimname)
print "Corrected Image written to " + ctdimpath
#pdb.set_trace()
acecorrectcommand=("ctfcorrect1('%s', '%s', '%.32f', '%.32f', '%f', '%f', '%f');" % \
(imgpath, ctdimpath, ctfvalues['defocus1'], ctfvalues['defocus2'], -ctfvalues['angle_astigmatism'], voltage, apix))
print acecorrectcommand
try:
matlab = pymat.open("matlab -nosplash")
except:
apDisplay.environmentError()
raise
pymat.eval(matlab, acecorrectcommand)
pymat.close(matlab)
return
def rejectAceInfo(self, imgdata):
shortname = apDisplay.short(imgdata["filename"])
### get best defocus value
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata)
if ctfvalue is None:
if self.params["noace"] is True:
apDisplay.printColor("\nrejecting no ACE values: " + apDisplay.short(imgdata["filename"]), "yellow")
return False
else:
# apDisplay.printWarning("skipping no ACE values for "+apDisplay.short(imgdata['filename']))
return True
### check that CTF estimation is above confidence threshold
if self.params["ctfcutoff"] and conf < self.params["ctfcutoff"]:
apDisplay.printColor(
"\nrejecting below CTF cutoff: "
+ apDisplay.short(imgdata["filename"])
+ " conf="
+ str(round(conf, 3)),
"cyan",
)
return False
### defocus should be in negative meters
if ctfvalue["defocus2"] is not None and ctfvalue["defocus1"] != ctfvalue["defocus2"]:
defocus = (ctfvalue["defocus1"] + ctfvalue["defocus2"]) / 2.0
else:
defocus = ctfvalue["defocus1"]
defocus = -1.0 * abs(defocus)
### skip micrograph that have defocus above or below min & max defocus levels
if self.params["mindefocus"] and defocus > self.params["mindefocus"]:
apDisplay.printColor(
shortname
+ " defocus ("
+ str(round(defocus * 1e6, 2))
+ " um) is less than mindefocus ("
+ str(self.params["mindefocus"] * 1e6)
+ " um)\n",
"cyan",
)
return False
if self.params["maxdefocus"] and defocus < self.params["maxdefocus"]:
apDisplay.printColor(
shortname
+ " defocus ("
+ str(round(defocus * 1e6, 2))
+ " um) is greater than maxdefocus ("
+ str(self.params["maxdefocus"] * 1e6)
+ " um)\n",
"cyan",
)
return False
return True
def getDefocusAmpConstForImage(self,imgdata,msg=False):
### This function returns defocus defined as negative underfocus
method = self.params['ctfmethod']
ctfrunid = self.params['ctfrunid']
if ctfrunid is None:
ctfvalue,conf = ctfdb.getBestCtfValueForImage(imgdata,msg=msg,method=method)
# make sure we return a negative underfocus value
defocus = -(abs(ctfvalue['defocus1'])+abs(ctfvalue['defocus2']))/2
return defocus, ctfvalue['amplitude_contrast']
else:
return ctfdb.getDefocusAndampcontrastForImage(imgdata, ctf_estimation_runid=ctfrunid, msg=msg, method=method)
def reprocessImage(self, imgdata): """ Returns True, if an image should be reprocessed False, if an image was processed and should NOT be reprocessed None, if image has not yet been processed e.g. a confidence less than 80% """ if self.params['reprocess'] is None: return None ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata) if ctfvalue is None: return None if conf > self.params['reprocess']: return False else: return True
def rejectAceInfo(self, imgdata):
shortname = apDisplay.short(imgdata['filename'])
### get best defocus value
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata)
if ctfvalue is None:
if self.params['noace'] is True:
apDisplay.printColor(
"\nrejecting no ACE values: " +
apDisplay.short(imgdata['filename']), "yellow")
return False
else:
#apDisplay.printWarning("skipping no ACE values for "+apDisplay.short(imgdata['filename']))
return True
### check that CTF estimation is above confidence threshold
if self.params['ctfcutoff'] and conf < self.params['ctfcutoff']:
apDisplay.printColor(
"\nrejecting below CTF cutoff: " +
apDisplay.short(imgdata['filename']) + " conf=" +
str(round(conf, 3)), "cyan")
return False
### defocus should be in negative meters
if ctfvalue['defocus2'] is not None and ctfvalue[
'defocus1'] != ctfvalue['defocus2']:
defocus = (ctfvalue['defocus1'] + ctfvalue['defocus2']) / 2.0
else:
defocus = ctfvalue['defocus1']
defocus = -1.0 * abs(defocus)
### skip micrograph that have defocus above or below min & max defocus levels
if self.params['mindefocus'] and defocus > self.params['mindefocus']:
apDisplay.printColor(shortname+" defocus ("+str(round(defocus*1e6,2))+\
" um) is less than mindefocus ("+str(self.params['mindefocus']*1e6)+" um)\n","cyan")
return False
if self.params['maxdefocus'] and defocus < self.params['maxdefocus']:
apDisplay.printColor(shortname+" defocus ("+str(round(defocus*1e6,2))+\
" um) is greater than maxdefocus ("+str(self.params['maxdefocus']*1e6)+" um)\n","cyan")
return False
return True
def processImage(self, imgdata):
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata)
if ctfvalue is None:
return None
defocus = ctfvalue['defocus1']
acepath = ctfvalue['acerun']['path']['path']
opimgpath = os.path.join(acepath,"opimages")
#print opimgpath
#for i,v in ctfvalue.items():
# print i,v
if conf < 0.5:
confdir = "doubtful"
elif conf > 0.8:
confdir = "probable"
else:
confdir = "questionable"
defocusdir = "defocus"+str(int(math.floor(defocus*-1e6)))
apParam.createDirectory(os.path.join(self.params['opimage1'], defocusdir), warning=False)
apParam.createDirectory(os.path.join(self.params['opimage2'], defocusdir), warning=False)
#copy first file
opfile = os.path.join(opimgpath, ctfvalue['graph1'])
if os.path.isfile(opfile):
shutil.copyfile(opfile, os.path.join(self.params['opimage1'], "all", ctfvalue['graph1']))
shutil.copyfile(opfile, os.path.join(self.params['opimage1'], confdir, ctfvalue['graph1']))
shutil.copyfile(opfile, os.path.join(self.params['opimage1'], defocusdir, ctfvalue['graph1']))
else:
apDisplay.printWarning("could not find opimage: "+opfile)
#copy second file
opfile = os.path.join(opimgpath, ctfvalue['graph2'])
if os.path.isfile(opfile):
shutil.copyfile(opfile, os.path.join(self.params['opimage2'], "all", ctfvalue['graph2']))
shutil.copyfile(opfile, os.path.join(self.params['opimage2'], confdir, ctfvalue['graph2']))
shutil.copyfile(opfile, os.path.join(self.params['opimage2'], defocusdir, ctfvalue['graph2']))
else:
apDisplay.printWarning("could not find opimage: "+opfile)
def start(self):
### default parameters
starfile = self.params['starfile']
images = self.params['images']
dbnum = self.params['dbnum']
ctfrunid = self.params['ctfrunid']
stackid = self.params['stackid']
voltage = self.params['voltage']
cs = self.params['cs']
wgh = self.params['wgh']
### particles, angles
appiondata.sinedon.setConfig('appiondata', db="ap%d" % dbnum)
particledata = apStack.getStackParticlesFromId(stackid)
### write Relion starfile header
sf = open(starfile, "w")
sf.write("data_images\n")
sf.write("loop_\n")
sf.write("_rlnImageName\n")
sf.write("_rlnMicrographName\n")
sf.write("_rlnDefocusU\n")
sf.write("_rlnDefocusV\n")
sf.write("_rlnDefocusAngle\n")
sf.write("_rlnVoltage\n")
sf.write("_rlnSphericalAberration\n")
sf.write("_rlnAmplitudeContrast\n")
### write info to starfile
olddx = 0
micn = 0
oldimgid = None
for i in range(len(particledata)):
if i % 1000 == 0:
print "done with %d particles" % i
### CTF info
image = particledata[i]['particle']['image']
imgid = image.dbid
try:
ctf = ctfdb.getCtfValueForCtfRunId(image, ctfrunid, msg=False)
dx = ctf['defocus1'] * 10e9
dy = ctf['defocus2'] * 10e9
astig = ctf['angle_astigmatism']
except:
ctf = None
# print "didn't find CTF values for image ", image.dbid
if ctf is None:
if oldimgid != imgid:
print "particle %d: " % i, "getting best value for image: %d" % imgid
ctf = ctfdb.getBestCtfValueForImage(image,
msg=False,
method='ctffind')
dx = ctf[0]['defocus1'] * 10e9
dy = ctf[0]['defocus2'] * 10e9
astig = ctf[0]['angle_astigmatism']
oldctf = ctf
oldimgid = imgid
else:
try:
ctf = oldctf
dx = oldctf[0]['defocus1'] * 10e9
dy = oldctf[0]['defocus2'] * 10e9
astig = oldctf[0]['angle_astigmatism']
except:
apDisplay.printError("no CTF information for image")
if dx != olddx:
micn += 1
olddx = dx
### write input Relion parameters
sf.write("%06d@%s%10d%12.3f%12.3f%12.3f%8.3f%8.3f%8.3f\n" %
(i + 1, images, micn, dx, dy, astig, voltage, cs, wgh))
sf.close()
def generateParticleParams(self):
paramfile = os.path.join(self.params['rundir'], 'params.iter000.par')
apDisplay.printMsg("Creating parameter file: "+paramfile)
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if os.path.isfile(paramfile):
f = open(paramfile, 'r')
numparam = len(f.readlines())
if numparam > self.params['last']:
apDisplay.printMsg("Param file exists")
return paramfile
else:
apDisplay.printWarning("Param file exists with too few particles: %d vs %d"%(numparam,numpart))
stackdata = apStack.getStackParticlesFromId(self.params['stackid'])
particleparams={}
f = open(paramfile, 'w')
f.write("C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n")
apDisplay.printMsg("Writing out particle parameters")
count = 0
t0 = time.time()
self.noeulers = 0
for particle in stackdata:
count += 1
if (count % 200 == 0):
estime = (time.time() - t0) * (numpart-count) / float(count)
apDisplay.printMsg("particle %d -- %s remain"%(count, apDisplay.timeString(estime)))
if count > self.params['last']:
break
# defaults
## Niko says that if the defocus is negative it will not perform CTF correction
## But it will also not correct the amplitudes
particleparams = {
'ptclnum': particle['particleNumber'],
'psi': 0.0,
'theta': 0.0,
'phi': 0.0,
'df1': -1.0, # workaround if no ctf correction
'df2': -1.0, # set defocus to -1.0 Angstroms
'angast': 0.0,
'mag': 10000, # workaround to get around dstep
'shx': 0.0,
'shy': 0.0,
'film': 1,
'presa': 0.0,
'dpres': 0.0,
}
imagedata = particle['particle']['image']
if self.params['noctf'] is False:
ctfdata, confidence = ctfdb.getBestCtfValueForImage(imagedata, msg=False, method=self.params['ctfmethod'])
if ctfdata is not None:
### use defocus and astigmatism values
particleparams['df1'] = abs(ctfdata['defocus1']*1e10)
particleparams['df2'] = abs(ctfdata['defocus2']*1e10)
particleparams['angast'] = ctfdata['angle_astigmatism']
# if using parameters from previous reconstruction
if self.params['reconiterid'] is not None:
emaneuler = self.getStackParticleEulersForIteration(particle['particleNumber'])
frealigneulers = apFrealign.convertEmanEulersToFrealign(emaneuler, sym=self.params['sym'])
particleparams['psi'] = frealigneulers['psi']
particleparams['theta'] = frealigneulers['theta']
particleparams['phi'] = frealigneulers['phi']
particleparams['shx'] = emaneuler['shiftx']
particleparams['shy'] = emaneuler['shifty']
if emaneuler['mirror'] is True:
particleparams['shx'] *= -1
self.writeParticleParamLine(particleparams,f)
f.close()
return paramfile
def processImage(self, imgdata):
self.ctfvalues = {}
### convert image to spider
spiderimage = apDisplay.short(imgdata['filename']) + ".spi"
spider.write(imgdata['image'], spiderimage)
### high tension in kiloVolts
kvolts = imgdata['scope']['high tension'] / 1000.0
### spherical aberration in millimeters
cs = apInstrument.getCsValueFromSession(self.getSessionData())
### pixel size in Angstroms
apix = apDatabase.getPixelSize(imgdata)
### write image name
inputstr = ""
inputstr += ("\n")
inputstr += ("# image filename in spider format\n")
inputstr += ("image=%s\n" % (spiderimage))
### common parameters that never change
inputstr += ("\n")
inputstr += ("# common parameters that never change\n")
inputstr += ("N_horizontal=%d\n" % (self.params['fieldsize']))
#inputstr += ("particle_horizontal=%d\n"%(128))
inputstr += ("show_optimization=yes\n")
inputstr += ("micrograph_averaging=yes\n")
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents the typically peak at the center of the PSD to interfere with CTF estimation.
The default value is 0.05, but for micrographs with a very fine sampling this may be lowered towards 0.0
"""
#minres = apix/minfreq => minfreq = apix/minres
minfreq = apix / self.params['resmin']
inputstr += ("min_freq=%.3f\n" % (minfreq))
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents high-resolution terms where only noise exists to interfere with CTF estimation.
The default value is 0.35, but it should be increased for micrographs with signals extending beyond this value
"""
#maxres = apix/maxfreq => maxfreq = apix/maxres
maxfreq = apix / self.params['resmax']
inputstr += ("max_freq=%.3f\n" % (maxfreq))
### CTF input parameters from database
inputstr += ("\n")
inputstr += ("# CTF input parameters from database\n")
inputstr += ("voltage=%d\n" % (kvolts))
inputstr += ("spherical_aberration=%.1f\n" % (cs))
inputstr += ("sampling_rate=%.4f\n" % (apix))
### best defocus in negative Angstroms
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata, msg=True)
nominal = (ctfvalue['defocus1'] + ctfvalue['defocus2']) / 2.0
inputstr += ("defocusU=%d\n" % (-abs(ctfvalue['defocus1']) * 1.0e10))
inputstr += ("defocusV=%d\n" % (-abs(ctfvalue['defocus2']) * 1.0e10))
inputstr += ("Q0=%d\n" % (-ctfvalue['amplitude_contrast']))
inputstr += ("\n")
### open and write to input parameter file
paramfile = apDisplay.short(imgdata['filename']) + "-CTF.prm"
f = open(paramfile, "w")
print inputstr
f.write(inputstr)
f.close()
#[min_freq=<f=0.05>]
#[max_freq=<f=0.35>]
xmippcmd = "xmipp_ctf_estimate_from_micrograph -i %s" % (paramfile)
#xmippcmd = "echo %s"%(paramfile)
t0 = time.time()
apDisplay.printMsg("running ctf estimation at " + time.asctime())
proc = subprocess.Popen(xmippcmd, shell=True, stdout=subprocess.PIPE)
#waittime = 2
#while proc.poll() is None:
# sys.stderr.write(".")
# time.sleep(waittime)
(stdout, stderr) = proc.communicate()
#print (stdout, stderr)
apDisplay.printMsg("ctf estimation completed in " +
apDisplay.timeString(time.time() - t0))
### read commandline output to get fit score
lines = stdout.split('\n')
lastline = ""
for line in lines[-50:]:
if "--->" in line:
lastline = line
if not "--->" in lastline:
apDisplay.printWarning("Xmipp CTF failed")
self.badprocess = True
return
bits = lastline.split('--->')
confidence = float(bits[1])
score = round(math.sqrt(1 - confidence), 5)
apDisplay.printColor(
"Confidence value is %.5f (score %.5f)" % (confidence, score),
"cyan")
f = open("confidence.log", "a")
f.write("Confidence value is %.5f for image %s (score %.3f)\n" %
(confidence, apDisplay.short(imgdata['filename']), score))
f.close()
### delete image in spider format no longer needed
apFile.removeFile(spiderimage)
### read output parameter file
outparamfile = apDisplay.short(
imgdata['filename']) + "_Periodogramavg.ctfparam"
f = open(outparamfile, "r")
for line in f:
sline = line.strip()
bits = sline.split('=')
if sline.startswith("defocusU"):
defocus1 = float(bits[1].strip())
elif sline.startswith("defocusV"):
defocus2 = float(bits[1].strip())
elif sline.startswith("azimuthal_angle"):
angle_astigmatism = float(bits[1].strip())
elif sline.startswith("Q0"):
amplitude_contrast = abs(float(bits[1].strip()))
print defocus1, defocus2, angle_astigmatism, amplitude_contrast
#defocusU= -18418.6
#defocusV= -24272.1
#azimuthal_angle= 79.7936
#Q0= -0.346951 #negative of ACE amplitude_contrast
print "AMP CONTRAST: %.4f -- %.4f" % (amplitude_contrast,
ctfvalue['amplitude_contrast'])
self.ctfvalues = {
'defocus1': defocus1 * 1e-10,
'defocus2': defocus2 * 1e-10,
'angle_astigmatism': angle_astigmatism,
'amplitude_contrast': amplitude_contrast,
'nominal': nominal,
'defocusinit': nominal,
'confidence_d': score,
'cs': self.params['cs'],
'volts': kvolts * 1000.0,
}
return
def start(self):
### default parameters
starfile = self.params['starfile']
images = self.params['images']
dbnum = self.params['dbnum']
ctfrunid = self.params['ctfrunid']
stackid = self.params['stackid']
voltage = self.params['voltage']
cs = self.params['cs']
wgh = self.params['wgh']
### particles, angles
appiondata.sinedon.setConfig('appiondata', db="ap%d" % dbnum)
particledata = apStack.getStackParticlesFromId(stackid)
### write Relion starfile header
sf = open(starfile, "w")
sf.write("data_images\n")
sf.write("loop_\n")
sf.write("_rlnImageName\n")
sf.write("_rlnMicrographName\n")
sf.write("_rlnDefocusU\n")
sf.write("_rlnDefocusV\n")
sf.write("_rlnDefocusAngle\n")
sf.write("_rlnVoltage\n")
sf.write("_rlnSphericalAberration\n")
sf.write("_rlnAmplitudeContrast\n")
### write info to starfile
olddx = 0
micn = 0
oldimgid = None
for i in range(len(particledata)):
if i % 1000 == 0:
print "done with %d particles" % i
### CTF info
image = particledata[i]['particle']['image']
imgid = image.dbid
try:
ctf = ctfdb.getCtfValueForCtfRunId(image, ctfrunid, msg=False)
dx = ctf['defocus1'] * 10e9
dy = ctf['defocus2'] * 10e9
astig = ctf['angle_astigmatism']
except:
ctf = None
# print "didn't find CTF values for image ", image.dbid
if ctf is None:
if oldimgid != imgid:
print "particle %d: " % i, "getting best value for image: %d" % imgid
ctf = ctfdb.getBestCtfValueForImage(image, msg=False, method='ctffind')
dx = ctf[0]['defocus1'] * 10e9
dy = ctf[0]['defocus2'] * 10e9
astig = ctf[0]['angle_astigmatism']
oldctf = ctf
oldimgid = imgid
else:
try:
ctf = oldctf
dx = oldctf[0]['defocus1'] * 10e9
dy = oldctf[0]['defocus2'] * 10e9
astig = oldctf[0]['angle_astigmatism']
except:
apDisplay.printError("no CTF information for image")
if dx != olddx:
micn += 1
olddx = dx
### write input Relion parameters
sf.write("%06d@%s%10d%12.3f%12.3f%12.3f%8.3f%8.3f%8.3f\n"
% (i+1, images, micn, dx, dy, astig, voltage, cs, wgh)
)
sf.close()
def processImage(self, imgdata):
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata)
if ctfvalue is None:
return None
defocus = ctfvalue['defocus1']
acepath = ctfvalue['acerun']['path']['path']
opimgpath = os.path.join(acepath, "opimages")
#print opimgpath
#for i,v in ctfvalue.items():
# print i,v
if conf < 0.5:
confdir = "doubtful"
elif conf > 0.8:
confdir = "probable"
else:
confdir = "questionable"
defocusdir = "defocus" + str(int(math.floor(defocus * -1e6)))
apParam.createDirectory(os.path.join(self.params['opimage1'],
defocusdir),
warning=False)
apParam.createDirectory(os.path.join(self.params['opimage2'],
defocusdir),
warning=False)
#copy first file
opfile = os.path.join(opimgpath, ctfvalue['graph1'])
if os.path.isfile(opfile):
shutil.copyfile(
opfile,
os.path.join(self.params['opimage1'], "all",
ctfvalue['graph1']))
shutil.copyfile(
opfile,
os.path.join(self.params['opimage1'], confdir,
ctfvalue['graph1']))
shutil.copyfile(
opfile,
os.path.join(self.params['opimage1'], defocusdir,
ctfvalue['graph1']))
else:
apDisplay.printWarning("could not find opimage: " + opfile)
#copy second file
opfile = os.path.join(opimgpath, ctfvalue['graph2'])
if os.path.isfile(opfile):
shutil.copyfile(
opfile,
os.path.join(self.params['opimage2'], "all",
ctfvalue['graph2']))
shutil.copyfile(
opfile,
os.path.join(self.params['opimage2'], confdir,
ctfvalue['graph2']))
shutil.copyfile(
opfile,
os.path.join(self.params['opimage2'], defocusdir,
ctfvalue['graph2']))
else:
apDisplay.printWarning("could not find opimage: " + opfile)
def generateParticleParams(self):
paramfile = os.path.join(self.params['rundir'], 'params.iter000.par')
apDisplay.printMsg("Creating parameter file: " + paramfile)
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if os.path.isfile(paramfile):
f = open(paramfile, 'r')
numparam = len(f.readlines())
if numparam > self.params['last']:
apDisplay.printMsg("Param file exists")
return paramfile
else:
apDisplay.printWarning(
"Param file exists with too few particles: %d vs %d" %
(numparam, numpart))
stackdata = apStack.getStackParticlesFromId(self.params['stackid'])
particleparams = {}
f = open(paramfile, 'w')
f.write(
"C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n"
)
apDisplay.printMsg("Writing out particle parameters")
count = 0
t0 = time.time()
self.noeulers = 0
for particle in stackdata:
count += 1
if (count % 200 == 0):
estime = (time.time() - t0) * (numpart - count) / float(count)
apDisplay.printMsg("particle %d -- %s remain" %
(count, apDisplay.timeString(estime)))
if count > self.params['last']:
break
# defaults
## Niko says that if the defocus is negative it will not perform CTF correction
## But it will also not correct the amplitudes
particleparams = {
'ptclnum': particle['particleNumber'],
'psi': 0.0,
'theta': 0.0,
'phi': 0.0,
'df1': -1.0, # workaround if no ctf correction
'df2': -1.0, # set defocus to -1.0 Angstroms
'angast': 0.0,
'mag': 10000, # workaround to get around dstep
'shx': 0.0,
'shy': 0.0,
'film': 1,
'presa': 0.0,
'dpres': 0.0,
}
imagedata = particle['particle']['image']
if self.params['noctf'] is False:
ctfdata, confidence = ctfdb.getBestCtfValueForImage(
imagedata, msg=False, method=self.params['ctfmethod'])
if ctfdata is not None:
### use defocus and astigmatism values
particleparams['df1'] = abs(ctfdata['defocus1'] * 1e10)
particleparams['df2'] = abs(ctfdata['defocus2'] * 1e10)
particleparams['angast'] = ctfdata['angle_astigmatism']
# if using parameters from previous reconstruction
if self.params['reconiterid'] is not None:
emaneuler = self.getStackParticleEulersForIteration(
particle['particleNumber'])
frealigneulers = apFrealign.convertEmanEulersToFrealign(
emaneuler, sym=self.params['sym'])
particleparams['psi'] = frealigneulers['psi']
particleparams['theta'] = frealigneulers['theta']
particleparams['phi'] = frealigneulers['phi']
particleparams['shx'] = emaneuler['shiftx']
particleparams['shy'] = emaneuler['shifty']
if emaneuler['mirror'] is True:
particleparams['shx'] *= -1
self.writeParticleParamLine(particleparams, f)
f.close()
return paramfile
def generateParticleParams(params, modeldata, initparfile='params.0.par'):
params['inpar'] = os.path.join(params['rundir'], initparfile)
apDisplay.printMsg("Creating parameter file: " + params['inpar'])
params['mode'] = 3
stackdata = getStackParticlesInOrder(params)
first_imageid = stackdata[0]['particle']['image'].dbid
f = open(params['inpar'], 'w')
params['noClassification'] = 0
if params['reconiterid']:
iterdata = apRecon.getRefineIterDataFromIterationId(
params['reconiterid'])
sym_name = modeldata['symmetry']['symmetry']
print "Writing out particle parameters"
if 'last' not in params:
params['last'] = len(stackdata)
for i, particle in enumerate(stackdata[:params['last']]):
# defaults
particleparams = initializeParticleParams(i)
# for helical reconstructions, film is helix number
if particle['particle']['helixnum']:
imgid = particle['particle']['image'].dbid
helix = particle['particle']['helixnum']
try:
if params['lastimgid'] != imgid or params['lasthelix'] != helix:
params['totalHelix'] += 1
except:
params['totalHelix'] = 1
params['lastimgid'] = imgid
params['lasthelix'] = helix
particleparams['film'] = params['totalHelix']
else:
particleparams['film'] = particle['particle'][
'image'].dbid - first_imageid + 1
# extra particle number information not read by Frealign
particleparams['pnumber'] = particle['particleNumber']
imagedata = particle['particle']['image']
if params['noctf'] is False:
if params['defocpair'] is True:
imagedata = apDefocalPairs.getDefocusPair(imagedata)
# get tilted parameters first:
if params['ctftilt'] is True:
ctfdata = ctfdb.getBestTiltCtfValueForImage(imagedata)
if ctfdata is None:
apDisplay.printError("Failed to get ctf parameters")
# get x,y coordinate of the particle
nx = particle['particle']['xcoord']
ny = particle['particle']['ycoord']
df1, df2 = ctfdb.getParticleTiltDefocus(
ctfdata, imagedata, nx, ny)
# use defocus & astigmatism values
particleparams['df1'] = abs(df1)
particleparams['df2'] = abs(df2)
particleparams['angast'] = ctfdata['angle_astigmatism']
else:
# first see if there are ctf values
ctfdata, confidence = ctfdb.getBestCtfValueForImage(
imagedata, msg=False, method=params['ctfmethod'])
## Do we really want value from any method if it is specified?
if ctfdata is None:
ctfdata, confidence = ctfdb.getBestCtfValueForImage(
imagedata, msg=False)
if ctfdata is not None:
# use defocus & astigmatism values
particleparams['df1'] = abs(ctfdata['defocus1'] * 1e10)
particleparams['df2'] = abs(ctfdata['defocus2'] * 1e10)
particleparams['angast'] = ctfdata['angle_astigmatism']
# if using parameters from previous reconstruction
if params['reconiterid'] is not None:
params['mode'] = 1
getStackParticleEulersForIteration(params,
particle['particleNumber'])
fr_eulers = convertAppionEmanEulersToFrealign(
params['eman_orient'], sym_name)
particleparams['psi'] = fr_eulers['psi']
particleparams['theta'] = fr_eulers['theta']
particleparams['phi'] = fr_eulers['phi']
particleparams['shx'] = -params['eman_orient']['shiftx']
particleparams['shy'] = -params['eman_orient']['shifty']
if params['eman_orient']['mirror'] is True:
particleparams['shx'] *= -1
writeParticleParamLine(particleparams, f)
f.close()
def runAce(matlab, imgdata, params, showprev=True):
imgname = imgdata['filename']
if showprev is True:
bestctfvalue, bestconf = ctfdb.getBestCtfValueForImage(imgdata)
if bestctfvalue:
print ( "Prev best: '"+bestctfvalue['acerun']['name']+"', conf="+
apDisplay.colorProb(bestconf)+", defocus="+str(round(-1.0*abs(bestctfvalue['defocus1']*1.0e6),2))+
" microns" )
if params['uncorrected']:
tmpname='temporaryCorrectedImage.mrc'
imgarray = apDBImage.correctImage(imgdata)
imgpath = os.path.join(params['rundir'],tmpname)
apImage.arrayToMrc(imgarray, imgpath)
print "processing", imgpath
else:
imgpath = os.path.join(imgdata['session']['image path'], imgname+'.mrc')
nominal = None
if params['nominal'] is not None:
nominal=params['nominal']
elif params['newnominal'] is True:
nominal = ctfdb.getBestDefocusForImage(imgdata, msg=True)
if nominal is None:
nominal = imgdata['scope']['defocus']
if nominal is None or nominal > 0 or nominal < -15e-6:
apDisplay.printWarning("Nominal should be of the form nominal=-1.2e-6"+\
" for -1.2 microns NOT:"+str(nominal))
#Neil's Hack
#if 'autosample' in params and params['autosample']:
# x = abs(nominal*1.0e6)
# val = 1.585 + 0.057587 * x - 0.044106 * x**2 + 0.010877 * x**3
# resamplefr_override = round(val,3)
# print "resamplefr_override=",resamplefr_override
# pymat.eval(matlab, "resamplefr="+str(resamplefr_override)+";")
pymat.eval(matlab,("dforig = %e;" % nominal))
if params['stig'] == 0:
plist = (imgpath, params['outtextfile'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/")
acecmd = makeMatlabCmd("ctfparams = ace(",");",plist)
else:
plist = (imgname, imgpath, params['outtextfile'], params['opimagedir'], \
params['matdir'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/", params['resamplefr'])
acecmd = makeMatlabCmd("ctfparams = measureAstigmatism(",");",plist)
#print acecmd
pymat.eval(matlab,acecmd)
matfile = os.path.join(params['matdir'], imgname+".mrc.mat")
if params['stig']==0:
savematcmd = "save('"+matfile+"','ctfparams','scopeparams', 'dforig');"
pymat.eval(matlab,savematcmd)
ctfvalue = pymat.get(matlab, 'ctfparams')
ctfvalue=ctfvalue[0]
ctfdb.printResults(params, nominal, ctfvalue)
return ctfvalue
def runAce(matlab, imgdata, params, showprev=True):
imgname = imgdata['filename']
if showprev is True:
bestctfvalue, bestconf = ctfdb.getBestCtfValueForImage(imgdata)
if bestctfvalue:
print("Prev best: '" + bestctfvalue['acerun']['name'] +
"', conf=" + apDisplay.colorProb(bestconf) + ", defocus=" +
str(round(-1.0 * abs(bestctfvalue['defocus1'] * 1.0e6), 2)) +
" microns")
if params['uncorrected']:
tmpname = 'temporaryCorrectedImage.mrc'
imgarray = apDBImage.correctImage(imgdata)
imgpath = os.path.join(params['rundir'], tmpname)
apImage.arrayToMrc(imgarray, imgpath)
print "processing", imgpath
else:
imgpath = os.path.join(imgdata['session']['image path'],
imgname + '.mrc')
nominal = None
if params['nominal'] is not None:
nominal = params['nominal']
elif params['newnominal'] is True:
nominal = ctfdb.getBestDefocusForImage(imgdata, msg=True)
if nominal is None:
nominal = imgdata['scope']['defocus']
if nominal is None or nominal > 0 or nominal < -15e-6:
apDisplay.printWarning("Nominal should be of the form nominal=-1.2e-6"+\
" for -1.2 microns NOT:"+str(nominal))
#Neil's Hack
#if 'autosample' in params and params['autosample']:
# x = abs(nominal*1.0e6)
# val = 1.585 + 0.057587 * x - 0.044106 * x**2 + 0.010877 * x**3
# resamplefr_override = round(val,3)
# print "resamplefr_override=",resamplefr_override
# pymat.eval(matlab, "resamplefr="+str(resamplefr_override)+";")
pymat.eval(matlab, ("dforig = %e;" % nominal))
if params['stig'] == 0:
plist = (imgpath, params['outtextfile'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/")
acecmd = makeMatlabCmd("ctfparams = ace(", ");", plist)
else:
plist = (imgname, imgpath, params['outtextfile'], params['opimagedir'], \
params['matdir'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/", params['resamplefr'])
acecmd = makeMatlabCmd("ctfparams = measureAstigmatism(", ");", plist)
#print acecmd
pymat.eval(matlab, acecmd)
matfile = os.path.join(params['matdir'], imgname + ".mrc.mat")
if params['stig'] == 0:
savematcmd = "save('" + matfile + "','ctfparams','scopeparams', 'dforig');"
pymat.eval(matlab, savematcmd)
ctfvalue = pymat.get(matlab, 'ctfparams')
ctfvalue = ctfvalue[0]
ctfdb.printResults(params, nominal, ctfvalue)
return ctfvalue
def generateParticleParams(params,modeldata,initparfile='params.0.par'):
params['inpar']=os.path.join(params['rundir'],initparfile)
apDisplay.printMsg("Creating parameter file: "+params['inpar'])
params['mode']=3
stackdata=getStackParticlesInOrder(params)
first_imageid = stackdata[0]['particle']['image'].dbid
f=open(params['inpar'],'w')
params['noClassification']=0
if params['reconiterid']:
iterdata = apRecon.getRefineIterDataFromIterationId(params['reconiterid'])
sym_name = modeldata['symmetry']['symmetry']
print "Writing out particle parameters"
if 'last' not in params:
params['last'] = len(stackdata)
for i, particle in enumerate(stackdata[:params['last']]):
# defaults
particleparams = initializeParticleParams(i)
# for helical reconstructions, film is helix number
if particle['particle']['helixnum']:
imgid=particle['particle']['image'].dbid
helix=particle['particle']['helixnum']
try:
if params['lastimgid']!=imgid or params['lasthelix']!=helix:
params['totalHelix']+=1
except:
params['totalHelix']=1
params['lastimgid']=imgid
params['lasthelix']=helix
particleparams['film']=params['totalHelix']
else:
particleparams['film']=particle['particle']['image'].dbid - first_imageid + 1
# extra particle number information not read by Frealign
particleparams['pnumber']=particle['particleNumber']
imagedata=particle['particle']['image']
if params['noctf'] is False:
if params['defocpair'] is True:
imagedata = apDefocalPairs.getDefocusPair(imagedata)
# get tilted parameters first:
if params['ctftilt'] is True:
ctfdata = ctfdb.getBestTiltCtfValueForImage(imagedata)
if ctfdata is None:
apDisplay.printError("Failed to get ctf parameters")
# get x,y coordinate of the particle
nx = particle['particle']['xcoord']
ny = particle['particle']['ycoord']
df1,df2 = ctfdb.getParticleTiltDefocus(ctfdata,imagedata,nx,ny)
# use defocus & astigmatism values
particleparams['df1']=abs(df1)
particleparams['df2']=abs(df2)
particleparams['angast']=ctfdata['angle_astigmatism']
else:
# first see if there are ctf values
ctfdata, confidence=ctfdb.getBestCtfValueForImage(imagedata, msg=False,method=params['ctfmethod'])
## Do we really want value from any method if it is specified?
if ctfdata is None:
ctfdata, confidence=ctfdb.getBestCtfValueForImage(imagedata, msg=False)
if ctfdata is not None:
# use defocus & astigmatism values
particleparams['df1']=abs(ctfdata['defocus1']*1e10)
particleparams['df2']=abs(ctfdata['defocus2']*1e10)
particleparams['angast']=ctfdata['angle_astigmatism']
# if using parameters from previous reconstruction
if params['reconiterid'] is not None:
params['mode']=1
getStackParticleEulersForIteration(params,particle['particleNumber'])
fr_eulers = convertAppionEmanEulersToFrealign(params['eman_orient'],sym_name)
particleparams['psi'] = fr_eulers['psi']
particleparams['theta'] = fr_eulers['theta']
particleparams['phi'] = fr_eulers['phi']
particleparams['shx']=-params['eman_orient']['shiftx']
particleparams['shy']=-params['eman_orient']['shifty']
if params['eman_orient']['mirror'] is True:
particleparams['shx']*=-1
writeParticleParamLine(particleparams,f)
f.close()
def processImage(self, imgdata):
self.ctfvalues = {}
### convert image to spider
spiderimage = apDisplay.short(imgdata['filename'])+".spi"
spider.write(imgdata['image'], spiderimage)
### high tension in kiloVolts
kvolts = imgdata['scope']['high tension']/1000.0
### spherical aberration in millimeters
cs = apInstrument.getCsValueFromSession(self.getSessionData())
### pixel size in Angstroms
apix = apDatabase.getPixelSize(imgdata)
### write image name
inputstr = ""
inputstr += ("\n")
inputstr += ("# image filename in spider format\n")
inputstr += ("image=%s\n"%(spiderimage))
### common parameters that never change
inputstr += ("\n")
inputstr += ("# common parameters that never change\n")
inputstr += ("N_horizontal=%d\n"%(self.params['fieldsize']))
#inputstr += ("particle_horizontal=%d\n"%(128))
inputstr += ("show_optimization=yes\n")
inputstr += ("micrograph_averaging=yes\n")
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents the typically peak at the center of the PSD to interfere with CTF estimation.
The default value is 0.05, but for micrographs with a very fine sampling this may be lowered towards 0.0
"""
#minres = apix/minfreq => minfreq = apix/minres
minfreq = apix/self.params['resmin']
inputstr += ("min_freq=%.3f\n"%(minfreq))
""" Give a value in digital frequency (i.e. between 0.0 and 0.5)
This cut-off prevents high-resolution terms where only noise exists to interfere with CTF estimation.
The default value is 0.35, but it should be increased for micrographs with signals extending beyond this value
"""
#maxres = apix/maxfreq => maxfreq = apix/maxres
maxfreq = apix/self.params['resmax']
inputstr += ("max_freq=%.3f\n"%(maxfreq))
### CTF input parameters from database
inputstr += ("\n")
inputstr += ("# CTF input parameters from database\n")
inputstr += ("voltage=%d\n"%(kvolts))
inputstr += ("spherical_aberration=%.1f\n"%(cs))
inputstr += ("sampling_rate=%.4f\n"%(apix))
### best defocus in negative Angstroms
ctfvalue, conf = ctfdb.getBestCtfValueForImage(imgdata, msg=True)
if ctfvalue is None:
apDisplay.printWarning("Xmipp CTF as implemented in Appion requires an initial CTF estimate to process"
+"\nPlease run another CTF program first and try again on this image")
self.ctfvalues = None
return
nominal = (ctfvalue['defocus1']+ctfvalue['defocus2'])/2.0
inputstr += ("defocusU=%d\n"%(-abs(ctfvalue['defocus1'])*1.0e10))
inputstr += ("defocusV=%d\n"%(-abs(ctfvalue['defocus2'])*1.0e10))
inputstr += ("Q0=%d\n"%(-ctfvalue['amplitude_contrast']))
inputstr += ("\n")
### open and write to input parameter file
paramfile = apDisplay.short(imgdata['filename'])+"-CTF.prm"
f = open(paramfile, "w")
print inputstr
f.write(inputstr)
f.close()
#[min_freq=<f=0.05>]
#[max_freq=<f=0.35>]
xmippcmd = "xmipp_ctf_estimate_from_micrograph -i %s"%(paramfile)
#xmippcmd = "echo %s"%(paramfile)
t0 = time.time()
apDisplay.printMsg("running ctf estimation at "+time.asctime())
proc = subprocess.Popen(xmippcmd, shell=True, stdout=subprocess.PIPE)
#waittime = 2
#while proc.poll() is None:
# sys.stderr.write(".")
# time.sleep(waittime)
(stdout, stderr) = proc.communicate()
#print (stdout, stderr)
apDisplay.printMsg("ctf estimation completed in "+apDisplay.timeString(time.time()-t0))
### read commandline output to get fit score
lines = stdout.split('\n')
lastline = ""
for line in lines[-50:]:
if "--->" in line:
lastline = line
if not "--->" in lastline:
apDisplay.printWarning("Xmipp CTF failed")
self.badprocess = True
return
bits = lastline.split('--->')
confidence = float(bits[1])
score = round(math.sqrt(1-confidence), 5)
apDisplay.printColor("Confidence value is %.5f (score %.5f)"%(confidence, score), "cyan")
f = open("confidence.log", "a")
f.write("Confidence value is %.5f for image %s (score %.3f)\n"
%(confidence, apDisplay.short(imgdata['filename']), score))
f.close()
### delete image in spider format no longer needed
apFile.removeFile(spiderimage)
### read output parameter file
outparamfile = apDisplay.short(imgdata['filename'])+"_Periodogramavg.ctfparam"
f = open(outparamfile, "r")
for line in f:
sline = line.strip()
bits = sline.split('=')
if sline.startswith("defocusU"):
defocus1 = float(bits[1].strip())
elif sline.startswith("defocusV"):
defocus2 = float(bits[1].strip())
elif sline.startswith("azimuthal_angle"):
angle_astigmatism = float(bits[1].strip())
elif sline.startswith("Q0"):
amplitude_contrast = abs(float(bits[1].strip()))
print defocus1, defocus2, angle_astigmatism, amplitude_contrast
#defocusU= -18418.6
#defocusV= -24272.1
#azimuthal_angle= 79.7936
#Q0= -0.346951 #negative of ACE amplitude_contrast
print "AMP CONTRAST: %.4f -- %.4f"%(amplitude_contrast, ctfvalue['amplitude_contrast'])
self.ctfvalues = {
'defocus1': defocus1*1e-10,
'defocus2': defocus2*1e-10,
'angle_astigmatism': angle_astigmatism,
'amplitude_contrast': amplitude_contrast,
'nominal': nominal,
'defocusinit': nominal,
'confidence_d': score,
'cs': self.params['cs'],
'volts': kvolts*1000.0,
}
return
def generateParticleParams(self):
paramfile = os.path.join(self.params["rundir"], "params.iter000.par")
apDisplay.printMsg("Creating parameter file: " + paramfile)
numpart = apStack.getNumberStackParticlesFromId(self.params["stackid"])
if os.path.isfile(paramfile):
f = open(paramfile, "r")
numparam = len(f.readlines())
if numparam > self.params["last"]:
apDisplay.printMsg("Param file exists")
return paramfile
else:
apDisplay.printWarning("Param file exists with too few particles: %d vs %d" % (numparam, numpart))
stackdata = apStack.getStackParticlesFromId(self.params["stackid"])
particleparams = {}
f = open(paramfile, "w")
f.write("C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n")
apDisplay.printMsg("Writing out particle parameters")
count = 0
t0 = time.time()
self.noeulers = 0
for particle in stackdata:
count += 1
if count % 200 == 0:
estime = (time.time() - t0) * (numpart - count) / float(count)
apDisplay.printMsg("particle %d -- %s remain" % (count, apDisplay.timeString(estime)))
if count > self.params["last"]:
break
# defaults
## Niko says that if the defocus is negative it will not perform CTF correction
## But it will also not correct the amplitudes
particleparams = {
"ptclnum": particle["particleNumber"],
"psi": 0.0,
"theta": 0.0,
"phi": 0.0,
"df1": -1.0, # workaround if no ctf correction
"df2": -1.0, # set defocus to -1.0 Angstroms
"angast": 0.0,
"mag": 10000, # workaround to get around dstep
"shx": 0.0,
"shy": 0.0,
"film": 1,
"presa": 0.0,
"dpres": 0.0,
}
imagedata = particle["particle"]["image"]
if self.params["noctf"] is False:
ctfdata, confidence = ctfdb.getBestCtfValueForImage(
imagedata, msg=False, method=self.params["ctfmethod"]
)
if ctfdata is not None:
### use defocus and astigmatism values
particleparams["df1"] = abs(ctfdata["defocus1"] * 1e10)
particleparams["df2"] = abs(ctfdata["defocus2"] * 1e10)
particleparams["angast"] = ctfdata["angle_astigmatism"]
# if using parameters from previous reconstruction
if self.params["reconiterid"] is not None:
emaneuler = self.getStackParticleEulersForIteration(particle["particleNumber"])
frealigneulers = apFrealign.convertEmanEulersToFrealign(emaneuler, sym=self.params["sym"])
particleparams["psi"] = frealigneulers["psi"]
particleparams["theta"] = frealigneulers["theta"]
particleparams["phi"] = frealigneulers["phi"]
particleparams["shx"] = emaneuler["shiftx"]
particleparams["shy"] = emaneuler["shifty"]
if emaneuler["mirror"] is True:
particleparams["shx"] *= -1
self.writeParticleParamLine(particleparams, f)
f.close()
return paramfile
