def defocus_guess(Res_roo, Res_TE, volt, Cs, Pixel_size, wgh, istart=0, istop=-1, defocus_estimation_method=2, round_off=1, dz_low=1000., dz_high=200000., nloop=100, ampcont=0.0): """ Use specified frequencies area (istart-istop)to estimate defocus 1. The searching range is limited to dz_low (.1um) ~ dz_high (20 um). The user can modify this limitation accordingly 2. changing nloop can speed up the estimation 3. mode=1 compare squared error mode=2; compare cross correlation coefficients """ from math import sqrt from utilities import generate_ctf if istop <= istart : istop=len(Res_roo) step = (dz_high-dz_low)/nloop if step > 10000. : step = 10000. # angstrom if defocus_estimation_method == 1 : diff_min = 1.e38 else: diff_min = -1.e38 xval_e = 0 for ifreq in xrange(len(Res_TE)): xval_e += Res_TE[ifreq]**2 if (xval_e == 0): defocus = 0 return defocus if round_off >= 1: cut_off = 1. else: cut_off = round_off # do extreme fitting while (step >= cut_off): for i_dz in xrange(nloop): dz = dz_low + step*i_dz diff = 0 length = len(Res_roo) nx = int(length*2) ctf = ctf_2(nx, generate_ctf([dz,Cs,volt,Pixel_size,ampcont,wgh])) if defocus_estimation_method == 1: for ifreq in xrange(istart, istop, 1): diff += (ctf[ifreq]*Res_TE[ifreq] - Res_roo[ifreq])**2 if diff < diff_min or dz == dz_low: defocus = dz diff_min = diff else: diff = 0.0 sum_a = 0.0 sum_b = 0.0 for ifreq in xrange(istart, istop, 1): xval = ctf[ifreq]**2*Res_TE[ifreq] diff += Res_roo[ifreq]**2*xval sum_a += Res_roo[ifreq]**4 sum_b += xval**2 diff/=(sqrt(sum_a*sum_b)*( istop - istart + 1 )) if diff > diff_min or dz == dz_low: defocus = dz diff_min = diff xscore = diff_min dz_low = defocus-step*2 if( dz_low < 0 ): dz_low=0.0 dz_high = defocus + step*2 step/=10. defocus=int( defocus/round_off )*round_off return defocus
def stackToHDF(infile, outfile, apix, pinfo=None):
"""
convert stack to an hdf stack
pinfo contains CTF information from apFrealign.getPerParticleCTF
pinfo may also contain helical information
"""
from utilities import generate_ctf
a = EMAN2.EMData()
imn = EMAN2.EMUtil.get_image_count(infile)
if pinfo is not None:
if len(pinfo) != imn:
apDisplay.printError("insufficient particle info for stack")
# output must end with hdf
outf, ext = os.path.splitext(outfile)
if ext != '.hdf':
outstack = outf + ".hdf"
apDisplay.printMsg("Generating '%s' with %i particles" % (outstack, imn))
for i in xrange(imn):
a.read_image(infile, i)
a.set_attr_dict({'active': 1})
t2 = EMAN2.Transform({
"type": "spider",
"phi": 0,
"theta": 0,
"psi": 0
})
a.set_attr("xform.projection", t2)
a.set_attr("apix_x", apix)
if pinfo is not None:
pdata = pinfo[i + 1]
df1 = pdata['df1']
df2 = pdata['df2']
astig = pdata['angastig']
kv = pdata['kev']
cs = pdata['cs']
ampc = pdata['ampc'] * 100
# save CTF dict (bfactor is hard coded to 0.0)
df = (float(df1) + float(df2)) / 2
ctfgen = generate_ctf([df, cs, kv, apix, 0.0, ampc])
a.set_attr("ctf", ctfgen)
# if helical info is present
if pdata['hnum'] is not None:
a.set_attr("h_angle", pdata['hangle'])
a.write_image(outstack, i)
return outstack
def main():
import os
import sys
from optparse import OptionParser
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = progname + """ input_ctf_params
input_ctf_params: output file from sx_cter.py, which was run in Multi-micrograph mode
Note: files to be flipped should be in same directory.
Outputs: phase-flipped files in hdf format, with identical filenames as input
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--input_ctf_params",
type="string",
default="partres.txt",
help="output from sxcter.py")
(options, args) = parser.parse_args(arglist[1:])
infile = options.input_ctf_params
padimg = True #ctf correction -- pad image
sign = 1 # ctf correction - sign
binary = 1 # ctf correction: phaseflip only
from utilities import generate_ctf
from filter import filt_ctf
global_def.BATCH = True
file_content = open(infile)
ctflist = file_content.readlines()
img = EMData()
for line in ctflist:
data = line.split()
imgfile = data[17]
print "Working on file %s\n" % (imgfile)
(name, ext) = imgfile.split(".")
outfile = name + '.hdf' # Assumes input files are NOT hdf format! otherwise will overwrite them
df = float(data[0])
cs = float(data[1])
voltage = float(data[2])
apix = float(data[3])
bfactor = float(data[4])
amp_contrast = float(data[5])
astig = float(data[6])
astig_ang = float(data[7])
ctf = generate_ctf(
[df, cs, voltage, apix, bfactor, amp_contrast, astig, astig_ang])
img.read_image(imgfile)
corr_img = filt_ctf(img, ctf, padimg, sign, binary)
corr_img.write_image(outfile)
def generate_helimic(refvol, outdir, pixel, CTF=False, Cs=2.0,voltage = 200.0, ampcont = 10.0, nonoise = False, rand_seed=14567):
from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im
from random import random
from projection import prgs, prep_vol
from filter import filt_gaussl, filt_ctf
from EMAN2 import EMAN2Ctf
if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxhelical_demo", 1)
os.mkdir(outdir)
seed(rand_seed)
Util.set_randnum_seed(rand_seed)
angles =[]
for i in xrange(3):
angles.append( [0.0+60.0*i, 90.0-i*5, 0.0, 0.0, 0.0] )
nangle = len(angles)
volfts = get_im(refvol)
nx = volfts.get_xsize()
ny = volfts.get_ysize()
nz = volfts.get_zsize()
volfts, kbx, kby, kbz = prep_vol( volfts )
iprj = 0
width = 500
xstart = 0
ystart = 0
for idef in xrange(3,6):
mic = model_blank(2048, 2048)
#defocus = idef*0.2
defocus = idef*0.6 ##@ming
if CTF :
#ctf = EMAN2Ctf()
#ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
from utilities import generate_ctf
ctf = generate_ctf([defocus,2,200,1.84,0.0,ampcont,defocus*0.2,80]) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
i = idef - 4
for k in xrange(1):
psi = 90 + 10*i
proj = prgs(volfts, kbz, [angles[idef-3][0], angles[idef-3][1], psi, 0.0, 0.0], kbx, kby)
proj = Util.window(proj, 320, nz)
mic += pad(proj, 2048, 2048, 1, 0.0, 750*i, 20*i, 0)
if not nonoise: mic += model_gauss_noise(30.0,2048,2048)
if CTF :
#apply CTF
mic = filt_ctf(mic, ctf)
if not nonoise: mic += filt_gaussl(model_gauss_noise(17.5,2048,2048), 0.3)
mic.write_image("%s/mic%1d.hdf"%(outdir, idef-3),0)
def stackToHDF(infile,outfile,apix,pinfo=None):
"""
convert stack to an hdf stack
pinfo contains CTF information from apFrealign.getPerParticleCTF
pinfo may also contain helical information
"""
from utilities import generate_ctf
a = EMAN2.EMData()
imn = EMAN2.EMUtil.get_image_count(infile)
if pinfo is not None:
if len(pinfo) != imn:
apDisplay.printError("insufficient particle info for stack")
# output must end with hdf
outf,ext = os.path.splitext(outfile)
if ext != '.hdf':
outstack = outf+".hdf"
apDisplay.printMsg("Generating '%s' with %i particles"%(outstack,imn))
for i in xrange(imn):
a.read_image(infile,i)
a.set_attr_dict({'active':1})
t2 = EMAN2.Transform({"type":"spider","phi":0,"theta":0,"psi":0})
a.set_attr("xform.projection", t2)
a.set_attr("apix_x",apix)
if pinfo is not None:
pdata = pinfo[i+1]
df1 = pdata['df1']
df2 = pdata['df2']
astig = pdata['angastig']
kv = pdata['kev']
cs = pdata['cs']
ampc = pdata['ampc']*100
# save CTF dict (bfactor is hard coded to 0.0)
df=(float(df1)+float(df2))/2
ctfgen = generate_ctf([df,cs,kv,apix,0.0,ampc])
a.set_attr("ctf",ctfgen)
# if helical info is present
if pdata['hnum'] is not None:
a.set_attr("h_angle",pdata['hangle'])
a.write_image(outstack,i)
return outstack
def defocus_gett(roo, voltage=300.0, Pixel_size=1.0, Cs=2.0, wgh=0.1, f_start=0.0, f_stop=-1.0, round_off=1.0, nr1=3, nr2=6, parent=None): """ 1. Estimate envelope function and baseline noise using constrained simplex method so as to extract CTF imprints from 1D power spectrum 2. Based one extracted ctf imprints, perform exhaustive defocus searching to get defocus which matches the extracted CTF imprints """ from utilities import generate_ctf #print "CTF params:", voltage, Pixel_size, Cs, wgh, f_start, f_stop, round_off, nr1, nr2, parent res = [] Res_roo = [] Res_TE = [] if f_start == 0 : i_start = 0 else: i_start = int(Pixel_size*2.*len(roo)*f_start) if f_stop <= f_start : i_stop = len(roo) else: i_stop = int(Pixel_size*2.*len(roo)*f_stop) if i_stop > len(roo): i_stop = len(roo) #print "f_start, i_start, f_stop, i_stop:", f_start, i_start, f_stop, i_stop TE = defocus_env_baseline_fit(roo, i_start, i_stop, int(nr1), 4) Pn1 = defocus_env_baseline_fit(roo, i_start, i_stop, int(nr2), 3) Res_roo = [] Res_TE = [] for i in xrange(len(roo)): Res_roo.append(roo[i] - Pn1[i]) Res_TE.append( TE[i] - Pn1[i]) defocus = defocus_guess(Res_roo, Res_TE, voltage, Cs, Pixel_size, wgh, i_start, i_stop, 2, round_off) nx = int(len(Res_roo)*2) #ctf = ctf_1d(nx, generate_ctf([defocus, Cs, voltage, Pixel_size, 0.0, wgh])) ctf = ctf_2(nx, generate_ctf([defocus, Cs, voltage, Pixel_size, 0.0, wgh])) if (parent is not None): parent.ctf_data=[roo, Res_roo, Res_TE] parent.i_start = i_start parent.i_stop = i_stop from utilities import write_text_file write_text_file([roo, Res_roo, Res_TE, ctf], "procpw.txt") else: from utilities import write_text_file write_text_file([roo, Res_roo, Res_TE, ctf], "procpw.txt") return defocus
def defocus_get_Eudis(fnam_roo, volt=300, Pixel_size=1, Cs=2, wgh=.1, f_start=0, f_stop=-1, docf="a" ,skip="#", round_off=1, nr1=3, nr2=6): """ 1. Estimating envelope function and baseline noise using constrained simplex method so as to extract CTF imprints from 1D power spectrum 2. Based one extracted ctf imprints, perform exhaustive defocus searching to get defocus which matches the extracted CTF imprints 3. It returns Euclidean distance for defocus selection """ from math import sqrt, atan from utilities import read_text_row, generate_ctf roo = [] res = [] if docf == "a": TMP_roo = read_text_row(fnam_roo, "a", skip) for i in xrange(len(TMP_roo)): # remove first record roo.append(TMP_roo[i][1]) else: skip = ";" TMP_roo = read_text_row(fnam_roo, "s", skip) for i in xrange(len(TMP_roo)): # remove first record roo.append(TMP_roo[i][2]) Res_roo = [] Res_TE = [] if f_start == 0 : i_start=0 else: i_start=int(Pixel_size*2.*len(roo)/f_start) if f_stop <= i_start : i_stop=len(roo) else: i_stop=int(Pixel_size*2.*len(roo)/f_stop) TE = defocus_env_baseline_fit(roo, i_start, i_stop, int(nr1), 4) Pn1 = defocus_env_baseline_fit(roo, i_start, i_stop, int(nr2), 3) Res_roo = [] Res_TE = [] for i in xrange(len(roo)): Res_roo.append( roo[i] - Pn1[i] ) Res_TE.append( TE[i] - Pn1[i] ) # defocus=defocus_guess(Res_roo, Res_TE, volt, Cs, Pixel_size, wgh, i_start, i_stop, 2, round_off) nx = int(len(roo)*2) ctf = ctf_2(nx, generate_ctf([defocus,Cs,voltage,Pixel_size, 0.0, wgh])) for i in xrange(len(Res_TE)): ctf[i]=ctf[i]*Res_TE[i] dis = defocus_L2_euc(ctf, Res_roo, i_start, i_stop) return [defocus, dis]
def generate_helimic(refvol,
outdir,
pixel,
CTF=False,
Cs=2.0,
voltage=200.0,
ampcont=10.0,
nonoise=False,
rand_seed=14567):
from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im
from random import random
from projection import prgs, prep_vol
from filter import filt_gaussl, filt_ctf
from EMAN2 import EMAN2Ctf
if os.path.exists(outdir):
ERROR(
'Output directory exists, please change the name and restart the program',
"sxhelical_demo", 1)
os.mkdir(outdir)
seed(rand_seed)
Util.set_randnum_seed(rand_seed)
angles = []
for i in range(3):
angles.append([0.0 + 60.0 * i, 90.0 - i * 5, 0.0, 0.0, 0.0])
nangle = len(angles)
volfts = get_im(refvol)
nx = volfts.get_xsize()
ny = volfts.get_ysize()
nz = volfts.get_zsize()
volfts, kbx, kby, kbz = prep_vol(volfts)
iprj = 0
width = 500
xstart = 0
ystart = 0
for idef in range(3, 6):
mic = model_blank(2048, 2048)
#defocus = idef*0.2
defocus = idef * 0.6 ##@ming
if CTF:
#ctf = EMAN2Ctf()
#ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
from utilities import generate_ctf
ctf = generate_ctf(
[defocus, 2, 200, 1.84, 0.0, ampcont, defocus * 0.2, 80]
) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
i = idef - 4
for k in range(1):
psi = 90 + 10 * i
proj = prgs(
volfts, kbz,
[angles[idef - 3][0], angles[idef - 3][1], psi, 0.0, 0.0], kbx,
kby)
proj = Util.window(proj, 320, nz)
mic += pad(proj, 2048, 2048, 1, 0.0, 750 * i, 20 * i, 0)
if not nonoise: mic += model_gauss_noise(30.0, 2048, 2048)
if CTF:
#apply CTF
mic = filt_ctf(mic, ctf)
if not nonoise:
mic += filt_gaussl(model_gauss_noise(17.5, 2048, 2048), 0.3)
mic.write_image("%s/mic%1d.hdf" % (outdir, idef - 3), 0)
def main2():
arglist = []
for arg in sys.argv:
arglist.append( arg )
progname = os.path.basename(arglist[0])
usage = progname + " raw_images apix <CTFparfile>"
parser = OptionParser(usage,version="1.0")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
else:
a = EMData()
fname = args[0]
apix = float(args[1])
ctfpar = None
helical = False
if len(args) == 3:
ctfpar = args[2]
if not os.path.isfile(ctfpar):
print "Error: ctf file '%s' is not found"%ctfpar
sys.exit()
# get ctf values from parfile
from utilities import generate_ctf
pfile = open(ctfpar).readlines()
ctfinfo=[0]*len(pfile)
# check if there are helical parameters:
if len(pfile[0].strip().split())==9:
helical = True
hinfo=[0]*len(pfile)
# store info
for p in pfile:
nfo = p.strip().split()
pnum=int(float(nfo[0]))-1
df1=float(nfo[1])
df2=float(nfo[2])
astig = float(nfo[3])
kv = int(float(nfo[4]))
cs = float(nfo[5])
ampc = float(nfo[6])*100
if helical is True:
hnum = int(float(nfo[7]))
angle = float(nfo[8])
angle = -angle+90
if angle > 360: angle -= 360
if angle < 0: angle += 360
hinfo[pnum]=angle
"""
# For future use if taking from FREALIGN
if p[0]=="C":
continue
(pnum,psi,theta,phi,shx,shy,mag,hnum,ctf1,ctf2,angle,pres,delta)=p.strip().split()
pnum=int(pnum)-1
kv=300
cs=2.7
ampc=5
"""
df=(float(df1)+float(df2))/2
ctfinfo[pnum]=generate_ctf([df,cs,kv,apix,0.0,ampc])
imn = EMUtil.get_image_count(fname)
print "Generating 'start.hdf' with %i particles"%imn
for i in xrange(imn):
a.read_image(fname,i)
a.set_attr_dict({'active':1})
if ctfpar is not None:
a.set_attr("ctf",ctfinfo[i])
if helical is True:
a.set_attr("h_angle",hinfo[i])
t2 = Transform({"type":"spider","phi":0,"theta":0,"psi":0})
a.set_attr("xform.projection", t2)
a.set_attr("apix_x",apix )
a.write_image("start.hdf",i)
print "%3i%% complete\t\r"%(int(float(i)/imn*100)),
print "100% complete\t"
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ input_micrograph_list_file input_micrograph_pattern input_coordinates_pattern output_directory --coordinates_format --box_size=box_size --invert --import_ctf=ctf_file --limit_ctf --resample_ratio=resample_ratio --defocus_error=defocus_error --astigmatism_error=astigmatism_error
Window particles from micrographs in input list file. The coordinates of the particles should be given as input.
Please specify name pattern of input micrographs and coordinates files with a wild card (*). Use the wild card to indicate the place of micrograph ID (e.g. serial number, time stamp, and etc).
The name patterns must be enclosed by single quotes (') or double quotes ("). (Note: sxgui.py automatically adds single quotes (')).
BDB files can not be selected as input micrographs.
sxwindow.py mic_list.txt ./mic*.hdf info/mic*_info.json particles --coordinates_format=eman2 --box_size=64 --invert --import_ctf=outdir_cter/partres/partres.txt
If micrograph list file name is not provided, all files matched with the micrograph name pattern will be processed.
sxwindow.py ./mic*.hdf info/mic*_info.json particles --coordinates_format=eman2 --box_size=64 --invert --import_ctf=outdir_cter/partres/partres.txt
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--coordinates_format",
type="string",
default="eman1",
help=
"format of input coordinates files: 'sparx', 'eman1', 'eman2', or 'spider'. the coordinates of sparx, eman2, and spider format is particle center. the coordinates of eman1 format is particle box conner associated with the original box size. (default eman1)"
)
parser.add_option(
"--box_size",
type="int",
default=256,
help=
"x and y dimension of square area to be windowed (in pixels): pixel size after resampling is assumed when resample_ratio < 1.0 (default 256)"
)
parser.add_option(
"--invert",
action="store_true",
default=False,
help="invert image contrast: recommended for cryo data (default False)"
)
parser.add_option(
"--import_ctf",
type="string",
default="",
help="file name of sxcter output: normally partres.txt (default none)")
parser.add_option(
"--limit_ctf",
action="store_true",
default=False,
help=
"filter micrographs based on the CTF limit: this option requires --import_ctf. (default False)"
)
parser.add_option(
"--resample_ratio",
type="float",
default=1.0,
help=
"ratio of new to old image size (or old to new pixel size) for resampling: Valid range is 0.0 < resample_ratio <= 1.0. (default 1.0)"
)
parser.add_option(
"--defocus_error",
type="float",
default=1000000.0,
help=
"defocus errror limit: exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocus_error percent. the error is computed as (std dev defocus)/defocus*100%. (default 1000000.0)"
)
parser.add_option(
"--astigmatism_error",
type="float",
default=360.0,
help=
"astigmatism error limit: Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatism_error degrees. (default 360.0)"
)
### detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
main_node = 0
if RUNNING_UNDER_MPI:
from mpi import mpi_init
from mpi import MPI_COMM_WORLD, mpi_comm_rank, mpi_comm_size, mpi_barrier, mpi_reduce, MPI_INT, MPI_SUM
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_processes = mpi_comm_size(MPI_COMM_WORLD)
else:
number_of_processes = 1
myid = 0
(options, args) = parser.parse_args(sys.argv[1:])
mic_list_file_path = None
mic_pattern = None
coords_pattern = None
error_status = None
while True:
if len(args) < 3 or len(args) > 4:
error_status = (
"Please check usage for number of arguments.\n Usage: " +
usage + "\n" + "Please run %s -h for help." % (progname),
getframeinfo(currentframe()))
break
if len(args) == 3:
mic_pattern = args[0]
coords_pattern = args[1]
out_dir = args[2]
else: # assert(len(args) == 4)
mic_list_file_path = args[0]
mic_pattern = args[1]
coords_pattern = args[2]
out_dir = args[3]
if mic_list_file_path != None:
if os.path.splitext(mic_list_file_path)[1] != ".txt":
error_status = (
"Extension of input micrograph list file must be \".txt\". Please check input_micrograph_list_file argument. Run %s -h for help."
% (progname), getframeinfo(currentframe()))
break
if mic_pattern[:len("bdb:")].lower() == "bdb":
error_status = (
"BDB file can not be selected as input micrographs. Please convert the format, and restart the program. Run %s -h for help."
% (progname), getframeinfo(currentframe()))
break
if mic_pattern.find("*") == -1:
error_status = (
"Input micrograph file name pattern must contain wild card (*). Please check input_micrograph_pattern argument. Run %s -h for help."
% (progname), getframeinfo(currentframe()))
break
if coords_pattern.find("*") == -1:
error_status = (
"Input coordinates file name pattern must contain wild card (*). Please check input_coordinates_pattern argument. Run %s -h for help."
% (progname), getframeinfo(currentframe()))
break
if myid == main_node:
if os.path.exists(out_dir):
error_status = (
"Output directory exists. Please change the name and restart the program.",
getframeinfo(currentframe()))
break
break
if_error_then_all_processes_exit_program(error_status)
# Check invalid conditions of options
check_options(options, progname)
mic_name_list = None
error_status = None
if myid == main_node:
if mic_list_file_path != None:
print("Loading micrograph list from %s file ..." %
(mic_list_file_path))
mic_name_list = read_text_file(mic_list_file_path)
if len(mic_name_list) == 0:
print("Directory of first micrograph entry is " %
(os.path.dirname(mic_name_list[0])))
else: # assert (mic_list_file_path == None)
print("Generating micrograph list in %s directory..." %
(os.path.dirname(mic_pattern)))
mic_name_list = glob.glob(mic_pattern)
if len(mic_name_list) == 0:
error_status = (
"No micrograph file is found. Please check input_micrograph_pattern and/or input_micrograph_list_file argument. Run %s -h for help."
% (progname), getframeinfo(currentframe()))
else:
print("Found %d microgarphs" % len(mic_name_list))
if_error_then_all_processes_exit_program(error_status)
if RUNNING_UNDER_MPI:
mic_name_list = wrap_mpi_bcast(mic_name_list, main_node)
coords_name_list = None
error_status = None
if myid == main_node:
coords_name_list = glob.glob(coords_pattern)
if len(coords_name_list) == 0:
error_status = (
"No coordinates file is found. Please check input_coordinates_pattern argument. Run %s -h for help."
% (progname), getframeinfo(currentframe()))
if_error_then_all_processes_exit_program(error_status)
if RUNNING_UNDER_MPI:
coords_name_list = wrap_mpi_bcast(coords_name_list, main_node)
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
# all processes must have access to indices
if options.import_ctf:
i_enum = -1
i_enum += 1
idx_cter_def = i_enum # defocus [um]; index must be same as ctf object format
i_enum += 1
idx_cter_cs = i_enum # Cs [mm]; index must be same as ctf object format
i_enum += 1
idx_cter_vol = i_enum # voltage[kV]; index must be same as ctf object format
i_enum += 1
idx_cter_apix = i_enum # pixel size [A]; index must be same as ctf object format
i_enum += 1
idx_cter_bfactor = i_enum # B-factor [A^2]; index must be same as ctf object format
i_enum += 1
idx_cter_ac = i_enum # amplitude contrast [%]; index must be same as ctf object format
i_enum += 1
idx_cter_astig_amp = i_enum # astigmatism amplitude [um]; index must be same as ctf object format
i_enum += 1
idx_cter_astig_ang = i_enum # astigmatism angle [degree]; index must be same as ctf object format
i_enum += 1
idx_cter_sd_def = i_enum # std dev of defocus [um]
i_enum += 1
idx_cter_sd_astig_amp = i_enum # std dev of ast amp [A]
i_enum += 1
idx_cter_sd_astig_ang = i_enum # std dev of ast angle [degree]
i_enum += 1
idx_cter_cv_def = i_enum # coefficient of variation of defocus [%]
i_enum += 1
idx_cter_cv_astig_amp = i_enum # coefficient of variation of ast amp [%]
i_enum += 1
idx_cter_spectra_diff = i_enum # average of differences between with- and without-astig. experimental 1D spectra at extrema
i_enum += 1
idx_cter_error_def = i_enum # frequency at which signal drops by 50% due to estimated error of defocus alone [1/A]
i_enum += 1
idx_cter_error_astig = i_enum # frequency at which signal drops by 50% due to estimated error of defocus and astigmatism [1/A]
i_enum += 1
idx_cter_error_ctf = i_enum # limit frequency by CTF error [1/A]
i_enum += 1
idx_cter_mic_name = i_enum # micrograph name
i_enum += 1
n_idx_cter = i_enum
# Prepare loop variables
mic_basename_pattern = os.path.basename(
mic_pattern) # file pattern without path
mic_baseroot_pattern = os.path.splitext(mic_basename_pattern)[
0] # file pattern without path and extension
coords_format = options.coordinates_format.lower()
box_size = options.box_size
box_half = box_size // 2
mask2d = model_circle(
box_size // 2, box_size, box_size
) # Create circular 2D mask to Util.infomask of particle images
resample_ratio = options.resample_ratio
n_mic_process = 0
n_mic_reject_no_coords = 0
n_mic_reject_no_cter_entry = 0
n_global_coords_detect = 0
n_global_coords_process = 0
n_global_coords_reject_out_of_boundary = 0
serial_id_list = []
error_status = None
## not a real while, an if with the opportunity to use break when errors need to be reported
while myid == main_node:
#
# NOTE: 2016/05/24 Toshio Moriya
# Now, ignores the path in mic_pattern and entries of mic_name_list to create serial ID
# Only the basename (file name) in micrograph path must be match
#
# Create list of micrograph serial ID
# Break micrograph name pattern into prefix and suffix to find the head index of the micrograph serial id
#
mic_basename_tokens = mic_basename_pattern.split('*')
# assert (len(mic_basename_tokens) == 2)
serial_id_head_index = len(mic_basename_tokens[0])
# Loop through micrograph names
for mic_name in mic_name_list:
# Find the tail index of the serial id and extract serial id from the micrograph name
mic_basename = os.path.basename(mic_name)
serial_id_tail_index = mic_basename.index(mic_basename_tokens[1])
serial_id = mic_basename[serial_id_head_index:serial_id_tail_index]
serial_id_list.append(serial_id)
# assert (len(serial_id_list) == len(mic_name))
del mic_name_list # Do not need this anymore
# Load CTFs if necessary
if options.import_ctf:
ctf_list = read_text_row(options.import_ctf)
# print("Detected CTF entries : %6d ..." % (len(ctf_list)))
if len(ctf_list) == 0:
error_status = (
"No CTF entry is found in %s. Please check --import_ctf option. Run %s -h for help."
% (options.import_ctf, progname),
getframeinfo(currentframe()))
break
if (len(ctf_list[0]) != n_idx_cter):
error_status = (
"Number of columns (%d) must be %d in %s. The format might be old. Please run sxcter.py again."
% (len(ctf_list[0]), n_idx_cter, options.import_ctf),
getframeinfo(currentframe()))
break
ctf_dict = {}
n_reject_defocus_error = 0
ctf_error_limit = [
options.defocus_error / 100.0, options.astigmatism_error
]
for ctf_params in ctf_list:
assert (len(ctf_params) == n_idx_cter)
# mic_baseroot is name of micrograph minus the path and extension
mic_baseroot = os.path.splitext(
os.path.basename(ctf_params[idx_cter_mic_name]))[0]
if (ctf_params[idx_cter_sd_def] / ctf_params[idx_cter_def] >
ctf_error_limit[0]):
print(
"Defocus error %f exceeds the threshold. Micrograph %s is rejected."
% (ctf_params[idx_cter_sd_def] /
ctf_params[idx_cter_def], mic_baseroot))
n_reject_defocus_error += 1
else:
if (ctf_params[idx_cter_sd_astig_ang] >
ctf_error_limit[1]):
ctf_params[idx_cter_astig_amp] = 0.0
ctf_params[idx_cter_astig_ang] = 0.0
ctf_dict[mic_baseroot] = ctf_params
del ctf_list # Do not need this anymore
break
if_error_then_all_processes_exit_program(error_status)
if options.import_ctf:
if options.limit_ctf:
cutoff_histogram = [
] #@ming compute the histogram for micrographs cut of by ctf_params limit.
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
restricted_serial_id_list = []
if myid == main_node:
# Loop over serial IDs of micrographs
for serial_id in serial_id_list:
# mic_baseroot is name of micrograph minus the path and extension
mic_baseroot = mic_baseroot_pattern.replace("*", serial_id)
mic_name = mic_pattern.replace("*", serial_id)
coords_name = coords_pattern.replace("*", serial_id)
########### # CHECKS: BEGIN
if coords_name not in coords_name_list:
print(" Cannot read %s. Skipping %s ..." %
(coords_name, mic_baseroot))
n_mic_reject_no_coords += 1
continue
# IF mic is in CTER results
if options.import_ctf:
if mic_baseroot not in ctf_dict:
print(
" Is not listed in CTER results. Skipping %s ..." %
(mic_baseroot))
n_mic_reject_no_cter_entry += 1
continue
else:
ctf_params = ctf_dict[mic_baseroot]
# CHECKS: END
n_mic_process += 1
restricted_serial_id_list.append(serial_id)
# restricted_serial_id_list = restricted_serial_id_list[:128] ## for testing against the nonMPI version
if myid != main_node:
if options.import_ctf:
ctf_dict = None
error_status = None
if len(restricted_serial_id_list) < number_of_processes:
error_status = (
'Number of processes (%d) supplied by --np in mpirun cannot be greater than %d (number of micrographs that satisfy all criteria to be processed) '
% (number_of_processes, len(restricted_serial_id_list)),
getframeinfo(currentframe()))
if_error_then_all_processes_exit_program(error_status)
## keep a copy of the original output directory where the final bdb will be created
original_out_dir = out_dir
if RUNNING_UNDER_MPI:
mpi_barrier(MPI_COMM_WORLD)
restricted_serial_id_list = wrap_mpi_bcast(restricted_serial_id_list,
main_node)
mic_start, mic_end = MPI_start_end(len(restricted_serial_id_list),
number_of_processes, myid)
restricted_serial_id_list_not_sliced = restricted_serial_id_list
restricted_serial_id_list = restricted_serial_id_list[
mic_start:mic_end]
if options.import_ctf:
ctf_dict = wrap_mpi_bcast(ctf_dict, main_node)
# generate subdirectories of out_dir, one for each process
out_dir = os.path.join(out_dir, "%03d" % myid)
if myid == main_node:
print(
"Micrographs processed by main process (including percent complete):"
)
len_processed_by_main_node_divided_by_100 = len(
restricted_serial_id_list) / 100.0
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
##### Starting main parallel execution
for my_idx, serial_id in enumerate(restricted_serial_id_list):
mic_baseroot = mic_baseroot_pattern.replace("*", serial_id)
mic_name = mic_pattern.replace("*", serial_id)
coords_name = coords_pattern.replace("*", serial_id)
if myid == main_node:
print(
mic_name, " ---> % 2.2f%%" %
(my_idx / len_processed_by_main_node_divided_by_100))
mic_img = get_im(mic_name)
# Read coordinates according to the specified format and
# make the coordinates the center of particle image
if coords_format == "sparx":
coords_list = read_text_row(coords_name)
elif coords_format == "eman1":
coords_list = read_text_row(coords_name)
for i in xrange(len(coords_list)):
coords_list[i] = [(coords_list[i][0] + coords_list[i][2] // 2),
(coords_list[i][1] + coords_list[i][3] // 2)]
elif coords_format == "eman2":
coords_list = js_open_dict(coords_name)["boxes"]
for i in xrange(len(coords_list)):
coords_list[i] = [coords_list[i][0], coords_list[i][1]]
elif coords_format == "spider":
coords_list = read_text_row(coords_name)
for i in xrange(len(coords_list)):
coords_list[i] = [coords_list[i][2], coords_list[i][3]]
# else: assert (False) # Unreachable code
# Calculate the new pixel size
if options.import_ctf:
ctf_params = ctf_dict[mic_baseroot]
pixel_size_origin = ctf_params[idx_cter_apix]
if resample_ratio < 1.0:
# assert (resample_ratio > 0.0)
new_pixel_size = pixel_size_origin / resample_ratio
print(
"Resample micrograph to pixel size %6.4f and window segments from resampled micrograph."
% new_pixel_size)
else:
# assert (resample_ratio == 1.0)
new_pixel_size = pixel_size_origin
# Set ctf along with new pixel size in resampled micrograph
ctf_params[idx_cter_apix] = new_pixel_size
else:
# assert (not options.import_ctf)
if resample_ratio < 1.0:
# assert (resample_ratio > 0.0)
print(
"Resample micrograph with ratio %6.4f and window segments from resampled micrograph."
% resample_ratio)
# else:
# assert (resample_ratio == 1.0)
# Apply filters to micrograph
fftip(mic_img)
if options.limit_ctf:
# assert (options.import_ctf)
# Cut off frequency components higher than CTF limit
q1, q2 = ctflimit(box_size, ctf_params[idx_cter_def],
ctf_params[idx_cter_cs],
ctf_params[idx_cter_vol], new_pixel_size)
# This is absolute frequency of CTF limit in scale of original micrograph
if resample_ratio < 1.0:
# assert (resample_ratio > 0.0)
q1 = resample_ratio * q1 / float(
box_size
) # q1 = (pixel_size_origin / new_pixel_size) * q1/float(box_size)
else:
# assert (resample_ratio == 1.0) -> pixel_size_origin == new_pixel_size -> pixel_size_origin / new_pixel_size == 1.0
q1 = q1 / float(box_size)
if q1 < 0.5:
mic_img = filt_tanl(mic_img, q1, 0.01)
cutoff_histogram.append(q1)
# Cut off frequency components lower than the box size can express
mic_img = fft(filt_gaussh(mic_img, resample_ratio / box_size))
# Resample micrograph, map coordinates, and window segments from resampled micrograph using new coordinates
# after resampling by resample_ratio, new pixel size will be pixel_size/resample_ratio = new_pixel_size
# NOTE: 2015/04/13 Toshio Moriya
# resample() efficiently takes care of the case resample_ratio = 1.0 but
# it does not set apix_*. Even though it sets apix_* when resample_ratio < 1.0 ...
mic_img = resample(mic_img, resample_ratio)
if options.invert:
mic_stats = Util.infomask(
mic_img, None, True) # mic_stat[0:mean, 1:SD, 2:min, 3:max]
Util.mul_scalar(mic_img, -1.0)
mic_img += 2 * mic_stats[0]
if options.import_ctf:
from utilities import generate_ctf
ctf_obj = generate_ctf(
ctf_params
) # indexes 0 to 7 (idx_cter_def to idx_cter_astig_ang) must be same in cter format & ctf object format.
# Prepare loop variables
nx = mic_img.get_xsize()
ny = mic_img.get_ysize()
x0 = nx // 2
y0 = ny // 2
n_coords_reject_out_of_boundary = 0
local_stack_name = "bdb:%s#" % out_dir + mic_baseroot + '_ptcls'
local_particle_id = 0 # can be different from coordinates_id
# Loop over coordinates
for coords_id in xrange(len(coords_list)):
x = int(coords_list[coords_id][0])
y = int(coords_list[coords_id][1])
if resample_ratio < 1.0:
# assert (resample_ratio > 0.0)
x = int(x * resample_ratio)
y = int(y * resample_ratio)
# else:
# assert(resample_ratio == 1.0)
if ((0 <= x - box_half) and (x + box_half <= nx)
and (0 <= y - box_half) and (y + box_half <= ny)):
particle_img = Util.window(mic_img, box_size, box_size, 1,
x - x0, y - y0)
else:
print(
"In %s, coordinates ID = %04d (x = %4d, y = %4d, box_size = %4d) is out of micrograph bound, skipping ..."
% (mic_baseroot, coords_id, x, y, box_size))
n_coords_reject_out_of_boundary += 1
continue
particle_img = ramp(particle_img)
particle_stats = Util.infomask(
particle_img, mask2d,
False) # particle_stats[0:mean, 1:SD, 2:min, 3:max]
particle_img -= particle_stats[0]
particle_img /= particle_stats[1]
# NOTE: 2015/04/09 Toshio Moriya
# ptcl_source_image might be redundant information ...
# Consider re-organizing header entries...
particle_img.set_attr("ptcl_source_image", mic_name)
particle_img.set_attr("ptcl_source_coord_id", coords_id)
particle_img.set_attr("ptcl_source_coord", [
int(coords_list[coords_id][0]),
int(coords_list[coords_id][1])
])
particle_img.set_attr("resample_ratio", resample_ratio)
# NOTE: 2015/04/13 Toshio Moriya
# apix_* attributes are updated by resample() only when resample_ratio != 1.0
# Let's make sure header info is consistent by setting apix_* = 1.0
# regardless of options, so it is not passed down the processing line
particle_img.set_attr("apix_x", 1.0)
particle_img.set_attr("apix_y", 1.0)
particle_img.set_attr("apix_z", 1.0)
if options.import_ctf:
particle_img.set_attr("ctf", ctf_obj)
particle_img.set_attr("ctf_applied", 0)
particle_img.set_attr("pixel_size_origin", pixel_size_origin)
# particle_img.set_attr("apix_x", new_pixel_size)
# particle_img.set_attr("apix_y", new_pixel_size)
# particle_img.set_attr("apix_z", new_pixel_size)
# NOTE: 2015/04/13 Toshio Moriya
# Pawel Comment: Micrograph is not supposed to have CTF header info.
# So, let's assume it does not exist & ignore its presence.
# Note that resample() "correctly" updates pixel size of CTF header info if it exists
# elif (particle_img.has_ctff()):
# assert(not options.import_ctf)
# ctf_origin = particle_img.get_attr("ctf_obj")
# pixel_size_origin = round(ctf_origin.apix, 5) # Because SXCTER ouputs up to 5 digits
# particle_img.set_attr("apix_x",pixel_size_origin)
# particle_img.set_attr("apix_y",pixel_size_origin)
# particle_img.set_attr("apix_z",pixel_size_origin)
# print("local_stack_name, local_particle_id", local_stack_name, local_particle_id)
particle_img.write_image(local_stack_name, local_particle_id)
local_particle_id += 1
n_global_coords_detect += len(coords_list)
n_global_coords_process += local_particle_id
n_global_coords_reject_out_of_boundary += n_coords_reject_out_of_boundary
# # MRK_DEBUG: Toshio Moriya 2016/05/03
# # Following codes are for debugging bdb. Delete in future
# result = db_check_dict(local_stack_name)
# print('# MRK_DEBUG: result = db_check_dict(local_stack_name): %s' % (result))
# result = db_list_dicts('bdb:%s' % out_dir)
# print('# MRK_DEBUG: result = db_list_dicts(out_dir): %s' % (result))
# result = db_get_image_info(local_stack_name)
# print('# MRK_DEBUG: result = db_get_image_info(local_stack_name)', result)
# Release the data base of local stack from this process
# so that the subprocess can access to the data base
db_close_dict(local_stack_name)
# # MRK_DEBUG: Toshio Moriya 2016/05/03
# # Following codes are for debugging bdb. Delete in future
# cmd_line = "e2iminfo.py %s" % (local_stack_name)
# print('# MRK_DEBUG: Executing the command: %s' % (cmd_line))
# cmdexecute(cmd_line)
# # MRK_DEBUG: Toshio Moriya 2016/05/03
# # Following codes are for debugging bdb. Delete in future
# cmd_line = "e2iminfo.py bdb:%s#data" % (out_dir)
# print('# MRK_DEBUG: Executing the command: %s' % (cmd_line))
# cmdexecute(cmd_line)
if RUNNING_UNDER_MPI:
if options.import_ctf:
if options.limit_ctf:
cutoff_histogram = wrap_mpi_gatherv(cutoff_histogram,
main_node)
if myid == main_node:
if options.limit_ctf:
# Print out the summary of CTF-limit filtering
print(" ")
print("Global summary of CTF-limit filtering (--limit_ctf) ...")
print("Percentage of filtered micrographs: %8.2f\n" %
(len(cutoff_histogram) * 100.0 /
len(restricted_serial_id_list_not_sliced)))
n_bins = 10
if len(cutoff_histogram) >= n_bins:
from statistics import hist_list
cutoff_region, cutoff_counts = hist_list(
cutoff_histogram, n_bins)
print(" Histogram of cut-off frequency")
print(" cut-off counts")
for bin_id in xrange(n_bins):
print(" %14.7f %7d" %
(cutoff_region[bin_id], cutoff_counts[bin_id]))
else:
print(
"The number of filtered micrographs (%d) is less than the number of bins (%d). No histogram is produced."
% (len(cutoff_histogram), n_bins))
n_mic_process = mpi_reduce(n_mic_process, 1, MPI_INT, MPI_SUM, main_node,
MPI_COMM_WORLD)
n_mic_reject_no_coords = mpi_reduce(n_mic_reject_no_coords, 1, MPI_INT,
MPI_SUM, main_node, MPI_COMM_WORLD)
n_mic_reject_no_cter_entry = mpi_reduce(n_mic_reject_no_cter_entry, 1,
MPI_INT, MPI_SUM, main_node,
MPI_COMM_WORLD)
n_global_coords_detect = mpi_reduce(n_global_coords_detect, 1, MPI_INT,
MPI_SUM, main_node, MPI_COMM_WORLD)
n_global_coords_process = mpi_reduce(n_global_coords_process, 1, MPI_INT,
MPI_SUM, main_node, MPI_COMM_WORLD)
n_global_coords_reject_out_of_boundary = mpi_reduce(
n_global_coords_reject_out_of_boundary, 1, MPI_INT, MPI_SUM, main_node,
MPI_COMM_WORLD)
# Print out the summary of all micrographs
if main_node == myid:
print(" ")
print("Global summary of micrographs ...")
print("Detected : %6d" %
(len(restricted_serial_id_list_not_sliced)))
print("Processed : %6d" % (n_mic_process))
print("Rejected by no coordinates file : %6d" %
(n_mic_reject_no_coords))
print("Rejected by no CTER entry : %6d" %
(n_mic_reject_no_cter_entry))
print(" ")
print("Global summary of coordinates ...")
print("Detected : %6d" %
(n_global_coords_detect))
print("Processed : %6d" %
(n_global_coords_process))
print("Rejected by out of boundary : %6d" %
(n_global_coords_reject_out_of_boundary))
# print(" ")
# print("DONE!!!")
mpi_barrier(MPI_COMM_WORLD)
if main_node == myid:
import time
time.sleep(1)
print("\n Creating bdb:%s/data\n" % original_out_dir)
for proc_i in range(number_of_processes):
mic_start, mic_end = MPI_start_end(
len(restricted_serial_id_list_not_sliced), number_of_processes,
proc_i)
for serial_id in restricted_serial_id_list_not_sliced[
mic_start:mic_end]:
e2bdb_command = "e2bdb.py "
mic_baseroot = mic_baseroot_pattern.replace("*", serial_id)
if RUNNING_UNDER_MPI:
e2bdb_command += "bdb:" + os.path.join(
original_out_dir,
"%03d/" % proc_i) + mic_baseroot + "_ptcls "
else:
e2bdb_command += "bdb:" + os.path.join(
original_out_dir, mic_baseroot + "_ptcls ")
e2bdb_command += " --appendvstack=bdb:%s/data 1>/dev/null" % original_out_dir
cmdexecute(e2bdb_command, printing_on_success=False)
print("Done!\n")
if RUNNING_UNDER_MPI:
mpi_barrier(MPI_COMM_WORLD)
from mpi import mpi_finalize
mpi_finalize()
sys.stdout.flush()
sys.exit(0)
def main(args):
# parser1 = argparse.ArgumentParser(description='This program is used to window particles from a micrograph. The coordinates of the particles are given as input.')
# parser1.add_argument()
progname = os.path.basename(sys.argv[0])
usage = progname + " micrographs_list --coords_dir=coords_dir --coords_suffix=coords_suffix" + \
" --coords_extension=coords_extension --coords_format=coords_format" + \
" --indir=input_dir --importctf=ctf_file --limitctf" + \
" --resample_ratio=resample_ratio --box_size=box_size" + \
" --outdir=outdir --outsuffix=outsuffix --micsuffix=micsuffix" + \
" --nameroot=nameroot --invert" + \
" --defocuserror=defocuserror --astigmatismerror=astigmatismerror"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--coords_dir",
type="string",
default=".",
help=
"<Coordinates Directory> Directory containing files with particle coordinates. (Default: current directory)"
)
parser.add_option(
"--coords_suffix",
type="string",
default="",
help=
"<Coordinates File Suffix> Suffix of coordinate files. For example '_ptcls'. "
)
parser.add_option(
"--coords_extension",
type="string",
default="box",
help=
"<Coordinates File Extension> File extension of coordinate files. e.g 'box' for eman1, 'json' for eman2, ..."
)
parser.add_option(
"--coords_format",
type="string",
default="eman1",
help=
"<Coordinates File Format> Format of coordinates file: 'sparx', 'eman1', 'eman2', or 'spider'. The coordinates of sparx, eman2, and spider format is particle center. The coordinates of eman1 format is particle box conner associated with the original box size."
)
parser.add_option(
"--indir",
type="string",
default=".",
help=
"<Micrograph Directory> Directory containing micrographs to be processed. (Default: current directory)"
)
parser.add_option(
"--nameroot",
type="string",
default="",
help=
"<Micrograph Root Name> Root name (Prefix) of micrographs to be processed."
)
parser.add_option(
"--micsuffix",
type="string",
default="hdf",
help=
"<Micrograph Extension > A string denoting micrograph type. (Default 'hdf')"
)
parser.add_option(
"--outdir",
type="string",
default=".",
help="<Output Directory> Output directory (Default: current directory)"
)
parser.add_option(
"--outsuffix",
type="string",
default="_ptcls",
help="<Output File Suffix> Suffix for output stack. (Default '_ptcls')"
)
parser.add_option(
"--importctf",
type="string",
default="",
help="<CTER CTF File> File name with CTF parameters produced by sxcter."
)
parser.add_option(
"--box_size",
type="int",
default=256,
help=
"<Box Size> x and y dimension in pixels of square area to be windowed. Pixel size after resampling is assumed when resample_ratio < 1.0 (Default 256)"
)
parser.add_option(
"--invert",
action="store_true",
default=False,
help=
"<Invert Contrast> Invert image contrast (recommended for cryo data) (Default, no contrast inversion)"
)
parser.add_option(
"--resample_ratio",
type="float",
default=1.0,
help=
"<Resample Ratio> Ratio of new to old image size (or old to new pixel size) for resampling. Valid range is 0.0 < resample_ratio <= 1.0. (Default: 1.0) (advanced)"
)
parser.add_option(
"--limitctf",
action="store_true",
default=False,
help=
"<Apply CTF-Limit Filter> Filter micrographs based on the CTF limit. It requires --importctf. (Default: no filter) (advanced)"
)
parser.add_option(
"--defocuserror",
type="float",
default=1000000.0,
help=
"<Defocus Error Limit> Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%. (Default: include all irrespective of error values.) (advanced)"
)
parser.add_option(
"--astigmatismerror",
type="float",
default=360.0,
help=
"<Astigmatism Error Limit> Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees. (Default: include all irrespective of error values.) (advanced)"
)
# must be switched off in production
# parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False)
#
# parser.add_option("--restart_section", type="string", default="", help="<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)")
# parser.add_option("--stop_after_candidates", action="store_true", default=False, help="<stop_after_candidates> stops after the 'candidate_class_averages' section")
#
parser.add_option("--return_options",
action="store_true",
dest="return_options",
default=False,
help=SUPPRESS_HELP)
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
# Set local constants
box_size = options.box_size
box_half = box_size // 2
options.micsuffix = "." + options.micsuffix
cterr = [options.defocuserror / 100.0, options.astigmatismerror]
check_options(options, progname)
extension_coord = options.coords_suffix + "." + options.coords_extension
# Build micrograph basename list
micnames = build_micnames(options, args)
print_msg('Detected micrographs : %6d ...\n' % (len(micnames)))
# If there is no micrographs, exit
if len(micnames) == 0:
print usage
sys.exit()
# Load CTFs
n_reject_defocus_error = 0
if options.importctf != None:
ctfs0 = read_text_row(options.importctf)
print_msg('Detected CTF entries : %6d ...\n' % (len(ctfs0)))
ctfs = {}
for i in xrange(len(ctfs0)):
ctf = ctfs0[i]
basemic = baseroot(ctf[-1])
if (ctf[8] / ctf[0] > cterr[0]):
print_msg(
'Defocus error %f exceeds the threshold. Micrograph %s rejected.\n'
% (ctf[8] / ctf[0], basemic))
n_reject_defocus_error += 1
else:
if (ctf[10] > cterr[1]):
ctf[6] = 0.0
ctf[7] = 0.0
ctfs[basemic] = ctf
print_msg('Rejected micrographs by defocus error : %6d ...\n' %
(n_reject_defocus_error))
# Create circular 2D mask for ...
mask = model_circle(box_size // 2, box_size, box_size)
# Prepare loop variables
n_micrographs_process = 0
n_micrographs_reject_no_micrograph = 0
n_micrographs_reject_no_coordinates = 0
n_micrographs_reject_no_cter_entry = 0
n_total_coordinates_detect = 0
n_total_coordinates_process = 0
n_total_coordinates_reject_out_of_boundary = 0
cutoffhistogram = [
] #@ming compute the histogram for micrographs cut of by ctf limit.
# Loop over micrographs
for k in range(len(micnames)):
# basename is name of micrograph minus the path and extension
# Here, assuming micrograph and coordinates have the same file basename
basename = micnames[k]
f_mic = os.path.join(os.path.abspath(options.indir),
basename + options.micsuffix)
f_info = os.path.join(options.coords_dir, basename + extension_coord)
# CHECKS: BEGIN
# IF micrograph exists
if not os.path.exists(f_mic):
print_msg(' Cannot read %s. Skipping %s ...\n' %
(f_mic, basename))
n_micrographs_reject_no_micrograph += 1
continue
# IF coordinates file exists
if not os.path.exists(f_info):
print_msg(' Cannot read %s. Skipping %s ...\n' %
(f_info, basename))
n_micrographs_reject_no_coordinates += 1
continue
# IF micrograph is in CTER results
if options.importctf != None:
if basename not in ctfs:
print_msg(
' Is not listed in CTER results, skipping %s...\n' %
(basename))
n_micrographs_reject_no_cter_entry += 1
continue
else:
ctf = ctfs[basename]
# CHECKS: END
n_micrographs_process += 1
print_msg('\n')
print_msg(
'Processing micrograph %s... Path: %s... Coordinates file %s\n' %
(basename, f_mic, f_info))
# Read coordinates according to the specified format and
# make the coordinates the center of particle image
if options.coords_format.lower() == 'sparx':
coords = read_text_row(f_info)
elif options.coords_format.lower() == 'eman1':
coords = read_text_row(f_info)
for i in range(len(coords)):
coords[i] = [
coords[i][0] + coords[i][2] // 2,
coords[i][1] + coords[i][3] // 2
]
elif options.coords_format.lower() == 'eman2':
coords = js_open_dict(f_info)["boxes"]
for i in range(len(coords)):
coords[i] = [coords[i][0], coords[i][1]]
elif options.coords_format.lower() == 'spider':
coords = read_text_row(f_info)
for i in range(len(coords)):
coords[i] = [coords[i][2], coords[i][3]]
else:
assert (False) # Unreachable code
# Load micrograph from the file
immic = get_im(f_mic)
# Calculate the new pixel size
resample_ratio = options.resample_ratio
if options.importctf != None:
pixel_size_orig = ctf[3]
if resample_ratio < 1.0:
assert (resample_ratio > 0.0)
new_pixel_size = pixel_size_orig / resample_ratio
print_msg(
'Resample micrograph to pixel size %6.4f and window segments from resampled micrograph\n'
% new_pixel_size)
else:
# assert(resample_ratio == 1.0)
new_pixel_size = pixel_size_orig
# Set ctf along with new pixel size in resampled micrograph
ctf[3] = new_pixel_size
else:
assert (options.importctf == None)
if resample_ratio < 1.0:
assert (resample_ratio > 0.0)
print_msg(
'Resample micrograph with ratio %6.4f and window segments from resampled micrograph\n'
% resample_ratio)
# else:
# assert(resample_ratio == 1.0)
# Apply filters to micrograph
fftip(immic)
if options.limitctf:
assert (options.importctf != None)
# Cut off frequency components higher than CTF limit
q1, q2 = ctflimit(box_size, ctf[0], ctf[1], ctf[2], new_pixel_size)
# This is absolute frequency of the CTF limit in the scale of original micrograph
if resample_ratio < 1.0:
assert (resample_ratio > 0.0)
q1 = resample_ratio * q1 / float(
box_size
) # q1 = (pixel_size_orig / new_pixel_size) * q1/float(box_size)
else:
# assert(resample_ratio == 1.0) -> pixel_size_orig == new_pixel_size -> pixel_size_orig / new_pixel_size == 1.0
q1 = q1 / float(box_size)
if q1 < 0.5: #@ming
immic = filt_tanl(immic, q1, 0.01)
cutoffhistogram.append(q1)
# Cut off frequency components lower than the box size can express
immic = fft(filt_gaussh(immic, resample_ratio / box_size))
# Resample micrograph, map coordinates, and window segments from resampled micrograph using new coordinates
# after resampling by resample_ratio, new pixel size will be pixel_size/resample_ratio = new_pixel_size
# NOTE: 2015/04/13 Toshio Moriya
# resample() efficiently takes care of the case resample_ratio = 1.0 but
# it does not set apix_*. Even though it sets apix_* when resample_ratio < 1.0 ...
immic = resample(immic, resample_ratio)
if options.invert:
stt = Util.infomask(immic, None, True)
Util.mul_scalar(immic, -1.0)
immic += 2 * stt[0]
if options.importctf != None:
from utilities import generate_ctf
ctf = generate_ctf(ctf)
# Prepare loop variables
nx = immic.get_xsize()
ny = immic.get_ysize()
x0 = nx // 2
y0 = ny // 2
print_msg('\n')
print_msg('Micrograph size := (%6d, %6d)\n' % (nx, ny))
otcl_images = "bdb:%s/" % options.outdir + basename + options.outsuffix
ind = 0
n_coordinates_reject_out_of_boundary = 0
# Loop over coordinates
for i in range(len(coords)):
source_x = int(coords[i][0])
source_y = int(coords[i][1])
x = source_x
y = source_y
if resample_ratio < 1.0:
assert (resample_ratio > 0.0)
x = int(x * resample_ratio)
y = int(y * resample_ratio)
# else:
# assert(resample_ratio == 1.0)
if ((0 <= x - box_half) and (x + box_half <= nx)
and (0 <= y - box_half) and (y + box_half <= ny)):
imw = Util.window(immic, box_size, box_size, 1, x - x0, y - y0)
else:
print_msg(
'Coordinates ID = %04d (x = %4d, y = %4d, box_size = %4d) is out of micrograph bound, skipping ....\n'
% (i, x, y, box_size))
n_coordinates_reject_out_of_boundary += 1
continue
imw = ramp(imw)
stat = Util.infomask(imw, mask, False)
imw -= stat[0]
imw /= stat[1]
# NOTE: 2015/04/09 Toshio Moriya
# ptcl_source_image might be redundant information ...
# Consider re-organizing header entries...
imw.set_attr("ptcl_source_image", f_mic)
imw.set_attr("ptcl_source_coord_id", i)
imw.set_attr("ptcl_source_coord", [source_x, source_y])
imw.set_attr("resample_ratio", resample_ratio)
# NOTE: 2015/04/13 Toshio Moriya
# apix_* attributes are updated by resample() only when resample_ratio != 1.0
# Let's make sure header info is consistent by setting apix_* = 1.0
# regardless of options, so it is not passed down the processing line
imw.set_attr("apix_x", 1.0)
imw.set_attr("apix_y", 1.0)
imw.set_attr("apix_z", 1.0)
if options.importctf != None:
imw.set_attr("ctf", ctf)
imw.set_attr("ctf_applied", 0)
imw.set_attr("pixel_size_orig", pixel_size_orig)
# imw.set_attr("apix_x", new_pixel_size)
# imw.set_attr("apix_y", new_pixel_size)
# imw.set_attr("apix_z", new_pixel_size)
# NOTE: 2015/04/13 Toshio Moriya
# Pawel Comment: Micrograph is not supposed to have CTF header info.
# So, let's assume it does not exist & ignore its presence.
# Note that resample() "correctly" updates pixel size of CTF header info if it exists
# elif (imw.has_ctff()):
# assert(options.importctf == None)
# ctf_origin = imw.get_attr("ctf")
# pixel_size_origin = round(ctf_origin.apix, 5) # Because SXCTER ouputs up to 5 digits
# imw.set_attr("apix_x",pixel_size_origin)
# imw.set_attr("apix_y",pixel_size_origin)
# imw.set_attr("apix_z",pixel_size_origin)
imw.write_image(otcl_images, ind)
ind += 1
n_total_coordinates_detect += len(coords)
n_total_coordinates_process += ind
n_total_coordinates_reject_out_of_boundary += n_coordinates_reject_out_of_boundary
# Print out the summary of this micrograph
print_msg('\n')
print_msg('Micrograph summary of coordinates...\n')
print_msg('Detected : %4d\n' % (len(coords)))
print_msg('Processed : %4d\n' % (ind))
print_msg('Rejected by out of boundary : %4d\n' %
(n_coordinates_reject_out_of_boundary))
if options.limitctf:
# Print out the summary of CTF-limit filtering
print_msg('\n')
print_msg('Global summary of CTF-limit filtering (--limitctf) ...\n')
print_msg('Percentage of filtered micrographs: %8.2f\n' %
(len(cutoffhistogram) * 100.0 / len(micnames)))
lhist = 10
if len(cutoffhistogram) >= lhist:
from statistics import hist_list
region, hist = hist_list(cutoffhistogram, lhist)
print_msg(" Histogram of cut off frequency\n")
print_msg(" ERROR number of frequencies\n")
for lhx in xrange(lhist):
print_msg(
" %14.7f %7d\n" % (region[lhx], hist[lhx])
) # print_msg(" %10.3f %7d\n" % (region[lhx], hist[lhx]))
else:
print_msg(
"The number of filtered micrographs (%d) is less than the number of bins (%d). No histogram is produced.\n"
% (len(cutoffhistogram), lhist))
# Print out the summary of all micrographs
print_msg('\n')
print_msg('Global summary of micrographs ...\n')
print_msg('Detected : %6d\n' % (len(micnames)))
print_msg('Processed : %6d\n' %
(n_micrographs_process))
print_msg('Rejected by no micrograph file : %6d\n' %
(n_micrographs_reject_no_micrograph))
print_msg('Rejected by no coordinates file : %6d\n' %
(n_micrographs_reject_no_coordinates))
print_msg('Rejected by no CTER entry : %6d\n' %
(n_micrographs_reject_no_cter_entry))
print_msg('\n')
print_msg('Global summary of coordinates ...\n')
print_msg('Detected : %6d\n' %
(n_total_coordinates_detect))
print_msg('Processed : %6d\n' %
(n_total_coordinates_process))
print_msg('Rejected by out of boundary : %6d\n' %
(n_total_coordinates_reject_out_of_boundary))
print_msg('\n')
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
540.0 - psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
360.0 - psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=0.2 --aa=0.1 --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--ave2D",
type="string",
default=False,
help="write to the disk a stack of 2D averages")
parser.add_option("--var2D",
type="string",
default=False,
help="write to the disk a stack of 2D variances")
parser.add_option("--ave3D",
type="string",
default=False,
help="write to the disk reconstructed 3D average")
parser.add_option("--var3D",
type="string",
default=False,
help="compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp",
type="int",
default=10,
help="number of neighbouring projections")
parser.add_option("--no_norm",
action="store_true",
default=False,
help="do not use normalization")
parser.add_option("--radiusvar",
type="int",
default=-1,
help="radius for 3D var")
parser.add_option("--npad",
type="int",
default=2,
help="number of time to pad the original images")
parser.add_option("--sym", type="string", default="c1", help="symmetry")
parser.add_option("--fl",
type="float",
default=0.0,
help="stop-band frequency (Default - no filtration)")
parser.add_option("--aa",
type="float",
default=0.0,
help="fall off of the filter (Default - no filtration)")
parser.add_option("--CTF",
action="store_true",
default=False,
help="use CFT correction")
parser.add_option("--VERBOSE",
action="store_true",
default=False,
help="Long output for debugging")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option(
"--VAR",
action="store_true",
default=False,
help="stack on input consists of 2D variances (Default False)")
parser.add_option(
"--SND",
action="store_true",
default=False,
help="compute squared normalized differences (Default False)")
#parser.add_option("--nvec", type="int" , default=0 , help="number of eigenvectors, default = 0 meaning no PCA calculated")
parser.add_option(
"--symmetrize",
action="store_true",
default=False,
help="Prepare input stack for handling symmetry (Default False)")
(options, args) = parser.parse_args()
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD, MPI_TAG_UB
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
if options.symmetrize:
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if (number_of_proc > 1):
ERROR(
"Cannot use more than one CPU for symmetry prepration",
"sx3dvariability", 1)
except:
pass
except:
pass
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
instack = args[0]
sym = options.sym
if (sym == "c1"):
ERROR("Thre is no need to symmetrize stack for C1 symmetry",
"sx3dvariability", 1)
if (instack[:4] != "bdb:"):
stack = "bdb:data"
delete_bdb(stack)
cmdexecute("sxcpy.py " + instack + " " + stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack, 'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None] * na
for k in xrange(ks):
delete_bdb("bdb:Q%1d" % k)
cmdexecute("e2bdb.py " + stack + " --makevstack=bdb:Q%1d" % k)
DB = db_open_dict("bdb:Q%1d" % k)
for i in xrange(na):
ut = qt[i] * ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
delete_bdb("bdb:sdata")
cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
#cmdexecute("ls EMAN2DB/sdata*")
a = get_im("bdb:sdata")
a.set_attr("variabilitysymmetry", sym)
a.write_image("bdb:sdata")
else:
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 1:
stack = args[0]
else:
print("usage: " + usage)
print("Please run '" + progname + " -h' for detailed options")
return 1
t0 = time()
# obsolete flags
options.MPI = True
options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter",
"sx3dvariability", 1, myid)
if options.VAR and options.SND:
ERROR("Only one of var and SND can be set!", "sx3dvariability",
myid)
exit()
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR(
"When VAR is set, the program cannot output ave2D, ave3D or var2D",
"sx3dvariability", 1, myid)
exit()
if options.SND and (options.ave2D or options.ave3D):
ERROR("When SND is set, the program cannot output ave2D or ave3D",
"sx3dvariability", 1, myid)
exit()
if options.nvec > 0:
ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
exit()
#if options.nvec > 0 and options.ave3D == None:
# ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid)
# exit()
import string
options.sym = options.sym.lower()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
if myid == main_node:
print_begin_msg("sx3dvariability")
print_msg("%-70s: %s\n" % ("Input stack", stack))
img_per_grp = options.img_per_grp
nvec = options.nvec
radiuspca = options.radiuspca
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
if options.sym != "c1":
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry")
if (i != options.sym):
ERROR(
"The symmetry provided does not agree with the symmetry of the input stack",
"sx3dvariability",
myid=myid)
except:
ERROR(
"Input stack is not prepared for symmetry, please follow instructions",
"sx3dvariability",
myid=myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if ((nima / i) * i != nima):
ERROR(
"The length of the input stack is incorrect for symmetry processing",
"sx3dvariability",
myid=myid)
symbaselen = nima / i
else:
symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
symbaselen = bcast_number_to_all(symbaselen)
if radiuspca == -1: radiuspca = nx / 2 - 2
if myid == main_node:
print_msg("%-70s: %d\n" % ("Number of projection", nima))
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid,
imgdata,
1.0,
symmetry=options.sym,
npad=options.npad,
xysize=-1,
zsize=-1)
else:
vol = recons3d_4nn_MPI(myid,
imgdata,
symmetry=options.sym,
npad=options.npad,
xysize=-1,
zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx / 2 - 2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i -
img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i - img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i - img_begin], mask)
diff2 = diff * diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
elif options.VAR:
varList = EMData.read_images(stack, range(img_begin, img_end))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in xrange(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x * 10000 + psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
print_msg("%-70s: %d\n" %
("Number of neighboring projections", img_per_grp))
print_msg("...... Finding neighboring projections\n")
if options.VERBOSE:
print "Number of images per group: ", img_per_grp
print "Now grouping projections"
proj_angles.sort()
proj_angles_list = [0.0] * (nima * 4)
if myid == main_node:
for i in xrange(nima):
proj_angles_list[i * 4] = proj_angles[i][1]
proj_angles_list[i * 4 + 1] = proj_angles[i][2]
proj_angles_list[i * 4 + 2] = proj_angles[i][3]
proj_angles_list[i * 4 + 3] = proj_angles[i][4]
proj_angles_list = bcast_list_to_all(proj_angles_list, myid,
main_node)
proj_angles = []
for i in xrange(nima):
proj_angles.append([
proj_angles_list[i * 4], proj_angles_list[i * 4 + 1],
proj_angles_list[i * 4 + 2],
int(proj_angles_list[i * 4 + 3])
])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp,
range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
if options.VERBOSE:
print "On node %2d, number of images needed to be read = %5d" % (
myid, len(all_proj))
index = {}
for i in xrange(len(all_proj)):
index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("%-70s: %.2f\n" %
("Finding neighboring projections lasted [s]",
time() - t2))
print_msg("%-70s: %d\n" %
("Number of groups processed on the main node",
len(proj_list)))
if options.VERBOSE:
print "Grouping projections took: ", (time() -
t2) / 60, "[min]"
print "Number of groups on main node: ", len(proj_list)
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("...... calculating the stack of 2D variances \n")
if options.VERBOSE:
print "Now calculating the stack of 2D variances"
proj_params = [0.0] * (nima * 5)
aveList = []
varList = []
if nvec > 0:
eigList = [[] for i in xrange(nvec)]
if options.VERBOSE:
print "Begin to read images on processor %d" % (myid)
ttt = time()
imgdata = EMData.read_images(stack, all_proj)
if options.VERBOSE:
print "Reading images on processor %d done, time = %.2f" % (
myid, time() - ttt)
print "On processor %d, we got %d images" % (myid,
len(imgdata))
mpi_barrier(MPI_COMM_WORLD)
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
for i in xrange(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])
grp_imgdata = []
for j in xrange(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
alpha, sx, sy, mirror = params_3D_2D_NEW(
phi, theta, psi, s2x, s2y, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, phiM - phi, 0.0, 0.0, 1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, 180 - (phiM - phi), 0.0,
0.0, 1.0)
else:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(phiM - phi), 0.0, 0.0,
1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(180 - (phiM - phi)), 0.0,
0.0, 1.0)
set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
grp_imgdata.append(imgdata[mj])
if not options.no_norm:
mask = model_circle(nx / 2 - 2, nx, nx)
for k in xrange(img_per_grp):
ave, std, minn, maxx = Util.infomask(
grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
del mask
if options.fl > 0.0:
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_tanl(
filt_ctf(
fft(
pad(grp_imgdata[k], nx2, ny2,
1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1), options.fl,
options.aa)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
for k in xrange(img_per_grp):
grp_imgdata[k] = filt_tanl(grp_imgdata[k],
options.fl, options.aa)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
from utilities import pad, read_text_file
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_ctf(fft(
pad(grp_imgdata[k], nx2, ny2, 1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
'''
if i < 10 and myid == main_node:
for k in xrange(10):
grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
'''
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("pp.hdf", pp)
"""
ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("qq.hdf", pp)
"""
var = model_blank(nx, ny)
for q in grp_imgdata:
Util.add_img2(var, q)
Util.mul_scalar(var, 1.0 / (len(grp_imgdata) - 1))
# Switch to std dev
var = square_root(threshold(var))
#if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a")
#else: ave, var = avgvar(grp_imgdata, mode="a")
"""
if myid == main_node:
ave.write_image("avgv.hdf",i)
var.write_image("varv.hdf",i)
"""
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
if options.VERBOSE:
print "%5.2f%% done on processor %d" % (
i * 100.0 / len(proj_list), myid)
if nvec > 0:
eig = pca(input_stacks=grp_imgdata,
subavg="",
mask_radius=radiuspca,
nvec=nvec,
incore=True,
shuffle=False,
genbuf=True)
for k in xrange(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
del imgdata
# To this point, all averages, variances, and eigenvectors are computed
if options.ave2D:
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(aveList)):
aveList[im].write_image(options.ave2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
"""
nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
ave.write_image(options.ave2D, km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB,
MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
"""
if options.ave3D:
if options.VERBOSE:
print "Reconstructing 3D average volume"
ave3D = recons3d_4nn_MPI(myid,
aveList,
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
ave3D.write_image(options.ave3D)
print_msg("%-70s: %s\n" % (
"Writing to the disk volume reconstructed from averages as",
options.ave3D))
del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList
if nvec > 0:
for k in xrange(nvec):
if options.VERBOSE:
print "Reconstruction eigenvolumes", k
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid,
eigList[k],
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image("eig3d_%03d.hdf" % k, ITER)
Util.mul_img(eig3D,
model_circle(radiuspca, nx, nx, nx))
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in xrange(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(
eigList[k][l])
proj = prgs(eig3Df, kb,
[phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node,
MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM,
main_node, MPI_COMM_WORLD)
if myid == main_node:
u = int(u[0])
print " Eigenvector: ", k, " number changed ", int(
icont[0])
else:
u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
if options.ave3D: del ave3D
if options.var2D:
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(varList)):
varList[im].write_image(options.var2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
ave.write_image(options.var2D, km)
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node, MPI_TAG_UB,
MPI_COMM_WORLD)
for im in xrange(len(varList)):
send_EMData(varList[im], main_node, im + myid +
70000) # What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node and options.VERBOSE:
print "Reconstructing 3D variability volume"
t6 = time()
radiusvar = options.radiusvar
if (radiusvar < 0): radiusvar = nx // 2 - 3
res = recons3d_4nn_MPI(myid,
varList,
symmetry=options.sym,
npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
res.write_image(options.var3D)
if myid == main_node:
print_msg(
"%-70s: %.2f\n" %
("Reconstructing 3D variability took [s]", time() - t6))
if options.VERBOSE:
print "Reconstruction took: %.2f [min]" % (
(time() - t6) / 60)
if myid == main_node:
print_msg(
"%-70s: %.2f\n" %
("Total time for these computations [s]", time() - t0))
if options.VERBOSE:
print "Total time for these computations: %.2f [min]" % (
(time() - t0) / 60)
print_end_msg("sx3dvariability")
global_def.BATCH = False
from mpi import mpi_finalize
mpi_finalize()
def main(args):
# parser1 = argparse.ArgumentParser(description='This program is used to window particles from a micrograph. The coordinates of the particles are given as input.')
# parser1.add_argument()
progname = os.path.basename(sys.argv[0])
usage = progname + " micrographs_list --coords_dir=coords_dir --coords_suffix=coords_suffix" + \
" --coords_extension=coords_extension --coords_format=coords_format" + \
" --indir=input_dir --importctf=ctf_file --limitctf" + \
" --resample_ratio=resample_ratio --box_size=box_size" + \
" --outdir=outdir --outsuffix=outsuffix --micsuffix=micsuffix" + \
" --nameroot=nameroot --invert" + \
" --defocuserror=defocuserror --astigmatismerror=astigmatismerror"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--coords_dir", type="string", default=".", help="<Coordinates Directory> Directory containing files with particle coordinates. (Default: current directory)")
parser.add_option("--coords_suffix", type="string", default="", help="<Coordinates File Suffix> Suffix of coordinate files. For example '_ptcls'. ")
parser.add_option("--coords_extension", type="string", default="box", help="<Coordinates File Extension> File extension of coordinate files. e.g 'box' for eman1, 'json' for eman2, ...")
parser.add_option("--coords_format", type="string", default="eman1", help="<Coordinates File Format> Format of coordinates file: 'sparx', 'eman1', 'eman2', or 'spider'. The coordinates of sparx, eman2, and spider format is particle center. The coordinates of eman1 format is particle box conner associated with the original box size.")
parser.add_option("--indir", type="string", default=".", help="<Micrograph Directory> Directory containing micrographs to be processed. (Default: current directory)")
parser.add_option("--nameroot", type="string", default="", help="<Micrograph Root Name> Root name (Prefix) of micrographs to be processed.")
parser.add_option("--micsuffix", type="string", default="hdf", help="<Micrograph Extension > A string denoting micrograph type. (Default 'hdf')")
parser.add_option("--outdir", type="string", default=".", help="<Output Directory> Output directory (Default: current directory)")
parser.add_option("--outsuffix", type="string", default="_ptcls", help="<Output File Suffix> Suffix for output stack. (Default '_ptcls')")
parser.add_option("--importctf", type="string", default="", help="<CTER CTF File> File name with CTF parameters produced by sxcter.")
parser.add_option("--box_size", type="int", default=256, help="<Box Size> x and y dimension in pixels of square area to be windowed. Pixel size after resampling is assumed when resample_ratio < 1.0 (Default 256)")
parser.add_option("--invert", action="store_true", default=False, help="<Invert Contrast> Invert image contrast (recommended for cryo data) (Default, no contrast inversion)")
parser.add_option("--resample_ratio", type="float", default=1.0, help="<Resample Ratio> Ratio of new to old image size (or old to new pixel size) for resampling. Valid range is 0.0 < resample_ratio <= 1.0. (Default: 1.0) (advanced)")
parser.add_option("--limitctf", action="store_true", default=False, help="<Apply CTF-Limit Filter> Filter micrographs based on the CTF limit. It requires --importctf. (Default: no filter) (advanced)")
parser.add_option("--defocuserror", type="float", default=1000000.0, help="<Defocus Error Limit> Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%. (Default: include all irrespective of error values.) (advanced)" )
parser.add_option("--astigmatismerror", type="float", default=360.0, help="<Astigmatism Error Limit> Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees. (Default: include all irrespective of error values.) (advanced)")
# must be switched off in production
# parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False)
#
# parser.add_option("--restart_section", type="string", default="", help="<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)")
# parser.add_option("--stop_after_candidates", action="store_true", default=False, help="<stop_after_candidates> stops after the 'candidate_class_averages' section")
#
parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP)
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
# Set local constants
box_size = options.box_size
box_half = box_size // 2
options.micsuffix = "." + options.micsuffix
cterr = [options.defocuserror/100.0, options.astigmatismerror]
check_options(options, progname)
extension_coord = options.coords_suffix + "." + options.coords_extension
# Build micrograph basename list
micnames = build_micnames(options, args)
print_msg('Detected micrographs : %6d ...\n' % (len(micnames)))
# If there is no micrographs, exit
if len(micnames) == 0:
print usage
sys.exit()
# Load CTFs
n_reject_defocus_error = 0
if options.importctf != None:
ctfs0 = read_text_row(options.importctf)
print_msg('Detected CTF entries : %6d ...\n' % (len(ctfs0)))
ctfs={}
for i in xrange(len(ctfs0)):
ctf=ctfs0[i]
basemic = baseroot(ctf[-1])
if(ctf[8]/ctf[0] > cterr[0]):
print_msg('Defocus error %f exceeds the threshold. Micrograph %s rejected.\n' % (ctf[8]/ctf[0], basemic))
n_reject_defocus_error += 1
else:
if(ctf[10] > cterr[1] ):
ctf[6] = 0.0
ctf[7] = 0.0
ctfs[basemic] = ctf
print_msg('Rejected micrographs by defocus error : %6d ...\n' % (n_reject_defocus_error))
# Create circular 2D mask for ...
mask = model_circle(box_size//2, box_size, box_size)
# Prepare loop variables
n_micrographs_process = 0
n_micrographs_reject_no_micrograph = 0
n_micrographs_reject_no_coordinates = 0
n_micrographs_reject_no_cter_entry = 0
n_total_coordinates_detect = 0
n_total_coordinates_process = 0
n_total_coordinates_reject_out_of_boundary = 0
cutoffhistogram = [] #@ming compute the histogram for micrographs cut of by ctf limit.
# Loop over micrographs
for k in range(len(micnames)):
# basename is name of micrograph minus the path and extension
# Here, assuming micrograph and coordinates have the same file basename
basename = micnames[k]
f_mic = os.path.join(os.path.abspath(options.indir), basename + options.micsuffix)
f_info = os.path.join(options.coords_dir, basename + extension_coord)
# CHECKS: BEGIN
# IF micrograph exists
if not os.path.exists(f_mic):
print_msg(' Cannot read %s. Skipping %s ...\n' % (f_mic, basename))
n_micrographs_reject_no_micrograph += 1
continue
# IF coordinates file exists
if not os.path.exists(f_info):
print_msg(' Cannot read %s. Skipping %s ...\n' % (f_info, basename))
n_micrographs_reject_no_coordinates += 1
continue
# IF micrograph is in CTER results
if options.importctf != None:
if basename not in ctfs:
print_msg(' Is not listed in CTER results, skipping %s...\n' % (basename))
n_micrographs_reject_no_cter_entry += 1
continue
else:
ctf = ctfs[basename]
# CHECKS: END
n_micrographs_process += 1
print_msg('\n')
print_msg('Processing micrograph %s... Path: %s... Coordinates file %s\n' % (basename, f_mic, f_info))
# Read coordinates according to the specified format and
# make the coordinates the center of particle image
if options.coords_format.lower() == 'sparx' :
coords = read_text_row(f_info)
elif options.coords_format.lower() == 'eman1':
coords = read_text_row(f_info)
for i in range(len(coords)):
coords[i] = [coords[i][0] + coords[i][2]//2 ,coords[i][1] + coords[i][3]//2]
elif options.coords_format.lower() == 'eman2':
coords = js_open_dict(f_info)["boxes"]
for i in range(len(coords)):
coords[i] = [coords[i][0],coords[i][1]]
elif options.coords_format.lower() == 'spider':
coords = read_text_row(f_info)
for i in range(len(coords)):
coords[i] = [coords[i][2] ,coords[i][3]]
else:
assert(False) # Unreachable code
# Load micrograph from the file
immic = get_im(f_mic)
# Calculate the new pixel size
resample_ratio = options.resample_ratio
if options.importctf != None:
pixel_size_orig = ctf[3]
if resample_ratio < 1.0:
assert(resample_ratio > 0.0)
new_pixel_size = pixel_size_orig / resample_ratio
print_msg('Resample micrograph to pixel size %6.4f and window segments from resampled micrograph\n' % new_pixel_size)
else:
# assert(resample_ratio == 1.0)
new_pixel_size = pixel_size_orig
# Set ctf along with new pixel size in resampled micrograph
ctf[3] = new_pixel_size
else:
assert(options.importctf == None)
if resample_ratio < 1.0:
assert(resample_ratio > 0.0)
print_msg('Resample micrograph with ratio %6.4f and window segments from resampled micrograph\n' % resample_ratio)
# else:
# assert(resample_ratio == 1.0)
# Apply filters to micrograph
fftip(immic)
if options.limitctf:
assert(options.importctf != None)
# Cut off frequency components higher than CTF limit
q1, q2 = ctflimit(box_size,ctf[0],ctf[1],ctf[2],new_pixel_size)
# This is absolute frequency of the CTF limit in the scale of original micrograph
if resample_ratio < 1.0:
assert(resample_ratio > 0.0)
q1 = resample_ratio * q1 / float(box_size) # q1 = (pixel_size_orig / new_pixel_size) * q1/float(box_size)
else:
# assert(resample_ratio == 1.0) -> pixel_size_orig == new_pixel_size -> pixel_size_orig / new_pixel_size == 1.0
q1 = q1 / float(box_size)
if q1 < 0.5: #@ming
immic = filt_tanl(immic, q1, 0.01)
cutoffhistogram.append(q1)
# Cut off frequency components lower than the box size can express
immic = fft(filt_gaussh( immic, resample_ratio/box_size ))
# Resample micrograph, map coordinates, and window segments from resampled micrograph using new coordinates
# after resampling by resample_ratio, new pixel size will be pixel_size/resample_ratio = new_pixel_size
# NOTE: 2015/04/13 Toshio Moriya
# resample() efficiently takes care of the case resample_ratio = 1.0 but
# it does not set apix_*. Even though it sets apix_* when resample_ratio < 1.0 ...
immic = resample(immic, resample_ratio)
if options.invert:
stt = Util.infomask(immic, None, True)
Util.mul_scalar(immic, -1.0)
immic += 2*stt[0]
if options.importctf != None:
from utilities import generate_ctf
ctf = generate_ctf(ctf)
# Prepare loop variables
nx = immic.get_xsize()
ny = immic.get_ysize()
x0 = nx//2
y0 = ny//2
print_msg('\n')
print_msg('Micrograph size := (%6d, %6d)\n' % (nx, ny))
otcl_images = "bdb:%s/" % options.outdir + basename + options.outsuffix
ind = 0
n_coordinates_reject_out_of_boundary = 0
# Loop over coordinates
for i in range(len(coords)):
source_x = int(coords[i][0])
source_y = int(coords[i][1])
x = source_x
y = source_y
if resample_ratio < 1.0:
assert(resample_ratio > 0.0)
x = int(x * resample_ratio)
y = int(y * resample_ratio)
# else:
# assert(resample_ratio == 1.0)
if( (0 <= x - box_half) and ( x + box_half <= nx ) and (0 <= y - box_half) and ( y + box_half <= ny ) ):
imw = Util.window(immic, box_size, box_size, 1, x-x0, y-y0)
else:
print_msg('Coordinates ID = %04d (x = %4d, y = %4d, box_size = %4d) is out of micrograph bound, skipping ....\n' % (i, x, y, box_size))
n_coordinates_reject_out_of_boundary += 1
continue
imw = ramp(imw)
stat = Util.infomask( imw, mask, False )
imw -= stat[0]
imw /= stat[1]
# NOTE: 2015/04/09 Toshio Moriya
# ptcl_source_image might be redundant information ...
# Consider re-organizing header entries...
imw.set_attr("ptcl_source_image", f_mic)
imw.set_attr("ptcl_source_coord_id", i)
imw.set_attr("ptcl_source_coord", [source_x, source_y])
imw.set_attr("resample_ratio", resample_ratio)
# NOTE: 2015/04/13 Toshio Moriya
# apix_* attributes are updated by resample() only when resample_ratio != 1.0
# Let's make sure header info is consistent by setting apix_* = 1.0
# regardless of options, so it is not passed down the processing line
imw.set_attr("apix_x", 1.0)
imw.set_attr("apix_y", 1.0)
imw.set_attr("apix_z", 1.0)
if options.importctf != None:
imw.set_attr("ctf",ctf)
imw.set_attr("ctf_applied", 0)
imw.set_attr("pixel_size_orig", pixel_size_orig)
# imw.set_attr("apix_x", new_pixel_size)
# imw.set_attr("apix_y", new_pixel_size)
# imw.set_attr("apix_z", new_pixel_size)
# NOTE: 2015/04/13 Toshio Moriya
# Pawel Comment: Micrograph is not supposed to have CTF header info.
# So, let's assume it does not exist & ignore its presence.
# Note that resample() "correctly" updates pixel size of CTF header info if it exists
# elif (imw.has_ctff()):
# assert(options.importctf == None)
# ctf_origin = imw.get_attr("ctf")
# pixel_size_origin = round(ctf_origin.apix, 5) # Because SXCTER ouputs up to 5 digits
# imw.set_attr("apix_x",pixel_size_origin)
# imw.set_attr("apix_y",pixel_size_origin)
# imw.set_attr("apix_z",pixel_size_origin)
imw.write_image(otcl_images, ind)
ind += 1
n_total_coordinates_detect += len(coords)
n_total_coordinates_process += ind
n_total_coordinates_reject_out_of_boundary += n_coordinates_reject_out_of_boundary
# Print out the summary of this micrograph
print_msg('\n')
print_msg('Micrograph summary of coordinates...\n')
print_msg('Detected : %4d\n' % (len(coords)))
print_msg('Processed : %4d\n' % (ind))
print_msg('Rejected by out of boundary : %4d\n' % (n_coordinates_reject_out_of_boundary))
if options.limitctf:
# Print out the summary of CTF-limit filtering
print_msg('\n')
print_msg('Global summary of CTF-limit filtering (--limitctf) ...\n')
print_msg('Percentage of filtered micrographs: %8.2f\n' % (len(cutoffhistogram) * 100.0 / len(micnames)))
lhist = 10
if len(cutoffhistogram) >= lhist:
from statistics import hist_list
region,hist = hist_list(cutoffhistogram, lhist)
print_msg(" Histogram of cut off frequency\n")
print_msg(" ERROR number of frequencies\n")
for lhx in xrange(lhist):
print_msg(" %14.7f %7d\n" % (region[lhx], hist[lhx])) # print_msg(" %10.3f %7d\n" % (region[lhx], hist[lhx]))
else:
print_msg("The number of filtered micrographs (%d) is less than the number of bins (%d). No histogram is produced.\n" % (len(cutoffhistogram), lhist))
# Print out the summary of all micrographs
print_msg('\n')
print_msg('Global summary of micrographs ...\n')
print_msg('Detected : %6d\n' % (len(micnames)))
print_msg('Processed : %6d\n' % (n_micrographs_process))
print_msg('Rejected by no micrograph file : %6d\n' % (n_micrographs_reject_no_micrograph))
print_msg('Rejected by no coordinates file : %6d\n' % (n_micrographs_reject_no_coordinates))
print_msg('Rejected by no CTER entry : %6d\n' % (n_micrographs_reject_no_cter_entry))
print_msg('\n')
print_msg('Global summary of coordinates ...\n')
print_msg('Detected : %6d\n' % (n_total_coordinates_detect))
print_msg('Processed : %6d\n' % (n_total_coordinates_process))
print_msg('Rejected by out of boundary : %6d\n' % (n_total_coordinates_reject_out_of_boundary))
print_msg('\n')
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ input_micrograph_list_file input_micrograph_pattern input_coordinates_pattern output_directory --coordinates_format --box_size=box_size --invert --import_ctf=ctf_file --limit_ctf --resample_ratio=resample_ratio --defocus_error=defocus_error --astigmatism_error=astigmatism_error
Window particles from micrographs in input list file. The coordinates of the particles should be given as input.
Please specify name pattern of input micrographs and coordinates files with a wild card (*). Use the wild card to indicate the place of micrograph ID (e.g. serial number, time stamp, and etc).
The name patterns must be enclosed by single quotes (') or double quotes ("). (Note: sxgui.py automatically adds single quotes (')).
BDB files can not be selected as input micrographs.
sxwindow.py mic_list.txt ./mic*.hdf info/mic*_info.json particles --coordinates_format=eman2 --box_size=64 --invert --import_ctf=outdir_cter/partres/partres.txt
If micrograph list file name is not provided, all files matched with the micrograph name pattern will be processed.
sxwindow.py ./mic*.hdf info/mic*_info.json particles --coordinates_format=eman2 --box_size=64 --invert --import_ctf=outdir_cter/partres/partres.txt
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--coordinates_format", type="string", default="eman1", help="format of input coordinates files: 'sparx', 'eman1', 'eman2', or 'spider'. the coordinates of sparx, eman2, and spider format is particle center. the coordinates of eman1 format is particle box conner associated with the original box size. (default eman1)")
parser.add_option("--box_size", type="int", default=256, help="x and y dimension of square area to be windowed (in pixels): pixel size after resampling is assumed when resample_ratio < 1.0 (default 256)")
parser.add_option("--invert", action="store_true", default=False, help="invert image contrast: recommended for cryo data (default False)")
parser.add_option("--import_ctf", type="string", default="", help="file name of sxcter output: normally partres.txt (default none)")
parser.add_option("--limit_ctf", action="store_true", default=False, help="filter micrographs based on the CTF limit: this option requires --import_ctf. (default False)")
parser.add_option("--resample_ratio", type="float", default=1.0, help="ratio of new to old image size (or old to new pixel size) for resampling: Valid range is 0.0 < resample_ratio <= 1.0. (default 1.0)")
parser.add_option("--defocus_error", type="float", default=1000000.0, help="defocus errror limit: exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocus_error percent. the error is computed as (std dev defocus)/defocus*100%. (default 1000000.0)" )
parser.add_option("--astigmatism_error", type="float", default=360.0, help="astigmatism error limit: Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatism_error degrees. (default 360.0)")
### detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
main_node = 0
if RUNNING_UNDER_MPI:
from mpi import mpi_init
from mpi import MPI_COMM_WORLD, mpi_comm_rank, mpi_comm_size, mpi_barrier, mpi_reduce, MPI_INT, MPI_SUM
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_processes = mpi_comm_size(MPI_COMM_WORLD)
else:
number_of_processes = 1
myid = 0
(options, args) = parser.parse_args(sys.argv[1:])
mic_list_file_path = None
mic_pattern = None
coords_pattern = None
error_status = None
while True:
if len(args) < 3 or len(args) > 4:
error_status = ("Please check usage for number of arguments.\n Usage: " + usage + "\n" + "Please run %s -h for help." % (progname), getframeinfo(currentframe()))
break
if len(args) == 3:
mic_pattern = args[0]
coords_pattern = args[1]
out_dir = args[2]
else: # assert(len(args) == 4)
mic_list_file_path = args[0]
mic_pattern = args[1]
coords_pattern = args[2]
out_dir = args[3]
if mic_list_file_path != None:
if os.path.splitext(mic_list_file_path)[1] != ".txt":
error_status = ("Extension of input micrograph list file must be \".txt\". Please check input_micrograph_list_file argument. Run %s -h for help." % (progname), getframeinfo(currentframe()))
break
if mic_pattern[:len("bdb:")].lower() == "bdb":
error_status = ("BDB file can not be selected as input micrographs. Please convert the format, and restart the program. Run %s -h for help." % (progname), getframeinfo(currentframe()))
break
if mic_pattern.find("*") == -1:
error_status = ("Input micrograph file name pattern must contain wild card (*). Please check input_micrograph_pattern argument. Run %s -h for help." % (progname), getframeinfo(currentframe()))
break
if coords_pattern.find("*") == -1:
error_status = ("Input coordinates file name pattern must contain wild card (*). Please check input_coordinates_pattern argument. Run %s -h for help." % (progname), getframeinfo(currentframe()))
break
if myid == main_node:
if os.path.exists(out_dir):
error_status = ("Output directory exists. Please change the name and restart the program.", getframeinfo(currentframe()))
break
break
if_error_then_all_processes_exit_program(error_status)
# Check invalid conditions of options
check_options(options, progname)
mic_name_list = None
error_status = None
if myid == main_node:
if mic_list_file_path != None:
print("Loading micrograph list from %s file ..." % (mic_list_file_path))
mic_name_list = read_text_file(mic_list_file_path)
if len(mic_name_list) == 0:
print("Directory of first micrograph entry is " % (os.path.dirname(mic_name_list[0])))
else: # assert (mic_list_file_path == None)
print("Generating micrograph list in %s directory..." % (os.path.dirname(mic_pattern)))
mic_name_list = glob.glob(mic_pattern)
if len(mic_name_list) == 0:
error_status = ("No micrograph file is found. Please check input_micrograph_pattern and/or input_micrograph_list_file argument. Run %s -h for help." % (progname), getframeinfo(currentframe()))
else:
print("Found %d microgarphs" % len(mic_name_list))
if_error_then_all_processes_exit_program(error_status)
if RUNNING_UNDER_MPI:
mic_name_list = wrap_mpi_bcast(mic_name_list, main_node)
coords_name_list = None
error_status = None
if myid == main_node:
coords_name_list = glob.glob(coords_pattern)
if len(coords_name_list) == 0:
error_status = ("No coordinates file is found. Please check input_coordinates_pattern argument. Run %s -h for help." % (progname), getframeinfo(currentframe()))
if_error_then_all_processes_exit_program(error_status)
if RUNNING_UNDER_MPI:
coords_name_list = wrap_mpi_bcast(coords_name_list, main_node)
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
# all processes must have access to indices
if options.import_ctf:
i_enum = -1
i_enum += 1; idx_cter_def = i_enum # defocus [um]; index must be same as ctf object format
i_enum += 1; idx_cter_cs = i_enum # Cs [mm]; index must be same as ctf object format
i_enum += 1; idx_cter_vol = i_enum # voltage[kV]; index must be same as ctf object format
i_enum += 1; idx_cter_apix = i_enum # pixel size [A]; index must be same as ctf object format
i_enum += 1; idx_cter_bfactor = i_enum # B-factor [A^2]; index must be same as ctf object format
i_enum += 1; idx_cter_ac = i_enum # amplitude contrast [%]; index must be same as ctf object format
i_enum += 1; idx_cter_astig_amp = i_enum # astigmatism amplitude [um]; index must be same as ctf object format
i_enum += 1; idx_cter_astig_ang = i_enum # astigmatism angle [degree]; index must be same as ctf object format
i_enum += 1; idx_cter_sd_def = i_enum # std dev of defocus [um]
i_enum += 1; idx_cter_sd_astig_amp = i_enum # std dev of ast amp [A]
i_enum += 1; idx_cter_sd_astig_ang = i_enum # std dev of ast angle [degree]
i_enum += 1; idx_cter_cv_def = i_enum # coefficient of variation of defocus [%]
i_enum += 1; idx_cter_cv_astig_amp = i_enum # coefficient of variation of ast amp [%]
i_enum += 1; idx_cter_spectra_diff = i_enum # average of differences between with- and without-astig. experimental 1D spectra at extrema
i_enum += 1; idx_cter_error_def = i_enum # frequency at which signal drops by 50% due to estimated error of defocus alone [1/A]
i_enum += 1; idx_cter_error_astig = i_enum # frequency at which signal drops by 50% due to estimated error of defocus and astigmatism [1/A]
i_enum += 1; idx_cter_error_ctf = i_enum # limit frequency by CTF error [1/A]
i_enum += 1; idx_cter_mic_name = i_enum # micrograph name
i_enum += 1; n_idx_cter = i_enum
# Prepare loop variables
mic_basename_pattern = os.path.basename(mic_pattern) # file pattern without path
mic_baseroot_pattern = os.path.splitext(mic_basename_pattern)[0] # file pattern without path and extension
coords_format = options.coordinates_format.lower()
box_size = options.box_size
box_half = box_size // 2
mask2d = model_circle(box_size//2, box_size, box_size) # Create circular 2D mask to Util.infomask of particle images
resample_ratio = options.resample_ratio
n_mic_process = 0
n_mic_reject_no_coords = 0
n_mic_reject_no_cter_entry = 0
n_global_coords_detect = 0
n_global_coords_process = 0
n_global_coords_reject_out_of_boundary = 0
serial_id_list = []
error_status = None
## not a real while, an if with the opportunity to use break when errors need to be reported
while myid == main_node:
#
# NOTE: 2016/05/24 Toshio Moriya
# Now, ignores the path in mic_pattern and entries of mic_name_list to create serial ID
# Only the basename (file name) in micrograph path must be match
#
# Create list of micrograph serial ID
# Break micrograph name pattern into prefix and suffix to find the head index of the micrograph serial id
#
mic_basename_tokens = mic_basename_pattern.split('*')
# assert (len(mic_basename_tokens) == 2)
serial_id_head_index = len(mic_basename_tokens[0])
# Loop through micrograph names
for mic_name in mic_name_list:
# Find the tail index of the serial id and extract serial id from the micrograph name
mic_basename = os.path.basename(mic_name)
serial_id_tail_index = mic_basename.index(mic_basename_tokens[1])
serial_id = mic_basename[serial_id_head_index:serial_id_tail_index]
serial_id_list.append(serial_id)
# assert (len(serial_id_list) == len(mic_name))
del mic_name_list # Do not need this anymore
# Load CTFs if necessary
if options.import_ctf:
ctf_list = read_text_row(options.import_ctf)
# print("Detected CTF entries : %6d ..." % (len(ctf_list)))
if len(ctf_list) == 0:
error_status = ("No CTF entry is found in %s. Please check --import_ctf option. Run %s -h for help." % (options.import_ctf, progname), getframeinfo(currentframe()))
break
if (len(ctf_list[0]) != n_idx_cter):
error_status = ("Number of columns (%d) must be %d in %s. The format might be old. Please run sxcter.py again." % (len(ctf_list[0]), n_idx_cter, options.import_ctf), getframeinfo(currentframe()))
break
ctf_dict={}
n_reject_defocus_error = 0
ctf_error_limit = [options.defocus_error/100.0, options.astigmatism_error]
for ctf_params in ctf_list:
assert(len(ctf_params) == n_idx_cter)
# mic_baseroot is name of micrograph minus the path and extension
mic_baseroot = os.path.splitext(os.path.basename(ctf_params[idx_cter_mic_name]))[0]
if(ctf_params[idx_cter_sd_def] / ctf_params[idx_cter_def] > ctf_error_limit[0]):
print("Defocus error %f exceeds the threshold. Micrograph %s is rejected." % (ctf_params[idx_cter_sd_def] / ctf_params[idx_cter_def], mic_baseroot))
n_reject_defocus_error += 1
else:
if(ctf_params[idx_cter_sd_astig_ang] > ctf_error_limit[1]):
ctf_params[idx_cter_astig_amp] = 0.0
ctf_params[idx_cter_astig_ang] = 0.0
ctf_dict[mic_baseroot] = ctf_params
del ctf_list # Do not need this anymore
break
if_error_then_all_processes_exit_program(error_status)
if options.import_ctf:
if options.limit_ctf:
cutoff_histogram = [] #@ming compute the histogram for micrographs cut of by ctf_params limit.
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
restricted_serial_id_list = []
if myid == main_node:
# Loop over serial IDs of micrographs
for serial_id in serial_id_list:
# mic_baseroot is name of micrograph minus the path and extension
mic_baseroot = mic_baseroot_pattern.replace("*", serial_id)
mic_name = mic_pattern.replace("*", serial_id)
coords_name = coords_pattern.replace("*", serial_id)
########### # CHECKS: BEGIN
if coords_name not in coords_name_list:
print(" Cannot read %s. Skipping %s ..." % (coords_name, mic_baseroot))
n_mic_reject_no_coords += 1
continue
# IF mic is in CTER results
if options.import_ctf:
if mic_baseroot not in ctf_dict:
print(" Is not listed in CTER results. Skipping %s ..." % (mic_baseroot))
n_mic_reject_no_cter_entry += 1
continue
else:
ctf_params = ctf_dict[mic_baseroot]
# CHECKS: END
n_mic_process += 1
restricted_serial_id_list.append(serial_id)
# restricted_serial_id_list = restricted_serial_id_list[:128] ## for testing against the nonMPI version
if myid != main_node:
if options.import_ctf:
ctf_dict = None
error_status = None
if len(restricted_serial_id_list) < number_of_processes:
error_status = ('Number of processes (%d) supplied by --np in mpirun cannot be greater than %d (number of micrographs that satisfy all criteria to be processed) ' % (number_of_processes, len(restricted_serial_id_list)), getframeinfo(currentframe()))
if_error_then_all_processes_exit_program(error_status)
## keep a copy of the original output directory where the final bdb will be created
original_out_dir = out_dir
if RUNNING_UNDER_MPI:
mpi_barrier(MPI_COMM_WORLD)
restricted_serial_id_list = wrap_mpi_bcast(restricted_serial_id_list, main_node)
mic_start, mic_end = MPI_start_end(len(restricted_serial_id_list), number_of_processes, myid)
restricted_serial_id_list_not_sliced = restricted_serial_id_list
restricted_serial_id_list = restricted_serial_id_list[mic_start:mic_end]
if options.import_ctf:
ctf_dict = wrap_mpi_bcast(ctf_dict, main_node)
# generate subdirectories of out_dir, one for each process
out_dir = os.path.join(out_dir,"%03d"%myid)
if myid == main_node:
print("Micrographs processed by main process (including percent complete):")
len_processed_by_main_node_divided_by_100 = len(restricted_serial_id_list)/100.0
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
##################################################################################################################################################################################################################
##### Starting main parallel execution
for my_idx, serial_id in enumerate(restricted_serial_id_list):
mic_baseroot = mic_baseroot_pattern.replace("*", serial_id)
mic_name = mic_pattern.replace("*", serial_id)
coords_name = coords_pattern.replace("*", serial_id)
if myid == main_node:
print(mic_name, " ---> % 2.2f%%"%(my_idx/len_processed_by_main_node_divided_by_100))
mic_img = get_im(mic_name)
# Read coordinates according to the specified format and
# make the coordinates the center of particle image
if coords_format == "sparx":
coords_list = read_text_row(coords_name)
elif coords_format == "eman1":
coords_list = read_text_row(coords_name)
for i in xrange(len(coords_list)):
coords_list[i] = [(coords_list[i][0] + coords_list[i][2] // 2), (coords_list[i][1] + coords_list[i][3] // 2)]
elif coords_format == "eman2":
coords_list = js_open_dict(coords_name)["boxes"]
for i in xrange(len(coords_list)):
coords_list[i] = [coords_list[i][0], coords_list[i][1]]
elif coords_format == "spider":
coords_list = read_text_row(coords_name)
for i in xrange(len(coords_list)):
coords_list[i] = [coords_list[i][2], coords_list[i][3]]
# else: assert (False) # Unreachable code
# Calculate the new pixel size
if options.import_ctf:
ctf_params = ctf_dict[mic_baseroot]
pixel_size_origin = ctf_params[idx_cter_apix]
if resample_ratio < 1.0:
# assert (resample_ratio > 0.0)
new_pixel_size = pixel_size_origin / resample_ratio
print("Resample micrograph to pixel size %6.4f and window segments from resampled micrograph." % new_pixel_size)
else:
# assert (resample_ratio == 1.0)
new_pixel_size = pixel_size_origin
# Set ctf along with new pixel size in resampled micrograph
ctf_params[idx_cter_apix] = new_pixel_size
else:
# assert (not options.import_ctf)
if resample_ratio < 1.0:
# assert (resample_ratio > 0.0)
print("Resample micrograph with ratio %6.4f and window segments from resampled micrograph." % resample_ratio)
# else:
# assert (resample_ratio == 1.0)
# Apply filters to micrograph
fftip(mic_img)
if options.limit_ctf:
# assert (options.import_ctf)
# Cut off frequency components higher than CTF limit
q1, q2 = ctflimit(box_size, ctf_params[idx_cter_def], ctf_params[idx_cter_cs], ctf_params[idx_cter_vol], new_pixel_size)
# This is absolute frequency of CTF limit in scale of original micrograph
if resample_ratio < 1.0:
# assert (resample_ratio > 0.0)
q1 = resample_ratio * q1 / float(box_size) # q1 = (pixel_size_origin / new_pixel_size) * q1/float(box_size)
else:
# assert (resample_ratio == 1.0) -> pixel_size_origin == new_pixel_size -> pixel_size_origin / new_pixel_size == 1.0
q1 = q1 / float(box_size)
if q1 < 0.5:
mic_img = filt_tanl(mic_img, q1, 0.01)
cutoff_histogram.append(q1)
# Cut off frequency components lower than the box size can express
mic_img = fft(filt_gaussh(mic_img, resample_ratio / box_size))
# Resample micrograph, map coordinates, and window segments from resampled micrograph using new coordinates
# after resampling by resample_ratio, new pixel size will be pixel_size/resample_ratio = new_pixel_size
# NOTE: 2015/04/13 Toshio Moriya
# resample() efficiently takes care of the case resample_ratio = 1.0 but
# it does not set apix_*. Even though it sets apix_* when resample_ratio < 1.0 ...
mic_img = resample(mic_img, resample_ratio)
if options.invert:
mic_stats = Util.infomask(mic_img, None, True) # mic_stat[0:mean, 1:SD, 2:min, 3:max]
Util.mul_scalar(mic_img, -1.0)
mic_img += 2 * mic_stats[0]
if options.import_ctf:
from utilities import generate_ctf
ctf_obj = generate_ctf(ctf_params) # indexes 0 to 7 (idx_cter_def to idx_cter_astig_ang) must be same in cter format & ctf object format.
# Prepare loop variables
nx = mic_img.get_xsize()
ny = mic_img.get_ysize()
x0 = nx//2
y0 = ny//2
n_coords_reject_out_of_boundary = 0
local_stack_name = "bdb:%s#" % out_dir + mic_baseroot + '_ptcls'
local_particle_id = 0 # can be different from coordinates_id
# Loop over coordinates
for coords_id in xrange(len(coords_list)):
x = int(coords_list[coords_id][0])
y = int(coords_list[coords_id][1])
if resample_ratio < 1.0:
# assert (resample_ratio > 0.0)
x = int(x * resample_ratio)
y = int(y * resample_ratio)
# else:
# assert(resample_ratio == 1.0)
if( (0 <= x - box_half) and ( x + box_half <= nx ) and (0 <= y - box_half) and ( y + box_half <= ny ) ):
particle_img = Util.window(mic_img, box_size, box_size, 1, x-x0, y-y0)
else:
print("In %s, coordinates ID = %04d (x = %4d, y = %4d, box_size = %4d) is out of micrograph bound, skipping ..." % (mic_baseroot, coords_id, x, y, box_size))
n_coords_reject_out_of_boundary += 1
continue
particle_img = ramp(particle_img)
particle_stats = Util.infomask(particle_img, mask2d, False) # particle_stats[0:mean, 1:SD, 2:min, 3:max]
particle_img -= particle_stats[0]
particle_img /= particle_stats[1]
# NOTE: 2015/04/09 Toshio Moriya
# ptcl_source_image might be redundant information ...
# Consider re-organizing header entries...
particle_img.set_attr("ptcl_source_image", mic_name)
particle_img.set_attr("ptcl_source_coord_id", coords_id)
particle_img.set_attr("ptcl_source_coord", [int(coords_list[coords_id][0]), int(coords_list[coords_id][1])])
particle_img.set_attr("resample_ratio", resample_ratio)
# NOTE: 2015/04/13 Toshio Moriya
# apix_* attributes are updated by resample() only when resample_ratio != 1.0
# Let's make sure header info is consistent by setting apix_* = 1.0
# regardless of options, so it is not passed down the processing line
particle_img.set_attr("apix_x", 1.0)
particle_img.set_attr("apix_y", 1.0)
particle_img.set_attr("apix_z", 1.0)
if options.import_ctf:
particle_img.set_attr("ctf",ctf_obj)
particle_img.set_attr("ctf_applied", 0)
particle_img.set_attr("pixel_size_origin", pixel_size_origin)
# particle_img.set_attr("apix_x", new_pixel_size)
# particle_img.set_attr("apix_y", new_pixel_size)
# particle_img.set_attr("apix_z", new_pixel_size)
# NOTE: 2015/04/13 Toshio Moriya
# Pawel Comment: Micrograph is not supposed to have CTF header info.
# So, let's assume it does not exist & ignore its presence.
# Note that resample() "correctly" updates pixel size of CTF header info if it exists
# elif (particle_img.has_ctff()):
# assert(not options.import_ctf)
# ctf_origin = particle_img.get_attr("ctf_obj")
# pixel_size_origin = round(ctf_origin.apix, 5) # Because SXCTER ouputs up to 5 digits
# particle_img.set_attr("apix_x",pixel_size_origin)
# particle_img.set_attr("apix_y",pixel_size_origin)
# particle_img.set_attr("apix_z",pixel_size_origin)
# print("local_stack_name, local_particle_id", local_stack_name, local_particle_id)
particle_img.write_image(local_stack_name, local_particle_id)
local_particle_id += 1
n_global_coords_detect += len(coords_list)
n_global_coords_process += local_particle_id
n_global_coords_reject_out_of_boundary += n_coords_reject_out_of_boundary
# # MRK_DEBUG: Toshio Moriya 2016/05/03
# # Following codes are for debugging bdb. Delete in future
# result = db_check_dict(local_stack_name)
# print('# MRK_DEBUG: result = db_check_dict(local_stack_name): %s' % (result))
# result = db_list_dicts('bdb:%s' % out_dir)
# print('# MRK_DEBUG: result = db_list_dicts(out_dir): %s' % (result))
# result = db_get_image_info(local_stack_name)
# print('# MRK_DEBUG: result = db_get_image_info(local_stack_name)', result)
# Release the data base of local stack from this process
# so that the subprocess can access to the data base
db_close_dict(local_stack_name)
# # MRK_DEBUG: Toshio Moriya 2016/05/03
# # Following codes are for debugging bdb. Delete in future
# cmd_line = "e2iminfo.py %s" % (local_stack_name)
# print('# MRK_DEBUG: Executing the command: %s' % (cmd_line))
# cmdexecute(cmd_line)
# # MRK_DEBUG: Toshio Moriya 2016/05/03
# # Following codes are for debugging bdb. Delete in future
# cmd_line = "e2iminfo.py bdb:%s#data" % (out_dir)
# print('# MRK_DEBUG: Executing the command: %s' % (cmd_line))
# cmdexecute(cmd_line)
if RUNNING_UNDER_MPI:
if options.import_ctf:
if options.limit_ctf:
cutoff_histogram = wrap_mpi_gatherv(cutoff_histogram, main_node)
if myid == main_node:
if options.limit_ctf:
# Print out the summary of CTF-limit filtering
print(" ")
print("Global summary of CTF-limit filtering (--limit_ctf) ...")
print("Percentage of filtered micrographs: %8.2f\n" % (len(cutoff_histogram) * 100.0 / len(restricted_serial_id_list_not_sliced)))
n_bins = 10
if len(cutoff_histogram) >= n_bins:
from statistics import hist_list
cutoff_region, cutoff_counts = hist_list(cutoff_histogram, n_bins)
print(" Histogram of cut-off frequency")
print(" cut-off counts")
for bin_id in xrange(n_bins):
print(" %14.7f %7d" % (cutoff_region[bin_id], cutoff_counts[bin_id]))
else:
print("The number of filtered micrographs (%d) is less than the number of bins (%d). No histogram is produced." % (len(cutoff_histogram), n_bins))
n_mic_process = mpi_reduce(n_mic_process, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
n_mic_reject_no_coords = mpi_reduce(n_mic_reject_no_coords, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
n_mic_reject_no_cter_entry = mpi_reduce(n_mic_reject_no_cter_entry, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
n_global_coords_detect = mpi_reduce(n_global_coords_detect, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
n_global_coords_process = mpi_reduce(n_global_coords_process, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
n_global_coords_reject_out_of_boundary = mpi_reduce(n_global_coords_reject_out_of_boundary, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
# Print out the summary of all micrographs
if main_node == myid:
print(" ")
print("Global summary of micrographs ...")
print("Detected : %6d" % (len(restricted_serial_id_list_not_sliced)))
print("Processed : %6d" % (n_mic_process))
print("Rejected by no coordinates file : %6d" % (n_mic_reject_no_coords))
print("Rejected by no CTER entry : %6d" % (n_mic_reject_no_cter_entry))
print(" ")
print("Global summary of coordinates ...")
print("Detected : %6d" % (n_global_coords_detect))
print("Processed : %6d" % (n_global_coords_process))
print("Rejected by out of boundary : %6d" % (n_global_coords_reject_out_of_boundary))
# print(" ")
# print("DONE!!!")
mpi_barrier(MPI_COMM_WORLD)
if main_node == myid:
import time
time.sleep(1)
print("\n Creating bdb:%s/data\n"%original_out_dir)
for proc_i in range(number_of_processes):
mic_start, mic_end = MPI_start_end(len(restricted_serial_id_list_not_sliced), number_of_processes, proc_i)
for serial_id in restricted_serial_id_list_not_sliced[mic_start:mic_end]:
e2bdb_command = "e2bdb.py "
mic_baseroot = mic_baseroot_pattern.replace("*", serial_id)
if RUNNING_UNDER_MPI:
e2bdb_command += "bdb:" + os.path.join(original_out_dir,"%03d/"%proc_i) + mic_baseroot + "_ptcls "
else:
e2bdb_command += "bdb:" + os.path.join(original_out_dir, mic_baseroot + "_ptcls ")
e2bdb_command += " --appendvstack=bdb:%s/data 1>/dev/null"%original_out_dir
cmdexecute(e2bdb_command, printing_on_success = False)
print("Done!\n")
if RUNNING_UNDER_MPI:
mpi_barrier(MPI_COMM_WORLD)
from mpi import mpi_finalize
mpi_finalize()
sys.stdout.flush()
sys.exit(0)