def defocus_guess(Res_roo, Res_TE, volt, Cs, Pixel_size, wgh, istart=0, istop=-1, defocus_estimation_method=2, round_off=1, dz_low=1000., dz_high=200000., nloop=100, ampcont=0.0): """ Use specified frequencies area (istart-istop)to estimate defocus 1. The searching range is limited to dz_low (.1um) ~ dz_high (20 um). The user can modify this limitation accordingly 2. changing nloop can speed up the estimation 3. mode=1 compare squared error mode=2; compare cross correlation coefficients """ from math import sqrt from utilities import generate_ctf if istop <= istart : istop=len(Res_roo) step = (dz_high-dz_low)/nloop if step > 10000. : step = 10000. # angstrom if defocus_estimation_method == 1 : diff_min = 1.e38 else: diff_min = -1.e38 xval_e = 0 for ifreq in xrange(len(Res_TE)): xval_e += Res_TE[ifreq]**2 if (xval_e == 0): defocus = 0 return defocus if round_off >= 1: cut_off = 1. else: cut_off = round_off # do extreme fitting while (step >= cut_off): for i_dz in xrange(nloop): dz = dz_low + step*i_dz diff = 0 length = len(Res_roo) nx = int(length*2) ctf = ctf_2(nx, generate_ctf([dz,Cs,volt,Pixel_size,ampcont,wgh])) if defocus_estimation_method == 1: for ifreq in xrange(istart, istop, 1): diff += (ctf[ifreq]*Res_TE[ifreq] - Res_roo[ifreq])**2 if diff < diff_min or dz == dz_low: defocus = dz diff_min = diff else: diff = 0.0 sum_a = 0.0 sum_b = 0.0 for ifreq in xrange(istart, istop, 1): xval = ctf[ifreq]**2*Res_TE[ifreq] diff += Res_roo[ifreq]**2*xval sum_a += Res_roo[ifreq]**4 sum_b += xval**2 diff/=(sqrt(sum_a*sum_b)*( istop - istart + 1 )) if diff > diff_min or dz == dz_low: defocus = dz diff_min = diff xscore = diff_min dz_low = defocus-step*2 if( dz_low < 0 ): dz_low=0.0 dz_high = defocus + step*2 step/=10. defocus=int( defocus/round_off )*round_off return defocus
def stackToHDF(infile, outfile, apix, pinfo=None): """ convert stack to an hdf stack pinfo contains CTF information from apFrealign.getPerParticleCTF pinfo may also contain helical information """ from utilities import generate_ctf a = EMAN2.EMData() imn = EMAN2.EMUtil.get_image_count(infile) if pinfo is not None: if len(pinfo) != imn: apDisplay.printError("insufficient particle info for stack") # output must end with hdf outf, ext = os.path.splitext(outfile) if ext != '.hdf': outstack = outf + ".hdf" apDisplay.printMsg("Generating '%s' with %i particles" % (outstack, imn)) for i in xrange(imn): a.read_image(infile, i) a.set_attr_dict({'active': 1}) t2 = EMAN2.Transform({ "type": "spider", "phi": 0, "theta": 0, "psi": 0 }) a.set_attr("xform.projection", t2) a.set_attr("apix_x", apix) if pinfo is not None: pdata = pinfo[i + 1] df1 = pdata['df1'] df2 = pdata['df2'] astig = pdata['angastig'] kv = pdata['kev'] cs = pdata['cs'] ampc = pdata['ampc'] * 100 # save CTF dict (bfactor is hard coded to 0.0) df = (float(df1) + float(df2)) / 2 ctfgen = generate_ctf([df, cs, kv, apix, 0.0, ampc]) a.set_attr("ctf", ctfgen) # if helical info is present if pdata['hnum'] is not None: a.set_attr("h_angle", pdata['hangle']) a.write_image(outstack, i) return outstack
def main(): import os import sys from optparse import OptionParser arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + """ input_ctf_params input_ctf_params: output file from sx_cter.py, which was run in Multi-micrograph mode Note: files to be flipped should be in same directory. Outputs: phase-flipped files in hdf format, with identical filenames as input """ parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--input_ctf_params", type="string", default="partres.txt", help="output from sxcter.py") (options, args) = parser.parse_args(arglist[1:]) infile = options.input_ctf_params padimg = True #ctf correction -- pad image sign = 1 # ctf correction - sign binary = 1 # ctf correction: phaseflip only from utilities import generate_ctf from filter import filt_ctf global_def.BATCH = True file_content = open(infile) ctflist = file_content.readlines() img = EMData() for line in ctflist: data = line.split() imgfile = data[17] print "Working on file %s\n" % (imgfile) (name, ext) = imgfile.split(".") outfile = name + '.hdf' # Assumes input files are NOT hdf format! otherwise will overwrite them df = float(data[0]) cs = float(data[1]) voltage = float(data[2]) apix = float(data[3]) bfactor = float(data[4]) amp_contrast = float(data[5]) astig = float(data[6]) astig_ang = float(data[7]) ctf = generate_ctf( [df, cs, voltage, apix, bfactor, amp_contrast, astig, astig_ang]) img.read_image(imgfile) corr_img = filt_ctf(img, ctf, padimg, sign, binary) corr_img.write_image(outfile)
def generate_helimic(refvol, outdir, pixel, CTF=False, Cs=2.0,voltage = 200.0, ampcont = 10.0, nonoise = False, rand_seed=14567): from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im from random import random from projection import prgs, prep_vol from filter import filt_gaussl, filt_ctf from EMAN2 import EMAN2Ctf if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxhelical_demo", 1) os.mkdir(outdir) seed(rand_seed) Util.set_randnum_seed(rand_seed) angles =[] for i in xrange(3): angles.append( [0.0+60.0*i, 90.0-i*5, 0.0, 0.0, 0.0] ) nangle = len(angles) volfts = get_im(refvol) nx = volfts.get_xsize() ny = volfts.get_ysize() nz = volfts.get_zsize() volfts, kbx, kby, kbz = prep_vol( volfts ) iprj = 0 width = 500 xstart = 0 ystart = 0 for idef in xrange(3,6): mic = model_blank(2048, 2048) #defocus = idef*0.2 defocus = idef*0.6 ##@ming if CTF : #ctf = EMAN2Ctf() #ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} ) from utilities import generate_ctf ctf = generate_ctf([defocus,2,200,1.84,0.0,ampcont,defocus*0.2,80]) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice. i = idef - 4 for k in xrange(1): psi = 90 + 10*i proj = prgs(volfts, kbz, [angles[idef-3][0], angles[idef-3][1], psi, 0.0, 0.0], kbx, kby) proj = Util.window(proj, 320, nz) mic += pad(proj, 2048, 2048, 1, 0.0, 750*i, 20*i, 0) if not nonoise: mic += model_gauss_noise(30.0,2048,2048) if CTF : #apply CTF mic = filt_ctf(mic, ctf) if not nonoise: mic += filt_gaussl(model_gauss_noise(17.5,2048,2048), 0.3) mic.write_image("%s/mic%1d.hdf"%(outdir, idef-3),0)
def stackToHDF(infile,outfile,apix,pinfo=None): """ convert stack to an hdf stack pinfo contains CTF information from apFrealign.getPerParticleCTF pinfo may also contain helical information """ from utilities import generate_ctf a = EMAN2.EMData() imn = EMAN2.EMUtil.get_image_count(infile) if pinfo is not None: if len(pinfo) != imn: apDisplay.printError("insufficient particle info for stack") # output must end with hdf outf,ext = os.path.splitext(outfile) if ext != '.hdf': outstack = outf+".hdf" apDisplay.printMsg("Generating '%s' with %i particles"%(outstack,imn)) for i in xrange(imn): a.read_image(infile,i) a.set_attr_dict({'active':1}) t2 = EMAN2.Transform({"type":"spider","phi":0,"theta":0,"psi":0}) a.set_attr("xform.projection", t2) a.set_attr("apix_x",apix) if pinfo is not None: pdata = pinfo[i+1] df1 = pdata['df1'] df2 = pdata['df2'] astig = pdata['angastig'] kv = pdata['kev'] cs = pdata['cs'] ampc = pdata['ampc']*100 # save CTF dict (bfactor is hard coded to 0.0) df=(float(df1)+float(df2))/2 ctfgen = generate_ctf([df,cs,kv,apix,0.0,ampc]) a.set_attr("ctf",ctfgen) # if helical info is present if pdata['hnum'] is not None: a.set_attr("h_angle",pdata['hangle']) a.write_image(outstack,i) return outstack
def defocus_gett(roo, voltage=300.0, Pixel_size=1.0, Cs=2.0, wgh=0.1, f_start=0.0, f_stop=-1.0, round_off=1.0, nr1=3, nr2=6, parent=None): """ 1. Estimate envelope function and baseline noise using constrained simplex method so as to extract CTF imprints from 1D power spectrum 2. Based one extracted ctf imprints, perform exhaustive defocus searching to get defocus which matches the extracted CTF imprints """ from utilities import generate_ctf #print "CTF params:", voltage, Pixel_size, Cs, wgh, f_start, f_stop, round_off, nr1, nr2, parent res = [] Res_roo = [] Res_TE = [] if f_start == 0 : i_start = 0 else: i_start = int(Pixel_size*2.*len(roo)*f_start) if f_stop <= f_start : i_stop = len(roo) else: i_stop = int(Pixel_size*2.*len(roo)*f_stop) if i_stop > len(roo): i_stop = len(roo) #print "f_start, i_start, f_stop, i_stop:", f_start, i_start, f_stop, i_stop TE = defocus_env_baseline_fit(roo, i_start, i_stop, int(nr1), 4) Pn1 = defocus_env_baseline_fit(roo, i_start, i_stop, int(nr2), 3) Res_roo = [] Res_TE = [] for i in xrange(len(roo)): Res_roo.append(roo[i] - Pn1[i]) Res_TE.append( TE[i] - Pn1[i]) defocus = defocus_guess(Res_roo, Res_TE, voltage, Cs, Pixel_size, wgh, i_start, i_stop, 2, round_off) nx = int(len(Res_roo)*2) #ctf = ctf_1d(nx, generate_ctf([defocus, Cs, voltage, Pixel_size, 0.0, wgh])) ctf = ctf_2(nx, generate_ctf([defocus, Cs, voltage, Pixel_size, 0.0, wgh])) if (parent is not None): parent.ctf_data=[roo, Res_roo, Res_TE] parent.i_start = i_start parent.i_stop = i_stop from utilities import write_text_file write_text_file([roo, Res_roo, Res_TE, ctf], "procpw.txt") else: from utilities import write_text_file write_text_file([roo, Res_roo, Res_TE, ctf], "procpw.txt") return defocus
def defocus_get_Eudis(fnam_roo, volt=300, Pixel_size=1, Cs=2, wgh=.1, f_start=0, f_stop=-1, docf="a" ,skip="#", round_off=1, nr1=3, nr2=6): """ 1. Estimating envelope function and baseline noise using constrained simplex method so as to extract CTF imprints from 1D power spectrum 2. Based one extracted ctf imprints, perform exhaustive defocus searching to get defocus which matches the extracted CTF imprints 3. It returns Euclidean distance for defocus selection """ from math import sqrt, atan from utilities import read_text_row, generate_ctf roo = [] res = [] if docf == "a": TMP_roo = read_text_row(fnam_roo, "a", skip) for i in xrange(len(TMP_roo)): # remove first record roo.append(TMP_roo[i][1]) else: skip = ";" TMP_roo = read_text_row(fnam_roo, "s", skip) for i in xrange(len(TMP_roo)): # remove first record roo.append(TMP_roo[i][2]) Res_roo = [] Res_TE = [] if f_start == 0 : i_start=0 else: i_start=int(Pixel_size*2.*len(roo)/f_start) if f_stop <= i_start : i_stop=len(roo) else: i_stop=int(Pixel_size*2.*len(roo)/f_stop) TE = defocus_env_baseline_fit(roo, i_start, i_stop, int(nr1), 4) Pn1 = defocus_env_baseline_fit(roo, i_start, i_stop, int(nr2), 3) Res_roo = [] Res_TE = [] for i in xrange(len(roo)): Res_roo.append( roo[i] - Pn1[i] ) Res_TE.append( TE[i] - Pn1[i] ) # defocus=defocus_guess(Res_roo, Res_TE, volt, Cs, Pixel_size, wgh, i_start, i_stop, 2, round_off) nx = int(len(roo)*2) ctf = ctf_2(nx, generate_ctf([defocus,Cs,voltage,Pixel_size, 0.0, wgh])) for i in xrange(len(Res_TE)): ctf[i]=ctf[i]*Res_TE[i] dis = defocus_L2_euc(ctf, Res_roo, i_start, i_stop) return [defocus, dis]
def generate_helimic(refvol, outdir, pixel, CTF=False, Cs=2.0, voltage=200.0, ampcont=10.0, nonoise=False, rand_seed=14567): from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im from random import random from projection import prgs, prep_vol from filter import filt_gaussl, filt_ctf from EMAN2 import EMAN2Ctf if os.path.exists(outdir): ERROR( 'Output directory exists, please change the name and restart the program', "sxhelical_demo", 1) os.mkdir(outdir) seed(rand_seed) Util.set_randnum_seed(rand_seed) angles = [] for i in range(3): angles.append([0.0 + 60.0 * i, 90.0 - i * 5, 0.0, 0.0, 0.0]) nangle = len(angles) volfts = get_im(refvol) nx = volfts.get_xsize() ny = volfts.get_ysize() nz = volfts.get_zsize() volfts, kbx, kby, kbz = prep_vol(volfts) iprj = 0 width = 500 xstart = 0 ystart = 0 for idef in range(3, 6): mic = model_blank(2048, 2048) #defocus = idef*0.2 defocus = idef * 0.6 ##@ming if CTF: #ctf = EMAN2Ctf() #ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} ) from utilities import generate_ctf ctf = generate_ctf( [defocus, 2, 200, 1.84, 0.0, ampcont, defocus * 0.2, 80] ) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice. i = idef - 4 for k in range(1): psi = 90 + 10 * i proj = prgs( volfts, kbz, [angles[idef - 3][0], angles[idef - 3][1], psi, 0.0, 0.0], kbx, kby) proj = Util.window(proj, 320, nz) mic += pad(proj, 2048, 2048, 1, 0.0, 750 * i, 20 * i, 0) if not nonoise: mic += model_gauss_noise(30.0, 2048, 2048) if CTF: #apply CTF mic = filt_ctf(mic, ctf) if not nonoise: mic += filt_gaussl(model_gauss_noise(17.5, 2048, 2048), 0.3) mic.write_image("%s/mic%1d.hdf" % (outdir, idef - 3), 0)
def main2(): arglist = [] for arg in sys.argv: arglist.append( arg ) progname = os.path.basename(arglist[0]) usage = progname + " raw_images apix <CTFparfile>" parser = OptionParser(usage,version="1.0") (options, args) = parser.parse_args(arglist[1:]) if len(args) < 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" else: a = EMData() fname = args[0] apix = float(args[1]) ctfpar = None helical = False if len(args) == 3: ctfpar = args[2] if not os.path.isfile(ctfpar): print "Error: ctf file '%s' is not found"%ctfpar sys.exit() # get ctf values from parfile from utilities import generate_ctf pfile = open(ctfpar).readlines() ctfinfo=[0]*len(pfile) # check if there are helical parameters: if len(pfile[0].strip().split())==9: helical = True hinfo=[0]*len(pfile) # store info for p in pfile: nfo = p.strip().split() pnum=int(float(nfo[0]))-1 df1=float(nfo[1]) df2=float(nfo[2]) astig = float(nfo[3]) kv = int(float(nfo[4])) cs = float(nfo[5]) ampc = float(nfo[6])*100 if helical is True: hnum = int(float(nfo[7])) angle = float(nfo[8]) angle = -angle+90 if angle > 360: angle -= 360 if angle < 0: angle += 360 hinfo[pnum]=angle """ # For future use if taking from FREALIGN if p[0]=="C": continue (pnum,psi,theta,phi,shx,shy,mag,hnum,ctf1,ctf2,angle,pres,delta)=p.strip().split() pnum=int(pnum)-1 kv=300 cs=2.7 ampc=5 """ df=(float(df1)+float(df2))/2 ctfinfo[pnum]=generate_ctf([df,cs,kv,apix,0.0,ampc]) imn = EMUtil.get_image_count(fname) print "Generating 'start.hdf' with %i particles"%imn for i in xrange(imn): a.read_image(fname,i) a.set_attr_dict({'active':1}) if ctfpar is not None: a.set_attr("ctf",ctfinfo[i]) if helical is True: a.set_attr("h_angle",hinfo[i]) t2 = Transform({"type":"spider","phi":0,"theta":0,"psi":0}) a.set_attr("xform.projection", t2) a.set_attr("apix_x",apix ) a.write_image("start.hdf",i) print "%3i%% complete\t\r"%(int(float(i)/imn*100)), print "100% complete\t"
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + """ input_micrograph_list_file input_micrograph_pattern input_coordinates_pattern output_directory --coordinates_format --box_size=box_size --invert --import_ctf=ctf_file --limit_ctf --resample_ratio=resample_ratio --defocus_error=defocus_error --astigmatism_error=astigmatism_error Window particles from micrographs in input list file. The coordinates of the particles should be given as input. Please specify name pattern of input micrographs and coordinates files with a wild card (*). Use the wild card to indicate the place of micrograph ID (e.g. serial number, time stamp, and etc). The name patterns must be enclosed by single quotes (') or double quotes ("). (Note: sxgui.py automatically adds single quotes (')). BDB files can not be selected as input micrographs. sxwindow.py mic_list.txt ./mic*.hdf info/mic*_info.json particles --coordinates_format=eman2 --box_size=64 --invert --import_ctf=outdir_cter/partres/partres.txt If micrograph list file name is not provided, all files matched with the micrograph name pattern will be processed. sxwindow.py ./mic*.hdf info/mic*_info.json particles --coordinates_format=eman2 --box_size=64 --invert --import_ctf=outdir_cter/partres/partres.txt """ parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--coordinates_format", type="string", default="eman1", help= "format of input coordinates files: 'sparx', 'eman1', 'eman2', or 'spider'. the coordinates of sparx, eman2, and spider format is particle center. the coordinates of eman1 format is particle box conner associated with the original box size. (default eman1)" ) parser.add_option( "--box_size", type="int", default=256, help= "x and y dimension of square area to be windowed (in pixels): pixel size after resampling is assumed when resample_ratio < 1.0 (default 256)" ) parser.add_option( "--invert", action="store_true", default=False, help="invert image contrast: recommended for cryo data (default False)" ) parser.add_option( "--import_ctf", type="string", default="", help="file name of sxcter output: normally partres.txt (default none)") parser.add_option( "--limit_ctf", action="store_true", default=False, help= "filter micrographs based on the CTF limit: this option requires --import_ctf. (default False)" ) parser.add_option( "--resample_ratio", type="float", default=1.0, help= "ratio of new to old image size (or old to new pixel size) for resampling: Valid range is 0.0 < resample_ratio <= 1.0. (default 1.0)" ) parser.add_option( "--defocus_error", type="float", default=1000000.0, help= "defocus errror limit: exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocus_error percent. the error is computed as (std dev defocus)/defocus*100%. (default 1000000.0)" ) parser.add_option( "--astigmatism_error", type="float", default=360.0, help= "astigmatism error limit: Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatism_error degrees. (default 360.0)" ) ### detect if program is running under MPI RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ main_node = 0 if RUNNING_UNDER_MPI: from mpi import mpi_init from mpi import MPI_COMM_WORLD, mpi_comm_rank, mpi_comm_size, mpi_barrier, mpi_reduce, MPI_INT, MPI_SUM mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_processes = mpi_comm_size(MPI_COMM_WORLD) else: number_of_processes = 1 myid = 0 (options, args) = parser.parse_args(sys.argv[1:]) mic_list_file_path = None mic_pattern = None coords_pattern = None error_status = None while True: if len(args) < 3 or len(args) > 4: error_status = ( "Please check usage for number of arguments.\n Usage: " + usage + "\n" + "Please run %s -h for help." % (progname), getframeinfo(currentframe())) break if len(args) == 3: mic_pattern = args[0] coords_pattern = args[1] out_dir = args[2] else: # assert(len(args) == 4) mic_list_file_path = args[0] mic_pattern = args[1] coords_pattern = args[2] out_dir = args[3] if mic_list_file_path != None: if os.path.splitext(mic_list_file_path)[1] != ".txt": error_status = ( "Extension of input micrograph list file must be \".txt\". Please check input_micrograph_list_file argument. Run %s -h for help." % (progname), getframeinfo(currentframe())) break if mic_pattern[:len("bdb:")].lower() == "bdb": error_status = ( "BDB file can not be selected as input micrographs. Please convert the format, and restart the program. Run %s -h for help." % (progname), getframeinfo(currentframe())) break if mic_pattern.find("*") == -1: error_status = ( "Input micrograph file name pattern must contain wild card (*). Please check input_micrograph_pattern argument. Run %s -h for help." % (progname), getframeinfo(currentframe())) break if coords_pattern.find("*") == -1: error_status = ( "Input coordinates file name pattern must contain wild card (*). Please check input_coordinates_pattern argument. Run %s -h for help." % (progname), getframeinfo(currentframe())) break if myid == main_node: if os.path.exists(out_dir): error_status = ( "Output directory exists. Please change the name and restart the program.", getframeinfo(currentframe())) break break if_error_then_all_processes_exit_program(error_status) # Check invalid conditions of options check_options(options, progname) mic_name_list = None error_status = None if myid == main_node: if mic_list_file_path != None: print("Loading micrograph list from %s file ..." % (mic_list_file_path)) mic_name_list = read_text_file(mic_list_file_path) if len(mic_name_list) == 0: print("Directory of first micrograph entry is " % (os.path.dirname(mic_name_list[0]))) else: # assert (mic_list_file_path == None) print("Generating micrograph list in %s directory..." % (os.path.dirname(mic_pattern))) mic_name_list = glob.glob(mic_pattern) if len(mic_name_list) == 0: error_status = ( "No micrograph file is found. Please check input_micrograph_pattern and/or input_micrograph_list_file argument. Run %s -h for help." % (progname), getframeinfo(currentframe())) else: print("Found %d microgarphs" % len(mic_name_list)) if_error_then_all_processes_exit_program(error_status) if RUNNING_UNDER_MPI: mic_name_list = wrap_mpi_bcast(mic_name_list, main_node) coords_name_list = None error_status = None if myid == main_node: coords_name_list = glob.glob(coords_pattern) if len(coords_name_list) == 0: error_status = ( "No coordinates file is found. Please check input_coordinates_pattern argument. Run %s -h for help." % (progname), getframeinfo(currentframe())) if_error_then_all_processes_exit_program(error_status) if RUNNING_UNDER_MPI: coords_name_list = wrap_mpi_bcast(coords_name_list, main_node) ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## # all processes must have access to indices if options.import_ctf: i_enum = -1 i_enum += 1 idx_cter_def = i_enum # defocus [um]; index must be same as ctf object format i_enum += 1 idx_cter_cs = i_enum # Cs [mm]; index must be same as ctf object format i_enum += 1 idx_cter_vol = i_enum # voltage[kV]; index must be same as ctf object format i_enum += 1 idx_cter_apix = i_enum # pixel size [A]; index must be same as ctf object format i_enum += 1 idx_cter_bfactor = i_enum # B-factor [A^2]; index must be same as ctf object format i_enum += 1 idx_cter_ac = i_enum # amplitude contrast [%]; index must be same as ctf object format i_enum += 1 idx_cter_astig_amp = i_enum # astigmatism amplitude [um]; index must be same as ctf object format i_enum += 1 idx_cter_astig_ang = i_enum # astigmatism angle [degree]; index must be same as ctf object format i_enum += 1 idx_cter_sd_def = i_enum # std dev of defocus [um] i_enum += 1 idx_cter_sd_astig_amp = i_enum # std dev of ast amp [A] i_enum += 1 idx_cter_sd_astig_ang = i_enum # std dev of ast angle [degree] i_enum += 1 idx_cter_cv_def = i_enum # coefficient of variation of defocus [%] i_enum += 1 idx_cter_cv_astig_amp = i_enum # coefficient of variation of ast amp [%] i_enum += 1 idx_cter_spectra_diff = i_enum # average of differences between with- and without-astig. experimental 1D spectra at extrema i_enum += 1 idx_cter_error_def = i_enum # frequency at which signal drops by 50% due to estimated error of defocus alone [1/A] i_enum += 1 idx_cter_error_astig = i_enum # frequency at which signal drops by 50% due to estimated error of defocus and astigmatism [1/A] i_enum += 1 idx_cter_error_ctf = i_enum # limit frequency by CTF error [1/A] i_enum += 1 idx_cter_mic_name = i_enum # micrograph name i_enum += 1 n_idx_cter = i_enum # Prepare loop variables mic_basename_pattern = os.path.basename( mic_pattern) # file pattern without path mic_baseroot_pattern = os.path.splitext(mic_basename_pattern)[ 0] # file pattern without path and extension coords_format = options.coordinates_format.lower() box_size = options.box_size box_half = box_size // 2 mask2d = model_circle( box_size // 2, box_size, box_size ) # Create circular 2D mask to Util.infomask of particle images resample_ratio = options.resample_ratio n_mic_process = 0 n_mic_reject_no_coords = 0 n_mic_reject_no_cter_entry = 0 n_global_coords_detect = 0 n_global_coords_process = 0 n_global_coords_reject_out_of_boundary = 0 serial_id_list = [] error_status = None ## not a real while, an if with the opportunity to use break when errors need to be reported while myid == main_node: # # NOTE: 2016/05/24 Toshio Moriya # Now, ignores the path in mic_pattern and entries of mic_name_list to create serial ID # Only the basename (file name) in micrograph path must be match # # Create list of micrograph serial ID # Break micrograph name pattern into prefix and suffix to find the head index of the micrograph serial id # mic_basename_tokens = mic_basename_pattern.split('*') # assert (len(mic_basename_tokens) == 2) serial_id_head_index = len(mic_basename_tokens[0]) # Loop through micrograph names for mic_name in mic_name_list: # Find the tail index of the serial id and extract serial id from the micrograph name mic_basename = os.path.basename(mic_name) serial_id_tail_index = mic_basename.index(mic_basename_tokens[1]) serial_id = mic_basename[serial_id_head_index:serial_id_tail_index] serial_id_list.append(serial_id) # assert (len(serial_id_list) == len(mic_name)) del mic_name_list # Do not need this anymore # Load CTFs if necessary if options.import_ctf: ctf_list = read_text_row(options.import_ctf) # print("Detected CTF entries : %6d ..." % (len(ctf_list))) if len(ctf_list) == 0: error_status = ( "No CTF entry is found in %s. Please check --import_ctf option. Run %s -h for help." % (options.import_ctf, progname), getframeinfo(currentframe())) break if (len(ctf_list[0]) != n_idx_cter): error_status = ( "Number of columns (%d) must be %d in %s. The format might be old. Please run sxcter.py again." % (len(ctf_list[0]), n_idx_cter, options.import_ctf), getframeinfo(currentframe())) break ctf_dict = {} n_reject_defocus_error = 0 ctf_error_limit = [ options.defocus_error / 100.0, options.astigmatism_error ] for ctf_params in ctf_list: assert (len(ctf_params) == n_idx_cter) # mic_baseroot is name of micrograph minus the path and extension mic_baseroot = os.path.splitext( os.path.basename(ctf_params[idx_cter_mic_name]))[0] if (ctf_params[idx_cter_sd_def] / ctf_params[idx_cter_def] > ctf_error_limit[0]): print( "Defocus error %f exceeds the threshold. Micrograph %s is rejected." % (ctf_params[idx_cter_sd_def] / ctf_params[idx_cter_def], mic_baseroot)) n_reject_defocus_error += 1 else: if (ctf_params[idx_cter_sd_astig_ang] > ctf_error_limit[1]): ctf_params[idx_cter_astig_amp] = 0.0 ctf_params[idx_cter_astig_ang] = 0.0 ctf_dict[mic_baseroot] = ctf_params del ctf_list # Do not need this anymore break if_error_then_all_processes_exit_program(error_status) if options.import_ctf: if options.limit_ctf: cutoff_histogram = [ ] #@ming compute the histogram for micrographs cut of by ctf_params limit. ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## restricted_serial_id_list = [] if myid == main_node: # Loop over serial IDs of micrographs for serial_id in serial_id_list: # mic_baseroot is name of micrograph minus the path and extension mic_baseroot = mic_baseroot_pattern.replace("*", serial_id) mic_name = mic_pattern.replace("*", serial_id) coords_name = coords_pattern.replace("*", serial_id) ########### # CHECKS: BEGIN if coords_name not in coords_name_list: print(" Cannot read %s. Skipping %s ..." % (coords_name, mic_baseroot)) n_mic_reject_no_coords += 1 continue # IF mic is in CTER results if options.import_ctf: if mic_baseroot not in ctf_dict: print( " Is not listed in CTER results. Skipping %s ..." % (mic_baseroot)) n_mic_reject_no_cter_entry += 1 continue else: ctf_params = ctf_dict[mic_baseroot] # CHECKS: END n_mic_process += 1 restricted_serial_id_list.append(serial_id) # restricted_serial_id_list = restricted_serial_id_list[:128] ## for testing against the nonMPI version if myid != main_node: if options.import_ctf: ctf_dict = None error_status = None if len(restricted_serial_id_list) < number_of_processes: error_status = ( 'Number of processes (%d) supplied by --np in mpirun cannot be greater than %d (number of micrographs that satisfy all criteria to be processed) ' % (number_of_processes, len(restricted_serial_id_list)), getframeinfo(currentframe())) if_error_then_all_processes_exit_program(error_status) ## keep a copy of the original output directory where the final bdb will be created original_out_dir = out_dir if RUNNING_UNDER_MPI: mpi_barrier(MPI_COMM_WORLD) restricted_serial_id_list = wrap_mpi_bcast(restricted_serial_id_list, main_node) mic_start, mic_end = MPI_start_end(len(restricted_serial_id_list), number_of_processes, myid) restricted_serial_id_list_not_sliced = restricted_serial_id_list restricted_serial_id_list = restricted_serial_id_list[ mic_start:mic_end] if options.import_ctf: ctf_dict = wrap_mpi_bcast(ctf_dict, main_node) # generate subdirectories of out_dir, one for each process out_dir = os.path.join(out_dir, "%03d" % myid) if myid == main_node: print( "Micrographs processed by main process (including percent complete):" ) len_processed_by_main_node_divided_by_100 = len( restricted_serial_id_list) / 100.0 ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## ##### Starting main parallel execution for my_idx, serial_id in enumerate(restricted_serial_id_list): mic_baseroot = mic_baseroot_pattern.replace("*", serial_id) mic_name = mic_pattern.replace("*", serial_id) coords_name = coords_pattern.replace("*", serial_id) if myid == main_node: print( mic_name, " ---> % 2.2f%%" % (my_idx / len_processed_by_main_node_divided_by_100)) mic_img = get_im(mic_name) # Read coordinates according to the specified format and # make the coordinates the center of particle image if coords_format == "sparx": coords_list = read_text_row(coords_name) elif coords_format == "eman1": coords_list = read_text_row(coords_name) for i in xrange(len(coords_list)): coords_list[i] = [(coords_list[i][0] + coords_list[i][2] // 2), (coords_list[i][1] + coords_list[i][3] // 2)] elif coords_format == "eman2": coords_list = js_open_dict(coords_name)["boxes"] for i in xrange(len(coords_list)): coords_list[i] = [coords_list[i][0], coords_list[i][1]] elif coords_format == "spider": coords_list = read_text_row(coords_name) for i in xrange(len(coords_list)): coords_list[i] = [coords_list[i][2], coords_list[i][3]] # else: assert (False) # Unreachable code # Calculate the new pixel size if options.import_ctf: ctf_params = ctf_dict[mic_baseroot] pixel_size_origin = ctf_params[idx_cter_apix] if resample_ratio < 1.0: # assert (resample_ratio > 0.0) new_pixel_size = pixel_size_origin / resample_ratio print( "Resample micrograph to pixel size %6.4f and window segments from resampled micrograph." % new_pixel_size) else: # assert (resample_ratio == 1.0) new_pixel_size = pixel_size_origin # Set ctf along with new pixel size in resampled micrograph ctf_params[idx_cter_apix] = new_pixel_size else: # assert (not options.import_ctf) if resample_ratio < 1.0: # assert (resample_ratio > 0.0) print( "Resample micrograph with ratio %6.4f and window segments from resampled micrograph." % resample_ratio) # else: # assert (resample_ratio == 1.0) # Apply filters to micrograph fftip(mic_img) if options.limit_ctf: # assert (options.import_ctf) # Cut off frequency components higher than CTF limit q1, q2 = ctflimit(box_size, ctf_params[idx_cter_def], ctf_params[idx_cter_cs], ctf_params[idx_cter_vol], new_pixel_size) # This is absolute frequency of CTF limit in scale of original micrograph if resample_ratio < 1.0: # assert (resample_ratio > 0.0) q1 = resample_ratio * q1 / float( box_size ) # q1 = (pixel_size_origin / new_pixel_size) * q1/float(box_size) else: # assert (resample_ratio == 1.0) -> pixel_size_origin == new_pixel_size -> pixel_size_origin / new_pixel_size == 1.0 q1 = q1 / float(box_size) if q1 < 0.5: mic_img = filt_tanl(mic_img, q1, 0.01) cutoff_histogram.append(q1) # Cut off frequency components lower than the box size can express mic_img = fft(filt_gaussh(mic_img, resample_ratio / box_size)) # Resample micrograph, map coordinates, and window segments from resampled micrograph using new coordinates # after resampling by resample_ratio, new pixel size will be pixel_size/resample_ratio = new_pixel_size # NOTE: 2015/04/13 Toshio Moriya # resample() efficiently takes care of the case resample_ratio = 1.0 but # it does not set apix_*. Even though it sets apix_* when resample_ratio < 1.0 ... mic_img = resample(mic_img, resample_ratio) if options.invert: mic_stats = Util.infomask( mic_img, None, True) # mic_stat[0:mean, 1:SD, 2:min, 3:max] Util.mul_scalar(mic_img, -1.0) mic_img += 2 * mic_stats[0] if options.import_ctf: from utilities import generate_ctf ctf_obj = generate_ctf( ctf_params ) # indexes 0 to 7 (idx_cter_def to idx_cter_astig_ang) must be same in cter format & ctf object format. # Prepare loop variables nx = mic_img.get_xsize() ny = mic_img.get_ysize() x0 = nx // 2 y0 = ny // 2 n_coords_reject_out_of_boundary = 0 local_stack_name = "bdb:%s#" % out_dir + mic_baseroot + '_ptcls' local_particle_id = 0 # can be different from coordinates_id # Loop over coordinates for coords_id in xrange(len(coords_list)): x = int(coords_list[coords_id][0]) y = int(coords_list[coords_id][1]) if resample_ratio < 1.0: # assert (resample_ratio > 0.0) x = int(x * resample_ratio) y = int(y * resample_ratio) # else: # assert(resample_ratio == 1.0) if ((0 <= x - box_half) and (x + box_half <= nx) and (0 <= y - box_half) and (y + box_half <= ny)): particle_img = Util.window(mic_img, box_size, box_size, 1, x - x0, y - y0) else: print( "In %s, coordinates ID = %04d (x = %4d, y = %4d, box_size = %4d) is out of micrograph bound, skipping ..." % (mic_baseroot, coords_id, x, y, box_size)) n_coords_reject_out_of_boundary += 1 continue particle_img = ramp(particle_img) particle_stats = Util.infomask( particle_img, mask2d, False) # particle_stats[0:mean, 1:SD, 2:min, 3:max] particle_img -= particle_stats[0] particle_img /= particle_stats[1] # NOTE: 2015/04/09 Toshio Moriya # ptcl_source_image might be redundant information ... # Consider re-organizing header entries... particle_img.set_attr("ptcl_source_image", mic_name) particle_img.set_attr("ptcl_source_coord_id", coords_id) particle_img.set_attr("ptcl_source_coord", [ int(coords_list[coords_id][0]), int(coords_list[coords_id][1]) ]) particle_img.set_attr("resample_ratio", resample_ratio) # NOTE: 2015/04/13 Toshio Moriya # apix_* attributes are updated by resample() only when resample_ratio != 1.0 # Let's make sure header info is consistent by setting apix_* = 1.0 # regardless of options, so it is not passed down the processing line particle_img.set_attr("apix_x", 1.0) particle_img.set_attr("apix_y", 1.0) particle_img.set_attr("apix_z", 1.0) if options.import_ctf: particle_img.set_attr("ctf", ctf_obj) particle_img.set_attr("ctf_applied", 0) particle_img.set_attr("pixel_size_origin", pixel_size_origin) # particle_img.set_attr("apix_x", new_pixel_size) # particle_img.set_attr("apix_y", new_pixel_size) # particle_img.set_attr("apix_z", new_pixel_size) # NOTE: 2015/04/13 Toshio Moriya # Pawel Comment: Micrograph is not supposed to have CTF header info. # So, let's assume it does not exist & ignore its presence. # Note that resample() "correctly" updates pixel size of CTF header info if it exists # elif (particle_img.has_ctff()): # assert(not options.import_ctf) # ctf_origin = particle_img.get_attr("ctf_obj") # pixel_size_origin = round(ctf_origin.apix, 5) # Because SXCTER ouputs up to 5 digits # particle_img.set_attr("apix_x",pixel_size_origin) # particle_img.set_attr("apix_y",pixel_size_origin) # particle_img.set_attr("apix_z",pixel_size_origin) # print("local_stack_name, local_particle_id", local_stack_name, local_particle_id) particle_img.write_image(local_stack_name, local_particle_id) local_particle_id += 1 n_global_coords_detect += len(coords_list) n_global_coords_process += local_particle_id n_global_coords_reject_out_of_boundary += n_coords_reject_out_of_boundary # # MRK_DEBUG: Toshio Moriya 2016/05/03 # # Following codes are for debugging bdb. Delete in future # result = db_check_dict(local_stack_name) # print('# MRK_DEBUG: result = db_check_dict(local_stack_name): %s' % (result)) # result = db_list_dicts('bdb:%s' % out_dir) # print('# MRK_DEBUG: result = db_list_dicts(out_dir): %s' % (result)) # result = db_get_image_info(local_stack_name) # print('# MRK_DEBUG: result = db_get_image_info(local_stack_name)', result) # Release the data base of local stack from this process # so that the subprocess can access to the data base db_close_dict(local_stack_name) # # MRK_DEBUG: Toshio Moriya 2016/05/03 # # Following codes are for debugging bdb. Delete in future # cmd_line = "e2iminfo.py %s" % (local_stack_name) # print('# MRK_DEBUG: Executing the command: %s' % (cmd_line)) # cmdexecute(cmd_line) # # MRK_DEBUG: Toshio Moriya 2016/05/03 # # Following codes are for debugging bdb. Delete in future # cmd_line = "e2iminfo.py bdb:%s#data" % (out_dir) # print('# MRK_DEBUG: Executing the command: %s' % (cmd_line)) # cmdexecute(cmd_line) if RUNNING_UNDER_MPI: if options.import_ctf: if options.limit_ctf: cutoff_histogram = wrap_mpi_gatherv(cutoff_histogram, main_node) if myid == main_node: if options.limit_ctf: # Print out the summary of CTF-limit filtering print(" ") print("Global summary of CTF-limit filtering (--limit_ctf) ...") print("Percentage of filtered micrographs: %8.2f\n" % (len(cutoff_histogram) * 100.0 / len(restricted_serial_id_list_not_sliced))) n_bins = 10 if len(cutoff_histogram) >= n_bins: from statistics import hist_list cutoff_region, cutoff_counts = hist_list( cutoff_histogram, n_bins) print(" Histogram of cut-off frequency") print(" cut-off counts") for bin_id in xrange(n_bins): print(" %14.7f %7d" % (cutoff_region[bin_id], cutoff_counts[bin_id])) else: print( "The number of filtered micrographs (%d) is less than the number of bins (%d). No histogram is produced." % (len(cutoff_histogram), n_bins)) n_mic_process = mpi_reduce(n_mic_process, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) n_mic_reject_no_coords = mpi_reduce(n_mic_reject_no_coords, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) n_mic_reject_no_cter_entry = mpi_reduce(n_mic_reject_no_cter_entry, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) n_global_coords_detect = mpi_reduce(n_global_coords_detect, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) n_global_coords_process = mpi_reduce(n_global_coords_process, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) n_global_coords_reject_out_of_boundary = mpi_reduce( n_global_coords_reject_out_of_boundary, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) # Print out the summary of all micrographs if main_node == myid: print(" ") print("Global summary of micrographs ...") print("Detected : %6d" % (len(restricted_serial_id_list_not_sliced))) print("Processed : %6d" % (n_mic_process)) print("Rejected by no coordinates file : %6d" % (n_mic_reject_no_coords)) print("Rejected by no CTER entry : %6d" % (n_mic_reject_no_cter_entry)) print(" ") print("Global summary of coordinates ...") print("Detected : %6d" % (n_global_coords_detect)) print("Processed : %6d" % (n_global_coords_process)) print("Rejected by out of boundary : %6d" % (n_global_coords_reject_out_of_boundary)) # print(" ") # print("DONE!!!") mpi_barrier(MPI_COMM_WORLD) if main_node == myid: import time time.sleep(1) print("\n Creating bdb:%s/data\n" % original_out_dir) for proc_i in range(number_of_processes): mic_start, mic_end = MPI_start_end( len(restricted_serial_id_list_not_sliced), number_of_processes, proc_i) for serial_id in restricted_serial_id_list_not_sliced[ mic_start:mic_end]: e2bdb_command = "e2bdb.py " mic_baseroot = mic_baseroot_pattern.replace("*", serial_id) if RUNNING_UNDER_MPI: e2bdb_command += "bdb:" + os.path.join( original_out_dir, "%03d/" % proc_i) + mic_baseroot + "_ptcls " else: e2bdb_command += "bdb:" + os.path.join( original_out_dir, mic_baseroot + "_ptcls ") e2bdb_command += " --appendvstack=bdb:%s/data 1>/dev/null" % original_out_dir cmdexecute(e2bdb_command, printing_on_success=False) print("Done!\n") if RUNNING_UNDER_MPI: mpi_barrier(MPI_COMM_WORLD) from mpi import mpi_finalize mpi_finalize() sys.stdout.flush() sys.exit(0)
def main(args): # parser1 = argparse.ArgumentParser(description='This program is used to window particles from a micrograph. The coordinates of the particles are given as input.') # parser1.add_argument() progname = os.path.basename(sys.argv[0]) usage = progname + " micrographs_list --coords_dir=coords_dir --coords_suffix=coords_suffix" + \ " --coords_extension=coords_extension --coords_format=coords_format" + \ " --indir=input_dir --importctf=ctf_file --limitctf" + \ " --resample_ratio=resample_ratio --box_size=box_size" + \ " --outdir=outdir --outsuffix=outsuffix --micsuffix=micsuffix" + \ " --nameroot=nameroot --invert" + \ " --defocuserror=defocuserror --astigmatismerror=astigmatismerror" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--coords_dir", type="string", default=".", help= "<Coordinates Directory> Directory containing files with particle coordinates. (Default: current directory)" ) parser.add_option( "--coords_suffix", type="string", default="", help= "<Coordinates File Suffix> Suffix of coordinate files. For example '_ptcls'. " ) parser.add_option( "--coords_extension", type="string", default="box", help= "<Coordinates File Extension> File extension of coordinate files. e.g 'box' for eman1, 'json' for eman2, ..." ) parser.add_option( "--coords_format", type="string", default="eman1", help= "<Coordinates File Format> Format of coordinates file: 'sparx', 'eman1', 'eman2', or 'spider'. The coordinates of sparx, eman2, and spider format is particle center. The coordinates of eman1 format is particle box conner associated with the original box size." ) parser.add_option( "--indir", type="string", default=".", help= "<Micrograph Directory> Directory containing micrographs to be processed. (Default: current directory)" ) parser.add_option( "--nameroot", type="string", default="", help= "<Micrograph Root Name> Root name (Prefix) of micrographs to be processed." ) parser.add_option( "--micsuffix", type="string", default="hdf", help= "<Micrograph Extension > A string denoting micrograph type. (Default 'hdf')" ) parser.add_option( "--outdir", type="string", default=".", help="<Output Directory> Output directory (Default: current directory)" ) parser.add_option( "--outsuffix", type="string", default="_ptcls", help="<Output File Suffix> Suffix for output stack. (Default '_ptcls')" ) parser.add_option( "--importctf", type="string", default="", help="<CTER CTF File> File name with CTF parameters produced by sxcter." ) parser.add_option( "--box_size", type="int", default=256, help= "<Box Size> x and y dimension in pixels of square area to be windowed. Pixel size after resampling is assumed when resample_ratio < 1.0 (Default 256)" ) parser.add_option( "--invert", action="store_true", default=False, help= "<Invert Contrast> Invert image contrast (recommended for cryo data) (Default, no contrast inversion)" ) parser.add_option( "--resample_ratio", type="float", default=1.0, help= "<Resample Ratio> Ratio of new to old image size (or old to new pixel size) for resampling. Valid range is 0.0 < resample_ratio <= 1.0. (Default: 1.0) (advanced)" ) parser.add_option( "--limitctf", action="store_true", default=False, help= "<Apply CTF-Limit Filter> Filter micrographs based on the CTF limit. It requires --importctf. (Default: no filter) (advanced)" ) parser.add_option( "--defocuserror", type="float", default=1000000.0, help= "<Defocus Error Limit> Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%. (Default: include all irrespective of error values.) (advanced)" ) parser.add_option( "--astigmatismerror", type="float", default=360.0, help= "<Astigmatism Error Limit> Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees. (Default: include all irrespective of error values.) (advanced)" ) # must be switched off in production # parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False) # # parser.add_option("--restart_section", type="string", default="", help="<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)") # parser.add_option("--stop_after_candidates", action="store_true", default=False, help="<stop_after_candidates> stops after the 'candidate_class_averages' section") # parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help=SUPPRESS_HELP) (options, args) = parser.parse_args(args) if options.return_options: return parser # Set local constants box_size = options.box_size box_half = box_size // 2 options.micsuffix = "." + options.micsuffix cterr = [options.defocuserror / 100.0, options.astigmatismerror] check_options(options, progname) extension_coord = options.coords_suffix + "." + options.coords_extension # Build micrograph basename list micnames = build_micnames(options, args) print_msg('Detected micrographs : %6d ...\n' % (len(micnames))) # If there is no micrographs, exit if len(micnames) == 0: print usage sys.exit() # Load CTFs n_reject_defocus_error = 0 if options.importctf != None: ctfs0 = read_text_row(options.importctf) print_msg('Detected CTF entries : %6d ...\n' % (len(ctfs0))) ctfs = {} for i in xrange(len(ctfs0)): ctf = ctfs0[i] basemic = baseroot(ctf[-1]) if (ctf[8] / ctf[0] > cterr[0]): print_msg( 'Defocus error %f exceeds the threshold. Micrograph %s rejected.\n' % (ctf[8] / ctf[0], basemic)) n_reject_defocus_error += 1 else: if (ctf[10] > cterr[1]): ctf[6] = 0.0 ctf[7] = 0.0 ctfs[basemic] = ctf print_msg('Rejected micrographs by defocus error : %6d ...\n' % (n_reject_defocus_error)) # Create circular 2D mask for ... mask = model_circle(box_size // 2, box_size, box_size) # Prepare loop variables n_micrographs_process = 0 n_micrographs_reject_no_micrograph = 0 n_micrographs_reject_no_coordinates = 0 n_micrographs_reject_no_cter_entry = 0 n_total_coordinates_detect = 0 n_total_coordinates_process = 0 n_total_coordinates_reject_out_of_boundary = 0 cutoffhistogram = [ ] #@ming compute the histogram for micrographs cut of by ctf limit. # Loop over micrographs for k in range(len(micnames)): # basename is name of micrograph minus the path and extension # Here, assuming micrograph and coordinates have the same file basename basename = micnames[k] f_mic = os.path.join(os.path.abspath(options.indir), basename + options.micsuffix) f_info = os.path.join(options.coords_dir, basename + extension_coord) # CHECKS: BEGIN # IF micrograph exists if not os.path.exists(f_mic): print_msg(' Cannot read %s. Skipping %s ...\n' % (f_mic, basename)) n_micrographs_reject_no_micrograph += 1 continue # IF coordinates file exists if not os.path.exists(f_info): print_msg(' Cannot read %s. Skipping %s ...\n' % (f_info, basename)) n_micrographs_reject_no_coordinates += 1 continue # IF micrograph is in CTER results if options.importctf != None: if basename not in ctfs: print_msg( ' Is not listed in CTER results, skipping %s...\n' % (basename)) n_micrographs_reject_no_cter_entry += 1 continue else: ctf = ctfs[basename] # CHECKS: END n_micrographs_process += 1 print_msg('\n') print_msg( 'Processing micrograph %s... Path: %s... Coordinates file %s\n' % (basename, f_mic, f_info)) # Read coordinates according to the specified format and # make the coordinates the center of particle image if options.coords_format.lower() == 'sparx': coords = read_text_row(f_info) elif options.coords_format.lower() == 'eman1': coords = read_text_row(f_info) for i in range(len(coords)): coords[i] = [ coords[i][0] + coords[i][2] // 2, coords[i][1] + coords[i][3] // 2 ] elif options.coords_format.lower() == 'eman2': coords = js_open_dict(f_info)["boxes"] for i in range(len(coords)): coords[i] = [coords[i][0], coords[i][1]] elif options.coords_format.lower() == 'spider': coords = read_text_row(f_info) for i in range(len(coords)): coords[i] = [coords[i][2], coords[i][3]] else: assert (False) # Unreachable code # Load micrograph from the file immic = get_im(f_mic) # Calculate the new pixel size resample_ratio = options.resample_ratio if options.importctf != None: pixel_size_orig = ctf[3] if resample_ratio < 1.0: assert (resample_ratio > 0.0) new_pixel_size = pixel_size_orig / resample_ratio print_msg( 'Resample micrograph to pixel size %6.4f and window segments from resampled micrograph\n' % new_pixel_size) else: # assert(resample_ratio == 1.0) new_pixel_size = pixel_size_orig # Set ctf along with new pixel size in resampled micrograph ctf[3] = new_pixel_size else: assert (options.importctf == None) if resample_ratio < 1.0: assert (resample_ratio > 0.0) print_msg( 'Resample micrograph with ratio %6.4f and window segments from resampled micrograph\n' % resample_ratio) # else: # assert(resample_ratio == 1.0) # Apply filters to micrograph fftip(immic) if options.limitctf: assert (options.importctf != None) # Cut off frequency components higher than CTF limit q1, q2 = ctflimit(box_size, ctf[0], ctf[1], ctf[2], new_pixel_size) # This is absolute frequency of the CTF limit in the scale of original micrograph if resample_ratio < 1.0: assert (resample_ratio > 0.0) q1 = resample_ratio * q1 / float( box_size ) # q1 = (pixel_size_orig / new_pixel_size) * q1/float(box_size) else: # assert(resample_ratio == 1.0) -> pixel_size_orig == new_pixel_size -> pixel_size_orig / new_pixel_size == 1.0 q1 = q1 / float(box_size) if q1 < 0.5: #@ming immic = filt_tanl(immic, q1, 0.01) cutoffhistogram.append(q1) # Cut off frequency components lower than the box size can express immic = fft(filt_gaussh(immic, resample_ratio / box_size)) # Resample micrograph, map coordinates, and window segments from resampled micrograph using new coordinates # after resampling by resample_ratio, new pixel size will be pixel_size/resample_ratio = new_pixel_size # NOTE: 2015/04/13 Toshio Moriya # resample() efficiently takes care of the case resample_ratio = 1.0 but # it does not set apix_*. Even though it sets apix_* when resample_ratio < 1.0 ... immic = resample(immic, resample_ratio) if options.invert: stt = Util.infomask(immic, None, True) Util.mul_scalar(immic, -1.0) immic += 2 * stt[0] if options.importctf != None: from utilities import generate_ctf ctf = generate_ctf(ctf) # Prepare loop variables nx = immic.get_xsize() ny = immic.get_ysize() x0 = nx // 2 y0 = ny // 2 print_msg('\n') print_msg('Micrograph size := (%6d, %6d)\n' % (nx, ny)) otcl_images = "bdb:%s/" % options.outdir + basename + options.outsuffix ind = 0 n_coordinates_reject_out_of_boundary = 0 # Loop over coordinates for i in range(len(coords)): source_x = int(coords[i][0]) source_y = int(coords[i][1]) x = source_x y = source_y if resample_ratio < 1.0: assert (resample_ratio > 0.0) x = int(x * resample_ratio) y = int(y * resample_ratio) # else: # assert(resample_ratio == 1.0) if ((0 <= x - box_half) and (x + box_half <= nx) and (0 <= y - box_half) and (y + box_half <= ny)): imw = Util.window(immic, box_size, box_size, 1, x - x0, y - y0) else: print_msg( 'Coordinates ID = %04d (x = %4d, y = %4d, box_size = %4d) is out of micrograph bound, skipping ....\n' % (i, x, y, box_size)) n_coordinates_reject_out_of_boundary += 1 continue imw = ramp(imw) stat = Util.infomask(imw, mask, False) imw -= stat[0] imw /= stat[1] # NOTE: 2015/04/09 Toshio Moriya # ptcl_source_image might be redundant information ... # Consider re-organizing header entries... imw.set_attr("ptcl_source_image", f_mic) imw.set_attr("ptcl_source_coord_id", i) imw.set_attr("ptcl_source_coord", [source_x, source_y]) imw.set_attr("resample_ratio", resample_ratio) # NOTE: 2015/04/13 Toshio Moriya # apix_* attributes are updated by resample() only when resample_ratio != 1.0 # Let's make sure header info is consistent by setting apix_* = 1.0 # regardless of options, so it is not passed down the processing line imw.set_attr("apix_x", 1.0) imw.set_attr("apix_y", 1.0) imw.set_attr("apix_z", 1.0) if options.importctf != None: imw.set_attr("ctf", ctf) imw.set_attr("ctf_applied", 0) imw.set_attr("pixel_size_orig", pixel_size_orig) # imw.set_attr("apix_x", new_pixel_size) # imw.set_attr("apix_y", new_pixel_size) # imw.set_attr("apix_z", new_pixel_size) # NOTE: 2015/04/13 Toshio Moriya # Pawel Comment: Micrograph is not supposed to have CTF header info. # So, let's assume it does not exist & ignore its presence. # Note that resample() "correctly" updates pixel size of CTF header info if it exists # elif (imw.has_ctff()): # assert(options.importctf == None) # ctf_origin = imw.get_attr("ctf") # pixel_size_origin = round(ctf_origin.apix, 5) # Because SXCTER ouputs up to 5 digits # imw.set_attr("apix_x",pixel_size_origin) # imw.set_attr("apix_y",pixel_size_origin) # imw.set_attr("apix_z",pixel_size_origin) imw.write_image(otcl_images, ind) ind += 1 n_total_coordinates_detect += len(coords) n_total_coordinates_process += ind n_total_coordinates_reject_out_of_boundary += n_coordinates_reject_out_of_boundary # Print out the summary of this micrograph print_msg('\n') print_msg('Micrograph summary of coordinates...\n') print_msg('Detected : %4d\n' % (len(coords))) print_msg('Processed : %4d\n' % (ind)) print_msg('Rejected by out of boundary : %4d\n' % (n_coordinates_reject_out_of_boundary)) if options.limitctf: # Print out the summary of CTF-limit filtering print_msg('\n') print_msg('Global summary of CTF-limit filtering (--limitctf) ...\n') print_msg('Percentage of filtered micrographs: %8.2f\n' % (len(cutoffhistogram) * 100.0 / len(micnames))) lhist = 10 if len(cutoffhistogram) >= lhist: from statistics import hist_list region, hist = hist_list(cutoffhistogram, lhist) print_msg(" Histogram of cut off frequency\n") print_msg(" ERROR number of frequencies\n") for lhx in xrange(lhist): print_msg( " %14.7f %7d\n" % (region[lhx], hist[lhx]) ) # print_msg(" %10.3f %7d\n" % (region[lhx], hist[lhx])) else: print_msg( "The number of filtered micrographs (%d) is less than the number of bins (%d). No histogram is produced.\n" % (len(cutoffhistogram), lhist)) # Print out the summary of all micrographs print_msg('\n') print_msg('Global summary of micrographs ...\n') print_msg('Detected : %6d\n' % (len(micnames))) print_msg('Processed : %6d\n' % (n_micrographs_process)) print_msg('Rejected by no micrograph file : %6d\n' % (n_micrographs_reject_no_micrograph)) print_msg('Rejected by no coordinates file : %6d\n' % (n_micrographs_reject_no_coordinates)) print_msg('Rejected by no CTER entry : %6d\n' % (n_micrographs_reject_no_cter_entry)) print_msg('\n') print_msg('Global summary of coordinates ...\n') print_msg('Detected : %6d\n' % (n_total_coordinates_detect)) print_msg('Processed : %6d\n' % (n_total_coordinates_process)) print_msg('Rejected by out of boundary : %6d\n' % (n_total_coordinates_reject_out_of_boundary)) print_msg('\n')
def main(): def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror): if mirror: m = 1 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0 - psi, 0, 0, 1.0) else: m = 0 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0 - psi, 0, 0, 1.0) return alpha, sx, sy, m progname = os.path.basename(sys.argv[0]) usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=0.2 --aa=0.1 --sym=symmetry --CTF" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--ave2D", type="string", default=False, help="write to the disk a stack of 2D averages") parser.add_option("--var2D", type="string", default=False, help="write to the disk a stack of 2D variances") parser.add_option("--ave3D", type="string", default=False, help="write to the disk reconstructed 3D average") parser.add_option("--var3D", type="string", default=False, help="compute 3D variability (time consuming!)") parser.add_option("--img_per_grp", type="int", default=10, help="number of neighbouring projections") parser.add_option("--no_norm", action="store_true", default=False, help="do not use normalization") parser.add_option("--radiusvar", type="int", default=-1, help="radius for 3D var") parser.add_option("--npad", type="int", default=2, help="number of time to pad the original images") parser.add_option("--sym", type="string", default="c1", help="symmetry") parser.add_option("--fl", type="float", default=0.0, help="stop-band frequency (Default - no filtration)") parser.add_option("--aa", type="float", default=0.0, help="fall off of the filter (Default - no filtration)") parser.add_option("--CTF", action="store_true", default=False, help="use CFT correction") parser.add_option("--VERBOSE", action="store_true", default=False, help="Long output for debugging") #parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version") #parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" ) #parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" ) #parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" ) #parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" ) parser.add_option( "--VAR", action="store_true", default=False, help="stack on input consists of 2D variances (Default False)") parser.add_option( "--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)") #parser.add_option("--nvec", type="int" , default=0 , help="number of eigenvectors, default = 0 meaning no PCA calculated") parser.add_option( "--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)") (options, args) = parser.parse_args() from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD, MPI_TAG_UB from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX from applications import MPI_start_end from reconstruction import recons3d_em, recons3d_em_MPI from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI from utilities import print_begin_msg, print_end_msg, print_msg from utilities import read_text_row, get_image, get_im from utilities import bcast_EMData_to_all, bcast_number_to_all from utilities import get_symt # This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015 from EMAN2db import db_open_dict if options.symmetrize: try: sys.argv = mpi_init(len(sys.argv), sys.argv) try: number_of_proc = mpi_comm_size(MPI_COMM_WORLD) if (number_of_proc > 1): ERROR( "Cannot use more than one CPU for symmetry prepration", "sx3dvariability", 1) except: pass except: pass # Input #instack = "Clean_NORM_CTF_start_wparams.hdf" #instack = "bdb:data" instack = args[0] sym = options.sym if (sym == "c1"): ERROR("Thre is no need to symmetrize stack for C1 symmetry", "sx3dvariability", 1) if (instack[:4] != "bdb:"): stack = "bdb:data" delete_bdb(stack) cmdexecute("sxcpy.py " + instack + " " + stack) else: stack = instack qt = EMUtil.get_all_attributes(stack, 'xform.projection') na = len(qt) ts = get_symt(sym) ks = len(ts) angsa = [None] * na for k in xrange(ks): delete_bdb("bdb:Q%1d" % k) cmdexecute("e2bdb.py " + stack + " --makevstack=bdb:Q%1d" % k) DB = db_open_dict("bdb:Q%1d" % k) for i in xrange(na): ut = qt[i] * ts[k] DB.set_attr(i, "xform.projection", ut) #bt = ut.get_params("spider") #angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]] #write_text_row(angsa, 'ptsma%1d.txt'%k) #cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k) #cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k)) DB.close() delete_bdb("bdb:sdata") cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q") #cmdexecute("ls EMAN2DB/sdata*") a = get_im("bdb:sdata") a.set_attr("variabilitysymmetry", sym) a.write_image("bdb:sdata") else: sys.argv = mpi_init(len(sys.argv), sys.argv) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 1: stack = args[0] else: print("usage: " + usage) print("Please run '" + progname + " -h' for detailed options") return 1 t0 = time() # obsolete flags options.MPI = True options.nvec = 0 options.radiuspca = -1 options.iter = 40 options.abs = 0.0 options.squ = 0.0 if options.fl > 0.0 and options.aa == 0.0: ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid) if options.VAR and options.SND: ERROR("Only one of var and SND can be set!", "sx3dvariability", myid) exit() if options.VAR and (options.ave2D or options.ave3D or options.var2D): ERROR( "When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid) exit() if options.SND and (options.ave2D or options.ave3D): ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid) exit() if options.nvec > 0: ERROR("PCA option not implemented", "sx3dvariability", 1, myid) exit() #if options.nvec > 0 and options.ave3D == None: # ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid) # exit() import string options.sym = options.sym.lower() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True if myid == main_node: print_begin_msg("sx3dvariability") print_msg("%-70s: %s\n" % ("Input stack", stack)) img_per_grp = options.img_per_grp nvec = options.nvec radiuspca = options.radiuspca symbaselen = 0 if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() if options.sym != "c1": imgdata = get_im(stack) try: i = imgdata.get_attr("variabilitysymmetry") if (i != options.sym): ERROR( "The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", myid=myid) except: ERROR( "Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", myid=myid) from utilities import get_symt i = len(get_symt(options.sym)) if ((nima / i) * i != nima): ERROR( "The length of the input stack is incorrect for symmetry processing", "sx3dvariability", myid=myid) symbaselen = nima / i else: symbaselen = nima else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) symbaselen = bcast_number_to_all(symbaselen) if radiuspca == -1: radiuspca = nx / 2 - 2 if myid == main_node: print_msg("%-70s: %d\n" % ("Number of projection", nima)) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) if options.SND: from projection import prep_vol, prgs from statistics import im_diff from utilities import get_im, model_circle, get_params_proj, set_params_proj from utilities import get_ctf, generate_ctf from filter import filt_ctf imgdata = EMData.read_images(stack, range(img_begin, img_end)) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) bcast_EMData_to_all(vol, myid) volft, kb = prep_vol(vol) mask = model_circle(nx / 2 - 2, nx, ny) varList = [] for i in xrange(img_begin, img_end): phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i - img_begin]) ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y]) if options.CTF: ctf_params = get_ctf(imgdata[i - img_begin]) ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params)) diff, A, B = im_diff(ref_prj, imgdata[i - img_begin], mask) diff2 = diff * diff set_params_proj(diff2, [phi, theta, psi, s2x, s2y]) varList.append(diff2) mpi_barrier(MPI_COMM_WORLD) elif options.VAR: varList = EMData.read_images(stack, range(img_begin, img_end)) else: from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2 from utilities import model_blank, nearest_proj, model_circle from applications import pca from statistics import avgvar, avgvar_ctf, ccc from filter import filt_tanl from morphology import threshold, square_root from projection import project, prep_vol, prgs from sets import Set if myid == main_node: t1 = time() proj_angles = [] aveList = [] tab = EMUtil.get_all_attributes(stack, 'xform.projection') for i in xrange(nima): t = tab[i].get_params('spider') phi = t['phi'] theta = t['theta'] psi = t['psi'] x = theta if x > 90.0: x = 180.0 - x x = x * 10000 + psi proj_angles.append([x, t['phi'], t['theta'], t['psi'], i]) t2 = time() print_msg("%-70s: %d\n" % ("Number of neighboring projections", img_per_grp)) print_msg("...... Finding neighboring projections\n") if options.VERBOSE: print "Number of images per group: ", img_per_grp print "Now grouping projections" proj_angles.sort() proj_angles_list = [0.0] * (nima * 4) if myid == main_node: for i in xrange(nima): proj_angles_list[i * 4] = proj_angles[i][1] proj_angles_list[i * 4 + 1] = proj_angles[i][2] proj_angles_list[i * 4 + 2] = proj_angles[i][3] proj_angles_list[i * 4 + 3] = proj_angles[i][4] proj_angles_list = bcast_list_to_all(proj_angles_list, myid, main_node) proj_angles = [] for i in xrange(nima): proj_angles.append([ proj_angles_list[i * 4], proj_angles_list[i * 4 + 1], proj_angles_list[i * 4 + 2], int(proj_angles_list[i * 4 + 3]) ]) del proj_angles_list proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end)) all_proj = Set() for im in proj_list: for jm in im: all_proj.add(proj_angles[jm][3]) all_proj = list(all_proj) if options.VERBOSE: print "On node %2d, number of images needed to be read = %5d" % ( myid, len(all_proj)) index = {} for i in xrange(len(all_proj)): index[all_proj[i]] = i mpi_barrier(MPI_COMM_WORLD) if myid == main_node: print_msg("%-70s: %.2f\n" % ("Finding neighboring projections lasted [s]", time() - t2)) print_msg("%-70s: %d\n" % ("Number of groups processed on the main node", len(proj_list))) if options.VERBOSE: print "Grouping projections took: ", (time() - t2) / 60, "[min]" print "Number of groups on main node: ", len(proj_list) mpi_barrier(MPI_COMM_WORLD) if myid == main_node: print_msg("...... calculating the stack of 2D variances \n") if options.VERBOSE: print "Now calculating the stack of 2D variances" proj_params = [0.0] * (nima * 5) aveList = [] varList = [] if nvec > 0: eigList = [[] for i in xrange(nvec)] if options.VERBOSE: print "Begin to read images on processor %d" % (myid) ttt = time() imgdata = EMData.read_images(stack, all_proj) if options.VERBOSE: print "Reading images on processor %d done, time = %.2f" % ( myid, time() - ttt) print "On processor %d, we got %d images" % (myid, len(imgdata)) mpi_barrier(MPI_COMM_WORLD) ''' imgdata2 = EMData.read_images(stack, range(img_begin, img_end)) if options.fl > 0.0: for k in xrange(len(imgdata2)): imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) if myid == main_node: vol.write_image("vol_ctf.hdf") print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf")) del vol, imgdata2 mpi_barrier(MPI_COMM_WORLD) ''' from applications import prepare_2d_forPCA from utilities import model_blank for i in xrange(len(proj_list)): ki = proj_angles[proj_list[i][0]][3] if ki >= symbaselen: continue mi = index[ki] phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi]) grp_imgdata = [] for j in xrange(img_per_grp): mj = index[proj_angles[proj_list[i][j]][3]] phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj]) alpha, sx, sy, mirror = params_3D_2D_NEW( phi, theta, psi, s2x, s2y, mirror_list[i][j]) if thetaM <= 90: if mirror == 0: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, phiM - phi, 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, 180 - (phiM - phi), 0.0, 0.0, 1.0) else: if mirror == 0: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, -(phiM - phi), 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, -(180 - (phiM - phi)), 0.0, 0.0, 1.0) set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0]) grp_imgdata.append(imgdata[mj]) if not options.no_norm: mask = model_circle(nx / 2 - 2, nx, nx) for k in xrange(img_per_grp): ave, std, minn, maxx = Util.infomask( grp_imgdata[k], mask, False) grp_imgdata[k] -= ave grp_imgdata[k] /= std del mask if options.fl > 0.0: from filter import filt_ctf, filt_table from fundamentals import fft, window2d nx2 = 2 * nx ny2 = 2 * ny if options.CTF: from utilities import pad for k in xrange(img_per_grp): grp_imgdata[k] = window2d( fft( filt_tanl( filt_ctf( fft( pad(grp_imgdata[k], nx2, ny2, 1, 0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa)), nx, ny) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) else: for k in xrange(img_per_grp): grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) else: from utilities import pad, read_text_file from filter import filt_ctf, filt_table from fundamentals import fft, window2d nx2 = 2 * nx ny2 = 2 * ny if options.CTF: from utilities import pad for k in xrange(img_per_grp): grp_imgdata[k] = window2d( fft( filt_ctf(fft( pad(grp_imgdata[k], nx2, ny2, 1, 0.0)), grp_imgdata[k].get_attr("ctf"), binary=1)), nx, ny) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) ''' if i < 10 and myid == main_node: for k in xrange(10): grp_imgdata[k].write_image("grp%03d.hdf"%i, k) ''' """ if myid == main_node and i==0: for pp in xrange(len(grp_imgdata)): grp_imgdata[pp].write_image("pp.hdf", pp) """ ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata) """ if myid == main_node and i==0: for pp in xrange(len(grp_imgdata)): grp_imgdata[pp].write_image("qq.hdf", pp) """ var = model_blank(nx, ny) for q in grp_imgdata: Util.add_img2(var, q) Util.mul_scalar(var, 1.0 / (len(grp_imgdata) - 1)) # Switch to std dev var = square_root(threshold(var)) #if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a") #else: ave, var = avgvar(grp_imgdata, mode="a") """ if myid == main_node: ave.write_image("avgv.hdf",i) var.write_image("varv.hdf",i) """ set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0]) set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0]) aveList.append(ave) varList.append(var) if options.VERBOSE: print "%5.2f%% done on processor %d" % ( i * 100.0 / len(proj_list), myid) if nvec > 0: eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True) for k in xrange(nvec): set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0]) eigList[k].append(eig[k]) """ if myid == 0 and i == 0: for k in xrange(nvec): eig[k].write_image("eig.hdf", k) """ del imgdata # To this point, all averages, variances, and eigenvectors are computed if options.ave2D: if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node: for im in xrange(len(aveList)): aveList[im].write_image(options.ave2D, km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im + i + 70000) """ nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('members', map(int, members)) members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('pix_err', map(float, members)) members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('refprojdir', map(float, members)) """ ave.write_image(options.ave2D, km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) for im in xrange(len(aveList)): send_EMData(aveList[im], main_node, im + myid + 70000) """ members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) members = aveList[im].get_attr('pix_err') mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) except: mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) """ if options.ave3D: if options.VERBOSE: print "Reconstructing 3D average volume" ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(ave3D, myid) if myid == main_node: ave3D.write_image(options.ave3D) print_msg("%-70s: %s\n" % ( "Writing to the disk volume reconstructed from averages as", options.ave3D)) del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList if nvec > 0: for k in xrange(nvec): if options.VERBOSE: print "Reconstruction eigenvolumes", k cont = True ITER = 0 mask2d = model_circle(radiuspca, nx, nx) while cont: #print "On node %d, iteration %d"%(myid, ITER) eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(eig3D, myid, main_node) if options.fl > 0.0: eig3D = filt_tanl(eig3D, options.fl, options.aa) if myid == main_node: eig3D.write_image("eig3d_%03d.hdf" % k, ITER) Util.mul_img(eig3D, model_circle(radiuspca, nx, nx, nx)) eig3Df, kb = prep_vol(eig3D) del eig3D cont = False icont = 0 for l in xrange(len(eigList[k])): phi, theta, psi, s2x, s2y = get_params_proj( eigList[k][l]) proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y]) cl = ccc(proj, eigList[k][l], mask2d) if cl < 0.0: icont += 1 cont = True eigList[k][l] *= -1.0 u = int(cont) u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD) icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: u = int(u[0]) print " Eigenvector: ", k, " number changed ", int( icont[0]) else: u = 0 u = bcast_number_to_all(u, main_node) cont = bool(u) ITER += 1 del eig3Df, kb mpi_barrier(MPI_COMM_WORLD) del eigList, mask2d if options.ave3D: del ave3D if options.var2D: if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node: for im in xrange(len(varList)): varList[im].write_image(options.var2D, km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im + i + 70000) ave.write_image(options.var2D, km) km += 1 else: mpi_send(len(varList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) for im in xrange(len(varList)): send_EMData(varList[im], main_node, im + myid + 70000) # What with the attributes?? mpi_barrier(MPI_COMM_WORLD) if options.var3D: if myid == main_node and options.VERBOSE: print "Reconstructing 3D variability volume" t6 = time() radiusvar = options.radiusvar if (radiusvar < 0): radiusvar = nx // 2 - 3 res = recons3d_4nn_MPI(myid, varList, symmetry=options.sym, npad=options.npad) #res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ) if myid == main_node: res.write_image(options.var3D) if myid == main_node: print_msg( "%-70s: %.2f\n" % ("Reconstructing 3D variability took [s]", time() - t6)) if options.VERBOSE: print "Reconstruction took: %.2f [min]" % ( (time() - t6) / 60) if myid == main_node: print_msg( "%-70s: %.2f\n" % ("Total time for these computations [s]", time() - t0)) if options.VERBOSE: print "Total time for these computations: %.2f [min]" % ( (time() - t0) / 60) print_end_msg("sx3dvariability") global_def.BATCH = False from mpi import mpi_finalize mpi_finalize()
def main(args): # parser1 = argparse.ArgumentParser(description='This program is used to window particles from a micrograph. The coordinates of the particles are given as input.') # parser1.add_argument() progname = os.path.basename(sys.argv[0]) usage = progname + " micrographs_list --coords_dir=coords_dir --coords_suffix=coords_suffix" + \ " --coords_extension=coords_extension --coords_format=coords_format" + \ " --indir=input_dir --importctf=ctf_file --limitctf" + \ " --resample_ratio=resample_ratio --box_size=box_size" + \ " --outdir=outdir --outsuffix=outsuffix --micsuffix=micsuffix" + \ " --nameroot=nameroot --invert" + \ " --defocuserror=defocuserror --astigmatismerror=astigmatismerror" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--coords_dir", type="string", default=".", help="<Coordinates Directory> Directory containing files with particle coordinates. (Default: current directory)") parser.add_option("--coords_suffix", type="string", default="", help="<Coordinates File Suffix> Suffix of coordinate files. For example '_ptcls'. ") parser.add_option("--coords_extension", type="string", default="box", help="<Coordinates File Extension> File extension of coordinate files. e.g 'box' for eman1, 'json' for eman2, ...") parser.add_option("--coords_format", type="string", default="eman1", help="<Coordinates File Format> Format of coordinates file: 'sparx', 'eman1', 'eman2', or 'spider'. The coordinates of sparx, eman2, and spider format is particle center. The coordinates of eman1 format is particle box conner associated with the original box size.") parser.add_option("--indir", type="string", default=".", help="<Micrograph Directory> Directory containing micrographs to be processed. (Default: current directory)") parser.add_option("--nameroot", type="string", default="", help="<Micrograph Root Name> Root name (Prefix) of micrographs to be processed.") parser.add_option("--micsuffix", type="string", default="hdf", help="<Micrograph Extension > A string denoting micrograph type. (Default 'hdf')") parser.add_option("--outdir", type="string", default=".", help="<Output Directory> Output directory (Default: current directory)") parser.add_option("--outsuffix", type="string", default="_ptcls", help="<Output File Suffix> Suffix for output stack. (Default '_ptcls')") parser.add_option("--importctf", type="string", default="", help="<CTER CTF File> File name with CTF parameters produced by sxcter.") parser.add_option("--box_size", type="int", default=256, help="<Box Size> x and y dimension in pixels of square area to be windowed. Pixel size after resampling is assumed when resample_ratio < 1.0 (Default 256)") parser.add_option("--invert", action="store_true", default=False, help="<Invert Contrast> Invert image contrast (recommended for cryo data) (Default, no contrast inversion)") parser.add_option("--resample_ratio", type="float", default=1.0, help="<Resample Ratio> Ratio of new to old image size (or old to new pixel size) for resampling. Valid range is 0.0 < resample_ratio <= 1.0. (Default: 1.0) (advanced)") parser.add_option("--limitctf", action="store_true", default=False, help="<Apply CTF-Limit Filter> Filter micrographs based on the CTF limit. It requires --importctf. (Default: no filter) (advanced)") parser.add_option("--defocuserror", type="float", default=1000000.0, help="<Defocus Error Limit> Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%. (Default: include all irrespective of error values.) (advanced)" ) parser.add_option("--astigmatismerror", type="float", default=360.0, help="<Astigmatism Error Limit> Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees. (Default: include all irrespective of error values.) (advanced)") # must be switched off in production # parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False) # # parser.add_option("--restart_section", type="string", default="", help="<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)") # parser.add_option("--stop_after_candidates", action="store_true", default=False, help="<stop_after_candidates> stops after the 'candidate_class_averages' section") # parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP) (options, args) = parser.parse_args(args) if options.return_options: return parser # Set local constants box_size = options.box_size box_half = box_size // 2 options.micsuffix = "." + options.micsuffix cterr = [options.defocuserror/100.0, options.astigmatismerror] check_options(options, progname) extension_coord = options.coords_suffix + "." + options.coords_extension # Build micrograph basename list micnames = build_micnames(options, args) print_msg('Detected micrographs : %6d ...\n' % (len(micnames))) # If there is no micrographs, exit if len(micnames) == 0: print usage sys.exit() # Load CTFs n_reject_defocus_error = 0 if options.importctf != None: ctfs0 = read_text_row(options.importctf) print_msg('Detected CTF entries : %6d ...\n' % (len(ctfs0))) ctfs={} for i in xrange(len(ctfs0)): ctf=ctfs0[i] basemic = baseroot(ctf[-1]) if(ctf[8]/ctf[0] > cterr[0]): print_msg('Defocus error %f exceeds the threshold. Micrograph %s rejected.\n' % (ctf[8]/ctf[0], basemic)) n_reject_defocus_error += 1 else: if(ctf[10] > cterr[1] ): ctf[6] = 0.0 ctf[7] = 0.0 ctfs[basemic] = ctf print_msg('Rejected micrographs by defocus error : %6d ...\n' % (n_reject_defocus_error)) # Create circular 2D mask for ... mask = model_circle(box_size//2, box_size, box_size) # Prepare loop variables n_micrographs_process = 0 n_micrographs_reject_no_micrograph = 0 n_micrographs_reject_no_coordinates = 0 n_micrographs_reject_no_cter_entry = 0 n_total_coordinates_detect = 0 n_total_coordinates_process = 0 n_total_coordinates_reject_out_of_boundary = 0 cutoffhistogram = [] #@ming compute the histogram for micrographs cut of by ctf limit. # Loop over micrographs for k in range(len(micnames)): # basename is name of micrograph minus the path and extension # Here, assuming micrograph and coordinates have the same file basename basename = micnames[k] f_mic = os.path.join(os.path.abspath(options.indir), basename + options.micsuffix) f_info = os.path.join(options.coords_dir, basename + extension_coord) # CHECKS: BEGIN # IF micrograph exists if not os.path.exists(f_mic): print_msg(' Cannot read %s. Skipping %s ...\n' % (f_mic, basename)) n_micrographs_reject_no_micrograph += 1 continue # IF coordinates file exists if not os.path.exists(f_info): print_msg(' Cannot read %s. Skipping %s ...\n' % (f_info, basename)) n_micrographs_reject_no_coordinates += 1 continue # IF micrograph is in CTER results if options.importctf != None: if basename not in ctfs: print_msg(' Is not listed in CTER results, skipping %s...\n' % (basename)) n_micrographs_reject_no_cter_entry += 1 continue else: ctf = ctfs[basename] # CHECKS: END n_micrographs_process += 1 print_msg('\n') print_msg('Processing micrograph %s... Path: %s... Coordinates file %s\n' % (basename, f_mic, f_info)) # Read coordinates according to the specified format and # make the coordinates the center of particle image if options.coords_format.lower() == 'sparx' : coords = read_text_row(f_info) elif options.coords_format.lower() == 'eman1': coords = read_text_row(f_info) for i in range(len(coords)): coords[i] = [coords[i][0] + coords[i][2]//2 ,coords[i][1] + coords[i][3]//2] elif options.coords_format.lower() == 'eman2': coords = js_open_dict(f_info)["boxes"] for i in range(len(coords)): coords[i] = [coords[i][0],coords[i][1]] elif options.coords_format.lower() == 'spider': coords = read_text_row(f_info) for i in range(len(coords)): coords[i] = [coords[i][2] ,coords[i][3]] else: assert(False) # Unreachable code # Load micrograph from the file immic = get_im(f_mic) # Calculate the new pixel size resample_ratio = options.resample_ratio if options.importctf != None: pixel_size_orig = ctf[3] if resample_ratio < 1.0: assert(resample_ratio > 0.0) new_pixel_size = pixel_size_orig / resample_ratio print_msg('Resample micrograph to pixel size %6.4f and window segments from resampled micrograph\n' % new_pixel_size) else: # assert(resample_ratio == 1.0) new_pixel_size = pixel_size_orig # Set ctf along with new pixel size in resampled micrograph ctf[3] = new_pixel_size else: assert(options.importctf == None) if resample_ratio < 1.0: assert(resample_ratio > 0.0) print_msg('Resample micrograph with ratio %6.4f and window segments from resampled micrograph\n' % resample_ratio) # else: # assert(resample_ratio == 1.0) # Apply filters to micrograph fftip(immic) if options.limitctf: assert(options.importctf != None) # Cut off frequency components higher than CTF limit q1, q2 = ctflimit(box_size,ctf[0],ctf[1],ctf[2],new_pixel_size) # This is absolute frequency of the CTF limit in the scale of original micrograph if resample_ratio < 1.0: assert(resample_ratio > 0.0) q1 = resample_ratio * q1 / float(box_size) # q1 = (pixel_size_orig / new_pixel_size) * q1/float(box_size) else: # assert(resample_ratio == 1.0) -> pixel_size_orig == new_pixel_size -> pixel_size_orig / new_pixel_size == 1.0 q1 = q1 / float(box_size) if q1 < 0.5: #@ming immic = filt_tanl(immic, q1, 0.01) cutoffhistogram.append(q1) # Cut off frequency components lower than the box size can express immic = fft(filt_gaussh( immic, resample_ratio/box_size )) # Resample micrograph, map coordinates, and window segments from resampled micrograph using new coordinates # after resampling by resample_ratio, new pixel size will be pixel_size/resample_ratio = new_pixel_size # NOTE: 2015/04/13 Toshio Moriya # resample() efficiently takes care of the case resample_ratio = 1.0 but # it does not set apix_*. Even though it sets apix_* when resample_ratio < 1.0 ... immic = resample(immic, resample_ratio) if options.invert: stt = Util.infomask(immic, None, True) Util.mul_scalar(immic, -1.0) immic += 2*stt[0] if options.importctf != None: from utilities import generate_ctf ctf = generate_ctf(ctf) # Prepare loop variables nx = immic.get_xsize() ny = immic.get_ysize() x0 = nx//2 y0 = ny//2 print_msg('\n') print_msg('Micrograph size := (%6d, %6d)\n' % (nx, ny)) otcl_images = "bdb:%s/" % options.outdir + basename + options.outsuffix ind = 0 n_coordinates_reject_out_of_boundary = 0 # Loop over coordinates for i in range(len(coords)): source_x = int(coords[i][0]) source_y = int(coords[i][1]) x = source_x y = source_y if resample_ratio < 1.0: assert(resample_ratio > 0.0) x = int(x * resample_ratio) y = int(y * resample_ratio) # else: # assert(resample_ratio == 1.0) if( (0 <= x - box_half) and ( x + box_half <= nx ) and (0 <= y - box_half) and ( y + box_half <= ny ) ): imw = Util.window(immic, box_size, box_size, 1, x-x0, y-y0) else: print_msg('Coordinates ID = %04d (x = %4d, y = %4d, box_size = %4d) is out of micrograph bound, skipping ....\n' % (i, x, y, box_size)) n_coordinates_reject_out_of_boundary += 1 continue imw = ramp(imw) stat = Util.infomask( imw, mask, False ) imw -= stat[0] imw /= stat[1] # NOTE: 2015/04/09 Toshio Moriya # ptcl_source_image might be redundant information ... # Consider re-organizing header entries... imw.set_attr("ptcl_source_image", f_mic) imw.set_attr("ptcl_source_coord_id", i) imw.set_attr("ptcl_source_coord", [source_x, source_y]) imw.set_attr("resample_ratio", resample_ratio) # NOTE: 2015/04/13 Toshio Moriya # apix_* attributes are updated by resample() only when resample_ratio != 1.0 # Let's make sure header info is consistent by setting apix_* = 1.0 # regardless of options, so it is not passed down the processing line imw.set_attr("apix_x", 1.0) imw.set_attr("apix_y", 1.0) imw.set_attr("apix_z", 1.0) if options.importctf != None: imw.set_attr("ctf",ctf) imw.set_attr("ctf_applied", 0) imw.set_attr("pixel_size_orig", pixel_size_orig) # imw.set_attr("apix_x", new_pixel_size) # imw.set_attr("apix_y", new_pixel_size) # imw.set_attr("apix_z", new_pixel_size) # NOTE: 2015/04/13 Toshio Moriya # Pawel Comment: Micrograph is not supposed to have CTF header info. # So, let's assume it does not exist & ignore its presence. # Note that resample() "correctly" updates pixel size of CTF header info if it exists # elif (imw.has_ctff()): # assert(options.importctf == None) # ctf_origin = imw.get_attr("ctf") # pixel_size_origin = round(ctf_origin.apix, 5) # Because SXCTER ouputs up to 5 digits # imw.set_attr("apix_x",pixel_size_origin) # imw.set_attr("apix_y",pixel_size_origin) # imw.set_attr("apix_z",pixel_size_origin) imw.write_image(otcl_images, ind) ind += 1 n_total_coordinates_detect += len(coords) n_total_coordinates_process += ind n_total_coordinates_reject_out_of_boundary += n_coordinates_reject_out_of_boundary # Print out the summary of this micrograph print_msg('\n') print_msg('Micrograph summary of coordinates...\n') print_msg('Detected : %4d\n' % (len(coords))) print_msg('Processed : %4d\n' % (ind)) print_msg('Rejected by out of boundary : %4d\n' % (n_coordinates_reject_out_of_boundary)) if options.limitctf: # Print out the summary of CTF-limit filtering print_msg('\n') print_msg('Global summary of CTF-limit filtering (--limitctf) ...\n') print_msg('Percentage of filtered micrographs: %8.2f\n' % (len(cutoffhistogram) * 100.0 / len(micnames))) lhist = 10 if len(cutoffhistogram) >= lhist: from statistics import hist_list region,hist = hist_list(cutoffhistogram, lhist) print_msg(" Histogram of cut off frequency\n") print_msg(" ERROR number of frequencies\n") for lhx in xrange(lhist): print_msg(" %14.7f %7d\n" % (region[lhx], hist[lhx])) # print_msg(" %10.3f %7d\n" % (region[lhx], hist[lhx])) else: print_msg("The number of filtered micrographs (%d) is less than the number of bins (%d). No histogram is produced.\n" % (len(cutoffhistogram), lhist)) # Print out the summary of all micrographs print_msg('\n') print_msg('Global summary of micrographs ...\n') print_msg('Detected : %6d\n' % (len(micnames))) print_msg('Processed : %6d\n' % (n_micrographs_process)) print_msg('Rejected by no micrograph file : %6d\n' % (n_micrographs_reject_no_micrograph)) print_msg('Rejected by no coordinates file : %6d\n' % (n_micrographs_reject_no_coordinates)) print_msg('Rejected by no CTER entry : %6d\n' % (n_micrographs_reject_no_cter_entry)) print_msg('\n') print_msg('Global summary of coordinates ...\n') print_msg('Detected : %6d\n' % (n_total_coordinates_detect)) print_msg('Processed : %6d\n' % (n_total_coordinates_process)) print_msg('Rejected by out of boundary : %6d\n' % (n_total_coordinates_reject_out_of_boundary)) print_msg('\n')
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + """ input_micrograph_list_file input_micrograph_pattern input_coordinates_pattern output_directory --coordinates_format --box_size=box_size --invert --import_ctf=ctf_file --limit_ctf --resample_ratio=resample_ratio --defocus_error=defocus_error --astigmatism_error=astigmatism_error Window particles from micrographs in input list file. The coordinates of the particles should be given as input. Please specify name pattern of input micrographs and coordinates files with a wild card (*). Use the wild card to indicate the place of micrograph ID (e.g. serial number, time stamp, and etc). The name patterns must be enclosed by single quotes (') or double quotes ("). (Note: sxgui.py automatically adds single quotes (')). BDB files can not be selected as input micrographs. sxwindow.py mic_list.txt ./mic*.hdf info/mic*_info.json particles --coordinates_format=eman2 --box_size=64 --invert --import_ctf=outdir_cter/partres/partres.txt If micrograph list file name is not provided, all files matched with the micrograph name pattern will be processed. sxwindow.py ./mic*.hdf info/mic*_info.json particles --coordinates_format=eman2 --box_size=64 --invert --import_ctf=outdir_cter/partres/partres.txt """ parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--coordinates_format", type="string", default="eman1", help="format of input coordinates files: 'sparx', 'eman1', 'eman2', or 'spider'. the coordinates of sparx, eman2, and spider format is particle center. the coordinates of eman1 format is particle box conner associated with the original box size. (default eman1)") parser.add_option("--box_size", type="int", default=256, help="x and y dimension of square area to be windowed (in pixels): pixel size after resampling is assumed when resample_ratio < 1.0 (default 256)") parser.add_option("--invert", action="store_true", default=False, help="invert image contrast: recommended for cryo data (default False)") parser.add_option("--import_ctf", type="string", default="", help="file name of sxcter output: normally partres.txt (default none)") parser.add_option("--limit_ctf", action="store_true", default=False, help="filter micrographs based on the CTF limit: this option requires --import_ctf. (default False)") parser.add_option("--resample_ratio", type="float", default=1.0, help="ratio of new to old image size (or old to new pixel size) for resampling: Valid range is 0.0 < resample_ratio <= 1.0. (default 1.0)") parser.add_option("--defocus_error", type="float", default=1000000.0, help="defocus errror limit: exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocus_error percent. the error is computed as (std dev defocus)/defocus*100%. (default 1000000.0)" ) parser.add_option("--astigmatism_error", type="float", default=360.0, help="astigmatism error limit: Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatism_error degrees. (default 360.0)") ### detect if program is running under MPI RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ main_node = 0 if RUNNING_UNDER_MPI: from mpi import mpi_init from mpi import MPI_COMM_WORLD, mpi_comm_rank, mpi_comm_size, mpi_barrier, mpi_reduce, MPI_INT, MPI_SUM mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_processes = mpi_comm_size(MPI_COMM_WORLD) else: number_of_processes = 1 myid = 0 (options, args) = parser.parse_args(sys.argv[1:]) mic_list_file_path = None mic_pattern = None coords_pattern = None error_status = None while True: if len(args) < 3 or len(args) > 4: error_status = ("Please check usage for number of arguments.\n Usage: " + usage + "\n" + "Please run %s -h for help." % (progname), getframeinfo(currentframe())) break if len(args) == 3: mic_pattern = args[0] coords_pattern = args[1] out_dir = args[2] else: # assert(len(args) == 4) mic_list_file_path = args[0] mic_pattern = args[1] coords_pattern = args[2] out_dir = args[3] if mic_list_file_path != None: if os.path.splitext(mic_list_file_path)[1] != ".txt": error_status = ("Extension of input micrograph list file must be \".txt\". Please check input_micrograph_list_file argument. Run %s -h for help." % (progname), getframeinfo(currentframe())) break if mic_pattern[:len("bdb:")].lower() == "bdb": error_status = ("BDB file can not be selected as input micrographs. Please convert the format, and restart the program. Run %s -h for help." % (progname), getframeinfo(currentframe())) break if mic_pattern.find("*") == -1: error_status = ("Input micrograph file name pattern must contain wild card (*). Please check input_micrograph_pattern argument. Run %s -h for help." % (progname), getframeinfo(currentframe())) break if coords_pattern.find("*") == -1: error_status = ("Input coordinates file name pattern must contain wild card (*). Please check input_coordinates_pattern argument. Run %s -h for help." % (progname), getframeinfo(currentframe())) break if myid == main_node: if os.path.exists(out_dir): error_status = ("Output directory exists. Please change the name and restart the program.", getframeinfo(currentframe())) break break if_error_then_all_processes_exit_program(error_status) # Check invalid conditions of options check_options(options, progname) mic_name_list = None error_status = None if myid == main_node: if mic_list_file_path != None: print("Loading micrograph list from %s file ..." % (mic_list_file_path)) mic_name_list = read_text_file(mic_list_file_path) if len(mic_name_list) == 0: print("Directory of first micrograph entry is " % (os.path.dirname(mic_name_list[0]))) else: # assert (mic_list_file_path == None) print("Generating micrograph list in %s directory..." % (os.path.dirname(mic_pattern))) mic_name_list = glob.glob(mic_pattern) if len(mic_name_list) == 0: error_status = ("No micrograph file is found. Please check input_micrograph_pattern and/or input_micrograph_list_file argument. Run %s -h for help." % (progname), getframeinfo(currentframe())) else: print("Found %d microgarphs" % len(mic_name_list)) if_error_then_all_processes_exit_program(error_status) if RUNNING_UNDER_MPI: mic_name_list = wrap_mpi_bcast(mic_name_list, main_node) coords_name_list = None error_status = None if myid == main_node: coords_name_list = glob.glob(coords_pattern) if len(coords_name_list) == 0: error_status = ("No coordinates file is found. Please check input_coordinates_pattern argument. Run %s -h for help." % (progname), getframeinfo(currentframe())) if_error_then_all_processes_exit_program(error_status) if RUNNING_UNDER_MPI: coords_name_list = wrap_mpi_bcast(coords_name_list, main_node) ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## # all processes must have access to indices if options.import_ctf: i_enum = -1 i_enum += 1; idx_cter_def = i_enum # defocus [um]; index must be same as ctf object format i_enum += 1; idx_cter_cs = i_enum # Cs [mm]; index must be same as ctf object format i_enum += 1; idx_cter_vol = i_enum # voltage[kV]; index must be same as ctf object format i_enum += 1; idx_cter_apix = i_enum # pixel size [A]; index must be same as ctf object format i_enum += 1; idx_cter_bfactor = i_enum # B-factor [A^2]; index must be same as ctf object format i_enum += 1; idx_cter_ac = i_enum # amplitude contrast [%]; index must be same as ctf object format i_enum += 1; idx_cter_astig_amp = i_enum # astigmatism amplitude [um]; index must be same as ctf object format i_enum += 1; idx_cter_astig_ang = i_enum # astigmatism angle [degree]; index must be same as ctf object format i_enum += 1; idx_cter_sd_def = i_enum # std dev of defocus [um] i_enum += 1; idx_cter_sd_astig_amp = i_enum # std dev of ast amp [A] i_enum += 1; idx_cter_sd_astig_ang = i_enum # std dev of ast angle [degree] i_enum += 1; idx_cter_cv_def = i_enum # coefficient of variation of defocus [%] i_enum += 1; idx_cter_cv_astig_amp = i_enum # coefficient of variation of ast amp [%] i_enum += 1; idx_cter_spectra_diff = i_enum # average of differences between with- and without-astig. experimental 1D spectra at extrema i_enum += 1; idx_cter_error_def = i_enum # frequency at which signal drops by 50% due to estimated error of defocus alone [1/A] i_enum += 1; idx_cter_error_astig = i_enum # frequency at which signal drops by 50% due to estimated error of defocus and astigmatism [1/A] i_enum += 1; idx_cter_error_ctf = i_enum # limit frequency by CTF error [1/A] i_enum += 1; idx_cter_mic_name = i_enum # micrograph name i_enum += 1; n_idx_cter = i_enum # Prepare loop variables mic_basename_pattern = os.path.basename(mic_pattern) # file pattern without path mic_baseroot_pattern = os.path.splitext(mic_basename_pattern)[0] # file pattern without path and extension coords_format = options.coordinates_format.lower() box_size = options.box_size box_half = box_size // 2 mask2d = model_circle(box_size//2, box_size, box_size) # Create circular 2D mask to Util.infomask of particle images resample_ratio = options.resample_ratio n_mic_process = 0 n_mic_reject_no_coords = 0 n_mic_reject_no_cter_entry = 0 n_global_coords_detect = 0 n_global_coords_process = 0 n_global_coords_reject_out_of_boundary = 0 serial_id_list = [] error_status = None ## not a real while, an if with the opportunity to use break when errors need to be reported while myid == main_node: # # NOTE: 2016/05/24 Toshio Moriya # Now, ignores the path in mic_pattern and entries of mic_name_list to create serial ID # Only the basename (file name) in micrograph path must be match # # Create list of micrograph serial ID # Break micrograph name pattern into prefix and suffix to find the head index of the micrograph serial id # mic_basename_tokens = mic_basename_pattern.split('*') # assert (len(mic_basename_tokens) == 2) serial_id_head_index = len(mic_basename_tokens[0]) # Loop through micrograph names for mic_name in mic_name_list: # Find the tail index of the serial id and extract serial id from the micrograph name mic_basename = os.path.basename(mic_name) serial_id_tail_index = mic_basename.index(mic_basename_tokens[1]) serial_id = mic_basename[serial_id_head_index:serial_id_tail_index] serial_id_list.append(serial_id) # assert (len(serial_id_list) == len(mic_name)) del mic_name_list # Do not need this anymore # Load CTFs if necessary if options.import_ctf: ctf_list = read_text_row(options.import_ctf) # print("Detected CTF entries : %6d ..." % (len(ctf_list))) if len(ctf_list) == 0: error_status = ("No CTF entry is found in %s. Please check --import_ctf option. Run %s -h for help." % (options.import_ctf, progname), getframeinfo(currentframe())) break if (len(ctf_list[0]) != n_idx_cter): error_status = ("Number of columns (%d) must be %d in %s. The format might be old. Please run sxcter.py again." % (len(ctf_list[0]), n_idx_cter, options.import_ctf), getframeinfo(currentframe())) break ctf_dict={} n_reject_defocus_error = 0 ctf_error_limit = [options.defocus_error/100.0, options.astigmatism_error] for ctf_params in ctf_list: assert(len(ctf_params) == n_idx_cter) # mic_baseroot is name of micrograph minus the path and extension mic_baseroot = os.path.splitext(os.path.basename(ctf_params[idx_cter_mic_name]))[0] if(ctf_params[idx_cter_sd_def] / ctf_params[idx_cter_def] > ctf_error_limit[0]): print("Defocus error %f exceeds the threshold. Micrograph %s is rejected." % (ctf_params[idx_cter_sd_def] / ctf_params[idx_cter_def], mic_baseroot)) n_reject_defocus_error += 1 else: if(ctf_params[idx_cter_sd_astig_ang] > ctf_error_limit[1]): ctf_params[idx_cter_astig_amp] = 0.0 ctf_params[idx_cter_astig_ang] = 0.0 ctf_dict[mic_baseroot] = ctf_params del ctf_list # Do not need this anymore break if_error_then_all_processes_exit_program(error_status) if options.import_ctf: if options.limit_ctf: cutoff_histogram = [] #@ming compute the histogram for micrographs cut of by ctf_params limit. ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## restricted_serial_id_list = [] if myid == main_node: # Loop over serial IDs of micrographs for serial_id in serial_id_list: # mic_baseroot is name of micrograph minus the path and extension mic_baseroot = mic_baseroot_pattern.replace("*", serial_id) mic_name = mic_pattern.replace("*", serial_id) coords_name = coords_pattern.replace("*", serial_id) ########### # CHECKS: BEGIN if coords_name not in coords_name_list: print(" Cannot read %s. Skipping %s ..." % (coords_name, mic_baseroot)) n_mic_reject_no_coords += 1 continue # IF mic is in CTER results if options.import_ctf: if mic_baseroot not in ctf_dict: print(" Is not listed in CTER results. Skipping %s ..." % (mic_baseroot)) n_mic_reject_no_cter_entry += 1 continue else: ctf_params = ctf_dict[mic_baseroot] # CHECKS: END n_mic_process += 1 restricted_serial_id_list.append(serial_id) # restricted_serial_id_list = restricted_serial_id_list[:128] ## for testing against the nonMPI version if myid != main_node: if options.import_ctf: ctf_dict = None error_status = None if len(restricted_serial_id_list) < number_of_processes: error_status = ('Number of processes (%d) supplied by --np in mpirun cannot be greater than %d (number of micrographs that satisfy all criteria to be processed) ' % (number_of_processes, len(restricted_serial_id_list)), getframeinfo(currentframe())) if_error_then_all_processes_exit_program(error_status) ## keep a copy of the original output directory where the final bdb will be created original_out_dir = out_dir if RUNNING_UNDER_MPI: mpi_barrier(MPI_COMM_WORLD) restricted_serial_id_list = wrap_mpi_bcast(restricted_serial_id_list, main_node) mic_start, mic_end = MPI_start_end(len(restricted_serial_id_list), number_of_processes, myid) restricted_serial_id_list_not_sliced = restricted_serial_id_list restricted_serial_id_list = restricted_serial_id_list[mic_start:mic_end] if options.import_ctf: ctf_dict = wrap_mpi_bcast(ctf_dict, main_node) # generate subdirectories of out_dir, one for each process out_dir = os.path.join(out_dir,"%03d"%myid) if myid == main_node: print("Micrographs processed by main process (including percent complete):") len_processed_by_main_node_divided_by_100 = len(restricted_serial_id_list)/100.0 ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## ################################################################################################################################################################################################################## ##### Starting main parallel execution for my_idx, serial_id in enumerate(restricted_serial_id_list): mic_baseroot = mic_baseroot_pattern.replace("*", serial_id) mic_name = mic_pattern.replace("*", serial_id) coords_name = coords_pattern.replace("*", serial_id) if myid == main_node: print(mic_name, " ---> % 2.2f%%"%(my_idx/len_processed_by_main_node_divided_by_100)) mic_img = get_im(mic_name) # Read coordinates according to the specified format and # make the coordinates the center of particle image if coords_format == "sparx": coords_list = read_text_row(coords_name) elif coords_format == "eman1": coords_list = read_text_row(coords_name) for i in xrange(len(coords_list)): coords_list[i] = [(coords_list[i][0] + coords_list[i][2] // 2), (coords_list[i][1] + coords_list[i][3] // 2)] elif coords_format == "eman2": coords_list = js_open_dict(coords_name)["boxes"] for i in xrange(len(coords_list)): coords_list[i] = [coords_list[i][0], coords_list[i][1]] elif coords_format == "spider": coords_list = read_text_row(coords_name) for i in xrange(len(coords_list)): coords_list[i] = [coords_list[i][2], coords_list[i][3]] # else: assert (False) # Unreachable code # Calculate the new pixel size if options.import_ctf: ctf_params = ctf_dict[mic_baseroot] pixel_size_origin = ctf_params[idx_cter_apix] if resample_ratio < 1.0: # assert (resample_ratio > 0.0) new_pixel_size = pixel_size_origin / resample_ratio print("Resample micrograph to pixel size %6.4f and window segments from resampled micrograph." % new_pixel_size) else: # assert (resample_ratio == 1.0) new_pixel_size = pixel_size_origin # Set ctf along with new pixel size in resampled micrograph ctf_params[idx_cter_apix] = new_pixel_size else: # assert (not options.import_ctf) if resample_ratio < 1.0: # assert (resample_ratio > 0.0) print("Resample micrograph with ratio %6.4f and window segments from resampled micrograph." % resample_ratio) # else: # assert (resample_ratio == 1.0) # Apply filters to micrograph fftip(mic_img) if options.limit_ctf: # assert (options.import_ctf) # Cut off frequency components higher than CTF limit q1, q2 = ctflimit(box_size, ctf_params[idx_cter_def], ctf_params[idx_cter_cs], ctf_params[idx_cter_vol], new_pixel_size) # This is absolute frequency of CTF limit in scale of original micrograph if resample_ratio < 1.0: # assert (resample_ratio > 0.0) q1 = resample_ratio * q1 / float(box_size) # q1 = (pixel_size_origin / new_pixel_size) * q1/float(box_size) else: # assert (resample_ratio == 1.0) -> pixel_size_origin == new_pixel_size -> pixel_size_origin / new_pixel_size == 1.0 q1 = q1 / float(box_size) if q1 < 0.5: mic_img = filt_tanl(mic_img, q1, 0.01) cutoff_histogram.append(q1) # Cut off frequency components lower than the box size can express mic_img = fft(filt_gaussh(mic_img, resample_ratio / box_size)) # Resample micrograph, map coordinates, and window segments from resampled micrograph using new coordinates # after resampling by resample_ratio, new pixel size will be pixel_size/resample_ratio = new_pixel_size # NOTE: 2015/04/13 Toshio Moriya # resample() efficiently takes care of the case resample_ratio = 1.0 but # it does not set apix_*. Even though it sets apix_* when resample_ratio < 1.0 ... mic_img = resample(mic_img, resample_ratio) if options.invert: mic_stats = Util.infomask(mic_img, None, True) # mic_stat[0:mean, 1:SD, 2:min, 3:max] Util.mul_scalar(mic_img, -1.0) mic_img += 2 * mic_stats[0] if options.import_ctf: from utilities import generate_ctf ctf_obj = generate_ctf(ctf_params) # indexes 0 to 7 (idx_cter_def to idx_cter_astig_ang) must be same in cter format & ctf object format. # Prepare loop variables nx = mic_img.get_xsize() ny = mic_img.get_ysize() x0 = nx//2 y0 = ny//2 n_coords_reject_out_of_boundary = 0 local_stack_name = "bdb:%s#" % out_dir + mic_baseroot + '_ptcls' local_particle_id = 0 # can be different from coordinates_id # Loop over coordinates for coords_id in xrange(len(coords_list)): x = int(coords_list[coords_id][0]) y = int(coords_list[coords_id][1]) if resample_ratio < 1.0: # assert (resample_ratio > 0.0) x = int(x * resample_ratio) y = int(y * resample_ratio) # else: # assert(resample_ratio == 1.0) if( (0 <= x - box_half) and ( x + box_half <= nx ) and (0 <= y - box_half) and ( y + box_half <= ny ) ): particle_img = Util.window(mic_img, box_size, box_size, 1, x-x0, y-y0) else: print("In %s, coordinates ID = %04d (x = %4d, y = %4d, box_size = %4d) is out of micrograph bound, skipping ..." % (mic_baseroot, coords_id, x, y, box_size)) n_coords_reject_out_of_boundary += 1 continue particle_img = ramp(particle_img) particle_stats = Util.infomask(particle_img, mask2d, False) # particle_stats[0:mean, 1:SD, 2:min, 3:max] particle_img -= particle_stats[0] particle_img /= particle_stats[1] # NOTE: 2015/04/09 Toshio Moriya # ptcl_source_image might be redundant information ... # Consider re-organizing header entries... particle_img.set_attr("ptcl_source_image", mic_name) particle_img.set_attr("ptcl_source_coord_id", coords_id) particle_img.set_attr("ptcl_source_coord", [int(coords_list[coords_id][0]), int(coords_list[coords_id][1])]) particle_img.set_attr("resample_ratio", resample_ratio) # NOTE: 2015/04/13 Toshio Moriya # apix_* attributes are updated by resample() only when resample_ratio != 1.0 # Let's make sure header info is consistent by setting apix_* = 1.0 # regardless of options, so it is not passed down the processing line particle_img.set_attr("apix_x", 1.0) particle_img.set_attr("apix_y", 1.0) particle_img.set_attr("apix_z", 1.0) if options.import_ctf: particle_img.set_attr("ctf",ctf_obj) particle_img.set_attr("ctf_applied", 0) particle_img.set_attr("pixel_size_origin", pixel_size_origin) # particle_img.set_attr("apix_x", new_pixel_size) # particle_img.set_attr("apix_y", new_pixel_size) # particle_img.set_attr("apix_z", new_pixel_size) # NOTE: 2015/04/13 Toshio Moriya # Pawel Comment: Micrograph is not supposed to have CTF header info. # So, let's assume it does not exist & ignore its presence. # Note that resample() "correctly" updates pixel size of CTF header info if it exists # elif (particle_img.has_ctff()): # assert(not options.import_ctf) # ctf_origin = particle_img.get_attr("ctf_obj") # pixel_size_origin = round(ctf_origin.apix, 5) # Because SXCTER ouputs up to 5 digits # particle_img.set_attr("apix_x",pixel_size_origin) # particle_img.set_attr("apix_y",pixel_size_origin) # particle_img.set_attr("apix_z",pixel_size_origin) # print("local_stack_name, local_particle_id", local_stack_name, local_particle_id) particle_img.write_image(local_stack_name, local_particle_id) local_particle_id += 1 n_global_coords_detect += len(coords_list) n_global_coords_process += local_particle_id n_global_coords_reject_out_of_boundary += n_coords_reject_out_of_boundary # # MRK_DEBUG: Toshio Moriya 2016/05/03 # # Following codes are for debugging bdb. Delete in future # result = db_check_dict(local_stack_name) # print('# MRK_DEBUG: result = db_check_dict(local_stack_name): %s' % (result)) # result = db_list_dicts('bdb:%s' % out_dir) # print('# MRK_DEBUG: result = db_list_dicts(out_dir): %s' % (result)) # result = db_get_image_info(local_stack_name) # print('# MRK_DEBUG: result = db_get_image_info(local_stack_name)', result) # Release the data base of local stack from this process # so that the subprocess can access to the data base db_close_dict(local_stack_name) # # MRK_DEBUG: Toshio Moriya 2016/05/03 # # Following codes are for debugging bdb. Delete in future # cmd_line = "e2iminfo.py %s" % (local_stack_name) # print('# MRK_DEBUG: Executing the command: %s' % (cmd_line)) # cmdexecute(cmd_line) # # MRK_DEBUG: Toshio Moriya 2016/05/03 # # Following codes are for debugging bdb. Delete in future # cmd_line = "e2iminfo.py bdb:%s#data" % (out_dir) # print('# MRK_DEBUG: Executing the command: %s' % (cmd_line)) # cmdexecute(cmd_line) if RUNNING_UNDER_MPI: if options.import_ctf: if options.limit_ctf: cutoff_histogram = wrap_mpi_gatherv(cutoff_histogram, main_node) if myid == main_node: if options.limit_ctf: # Print out the summary of CTF-limit filtering print(" ") print("Global summary of CTF-limit filtering (--limit_ctf) ...") print("Percentage of filtered micrographs: %8.2f\n" % (len(cutoff_histogram) * 100.0 / len(restricted_serial_id_list_not_sliced))) n_bins = 10 if len(cutoff_histogram) >= n_bins: from statistics import hist_list cutoff_region, cutoff_counts = hist_list(cutoff_histogram, n_bins) print(" Histogram of cut-off frequency") print(" cut-off counts") for bin_id in xrange(n_bins): print(" %14.7f %7d" % (cutoff_region[bin_id], cutoff_counts[bin_id])) else: print("The number of filtered micrographs (%d) is less than the number of bins (%d). No histogram is produced." % (len(cutoff_histogram), n_bins)) n_mic_process = mpi_reduce(n_mic_process, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) n_mic_reject_no_coords = mpi_reduce(n_mic_reject_no_coords, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) n_mic_reject_no_cter_entry = mpi_reduce(n_mic_reject_no_cter_entry, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) n_global_coords_detect = mpi_reduce(n_global_coords_detect, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) n_global_coords_process = mpi_reduce(n_global_coords_process, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) n_global_coords_reject_out_of_boundary = mpi_reduce(n_global_coords_reject_out_of_boundary, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) # Print out the summary of all micrographs if main_node == myid: print(" ") print("Global summary of micrographs ...") print("Detected : %6d" % (len(restricted_serial_id_list_not_sliced))) print("Processed : %6d" % (n_mic_process)) print("Rejected by no coordinates file : %6d" % (n_mic_reject_no_coords)) print("Rejected by no CTER entry : %6d" % (n_mic_reject_no_cter_entry)) print(" ") print("Global summary of coordinates ...") print("Detected : %6d" % (n_global_coords_detect)) print("Processed : %6d" % (n_global_coords_process)) print("Rejected by out of boundary : %6d" % (n_global_coords_reject_out_of_boundary)) # print(" ") # print("DONE!!!") mpi_barrier(MPI_COMM_WORLD) if main_node == myid: import time time.sleep(1) print("\n Creating bdb:%s/data\n"%original_out_dir) for proc_i in range(number_of_processes): mic_start, mic_end = MPI_start_end(len(restricted_serial_id_list_not_sliced), number_of_processes, proc_i) for serial_id in restricted_serial_id_list_not_sliced[mic_start:mic_end]: e2bdb_command = "e2bdb.py " mic_baseroot = mic_baseroot_pattern.replace("*", serial_id) if RUNNING_UNDER_MPI: e2bdb_command += "bdb:" + os.path.join(original_out_dir,"%03d/"%proc_i) + mic_baseroot + "_ptcls " else: e2bdb_command += "bdb:" + os.path.join(original_out_dir, mic_baseroot + "_ptcls ") e2bdb_command += " --appendvstack=bdb:%s/data 1>/dev/null"%original_out_dir cmdexecute(e2bdb_command, printing_on_success = False) print("Done!\n") if RUNNING_UNDER_MPI: mpi_barrier(MPI_COMM_WORLD) from mpi import mpi_finalize mpi_finalize() sys.stdout.flush() sys.exit(0)