def main(): def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror): if mirror: m = 1 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0 - psi, 0, 0, 1.0) else: m = 0 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0 - psi, 0, 0, 1.0) return alpha, sx, sy, m progname = os.path.basename(sys.argv[0]) usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=15. --aa=0.01 --sym=symmetry --CTF" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--output_dir", type="string", default="./", help="output directory") parser.add_option("--ave2D", type="string", default=False, help="write to the disk a stack of 2D averages") parser.add_option("--var2D", type="string", default=False, help="write to the disk a stack of 2D variances") parser.add_option("--ave3D", type="string", default=False, help="write to the disk reconstructed 3D average") parser.add_option("--var3D", type="string", default=False, help="compute 3D variability (time consuming!)") parser.add_option("--img_per_grp", type="int", default=10, help="number of neighbouring projections") parser.add_option("--no_norm", action="store_true", default=False, help="do not use normalization") #parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" ) parser.add_option("--npad", type="int", default=2, help="number of time to pad the original images") parser.add_option("--sym", type="string", default="c1", help="symmetry") parser.add_option( "--fl", type="float", default=0.0, help= "cutoff freqency in absolute frequency (0.0-0.5). (Default - no filtration)" ) parser.add_option( "--aa", type="float", default=0.0, help= "fall off of the filter. Put 0.01 if user has no clue about falloff (Default - no filtration)" ) parser.add_option("--CTF", action="store_true", default=False, help="use CFT correction") parser.add_option("--VERBOSE", action="store_true", default=False, help="Long output for debugging") #parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version") #parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" ) #parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" ) #parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" ) #parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" ) parser.add_option( "--VAR", action="store_true", default=False, help="stack on input consists of 2D variances (Default False)") parser.add_option( "--decimate", type="float", default=1.0, help= "image decimate rate, a number larger (expand image) or less (shrink image) than 1. default is 1" ) parser.add_option( "--window", type="int", default=0, help= "reduce images to a small image size without changing pixel_size. Default value is zero." ) #parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)") parser.add_option( "--nvec", type="int", default=0, help="number of eigenvectors, default = 0 meaning no PCA calculated") parser.add_option( "--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)") (options, args) = parser.parse_args() ##### from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX from applications import MPI_start_end from reconstruction import recons3d_em, recons3d_em_MPI from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI from utilities import print_begin_msg, print_end_msg, print_msg from utilities import read_text_row, get_image, get_im from utilities import bcast_EMData_to_all, bcast_number_to_all from utilities import get_symt # This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015 from EMAN2db import db_open_dict # Set up global variables related to bdb cache if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() # Set up global variables related to ERROR function global_def.BATCH = True # detect if program is running under MPI RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ if RUNNING_UNDER_MPI: global_def.MPI = True if options.symmetrize: if RUNNING_UNDER_MPI: try: sys.argv = mpi_init(len(sys.argv), sys.argv) try: number_of_proc = mpi_comm_size(MPI_COMM_WORLD) if (number_of_proc > 1): ERROR( "Cannot use more than one CPU for symmetry prepration", "sx3dvariability", 1) except: pass except: pass if options.output_dir != "./" and not os.path.exists( options.output_dir): os.mkdir(options.output_dir) # Input #instack = "Clean_NORM_CTF_start_wparams.hdf" #instack = "bdb:data" from logger import Logger, BaseLogger_Files if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt")) log_main = Logger(BaseLogger_Files()) log_main.prefix = os.path.join(options.output_dir, "./") instack = args[0] sym = options.sym.lower() if (sym == "c1"): ERROR("There is no need to symmetrize stack for C1 symmetry", "sx3dvariability", 1) line = "" for a in sys.argv: line += " " + a log_main.add(line) if (instack[:4] != "bdb:"): if output_dir == "./": stack = "bdb:data" else: stack = "bdb:" + options.output_dir + "/data" delete_bdb(stack) junk = cmdexecute("sxcpy.py " + instack + " " + stack) else: stack = instack qt = EMUtil.get_all_attributes(stack, 'xform.projection') na = len(qt) ts = get_symt(sym) ks = len(ts) angsa = [None] * na for k in xrange(ks): #Qfile = "Q%1d"%k if options.output_dir != "./": Qfile = os.path.join(options.output_dir, "Q%1d" % k) else: Qfile = os.path.join(options.output_dir, "Q%1d" % k) #delete_bdb("bdb:Q%1d"%k) delete_bdb("bdb:" + Qfile) #junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k) junk = cmdexecute("e2bdb.py " + stack + " --makevstack=bdb:" + Qfile) #DB = db_open_dict("bdb:Q%1d"%k) DB = db_open_dict("bdb:" + Qfile) for i in xrange(na): ut = qt[i] * ts[k] DB.set_attr(i, "xform.projection", ut) #bt = ut.get_params("spider") #angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]] #write_text_row(angsa, 'ptsma%1d.txt'%k) #junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k) #junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k)) DB.close() if options.output_dir == "./": delete_bdb("bdb:sdata") else: delete_bdb("bdb:" + options.output_dir + "/" + "sdata") #junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q") sdata = "bdb:" + options.output_dir + "/" + "sdata" print(sdata) junk = cmdexecute("e2bdb.py " + options.output_dir + " --makevstack=" + sdata + " --filt=Q") #junk = cmdexecute("ls EMAN2DB/sdata*") #a = get_im("bdb:sdata") a = get_im(sdata) a.set_attr("variabilitysymmetry", sym) #a.write_image("bdb:sdata") a.write_image(sdata) else: sys.argv = mpi_init(len(sys.argv), sys.argv) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 1: stack = args[0] else: print(("usage: " + usage)) print(("Please run '" + progname + " -h' for detailed options")) return 1 t0 = time() # obsolete flags options.MPI = True options.nvec = 0 options.radiuspca = -1 options.iter = 40 options.abs = 0.0 options.squ = 0.0 if options.fl > 0.0 and options.aa == 0.0: ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid) if options.VAR and options.SND: ERROR("Only one of var and SND can be set!", "sx3dvariability", myid) exit() if options.VAR and (options.ave2D or options.ave3D or options.var2D): ERROR( "When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid) exit() #if options.SND and (options.ave2D or options.ave3D): # ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid) # exit() if options.nvec > 0: ERROR("PCA option not implemented", "sx3dvariability", 1, myid) exit() if options.nvec > 0 and options.ave3D == None: ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid) exit() import string options.sym = options.sym.lower() # if global_def.CACHE_DISABLE: # from utilities import disable_bdb_cache # disable_bdb_cache() # global_def.BATCH = True if myid == main_node: if options.output_dir != "./" and not os.path.exists( options.output_dir): os.mkdir(options.output_dir) img_per_grp = options.img_per_grp nvec = options.nvec radiuspca = options.radiuspca from logger import Logger, BaseLogger_Files #if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt")) log_main = Logger(BaseLogger_Files()) log_main.prefix = os.path.join(options.output_dir, "./") if myid == main_node: line = "" for a in sys.argv: line += " " + a log_main.add(line) log_main.add("-------->>>Settings given by all options<<<-------") log_main.add("instack :" + stack) log_main.add("output_dir :" + options.output_dir) log_main.add("var3d :" + options.var3D) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" #print_begin_msg("sx3dvariability") msg = "sx3dvariability" log_main.add(msg) print(line, msg) msg = ("%-70s: %s\n" % ("Input stack", stack)) log_main.add(msg) print(line, msg) symbaselen = 0 if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() if options.sym != "c1": imgdata = get_im(stack) try: i = imgdata.get_attr("variabilitysymmetry").lower() if (i != options.sym): ERROR( "The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", myid=myid) except: ERROR( "Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", myid=myid) from utilities import get_symt i = len(get_symt(options.sym)) if ((nima / i) * i != nima): ERROR( "The length of the input stack is incorrect for symmetry processing", "sx3dvariability", myid=myid) symbaselen = nima / i else: symbaselen = nima else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) Tracker = {} Tracker["total_stack"] = nima if options.decimate == 1.: if options.window != 0: nx = options.window ny = options.window else: if options.window == 0: nx = int(nx * options.decimate) ny = int(ny * options.decimate) else: nx = int(options.window * options.decimate) ny = nx Tracker["nx"] = nx Tracker["ny"] = ny Tracker["nz"] = nx symbaselen = bcast_number_to_all(symbaselen) if radiuspca == -1: radiuspca = nx / 2 - 2 if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = "%-70s: %d\n" % ("Number of projection", nima) log_main.add(msg) print(line, msg) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) """ if options.SND: from projection import prep_vol, prgs from statistics import im_diff from utilities import get_im, model_circle, get_params_proj, set_params_proj from utilities import get_ctf, generate_ctf from filter import filt_ctf imgdata = EMData.read_images(stack, range(img_begin, img_end)) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) bcast_EMData_to_all(vol, myid) volft, kb = prep_vol(vol) mask = model_circle(nx/2-2, nx, ny) varList = [] for i in xrange(img_begin, img_end): phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin]) ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y]) if options.CTF: ctf_params = get_ctf(imgdata[i-img_begin]) ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params)) diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask) diff2 = diff*diff set_params_proj(diff2, [phi, theta, psi, s2x, s2y]) varList.append(diff2) mpi_barrier(MPI_COMM_WORLD) """ if options.VAR: #varList = EMData.read_images(stack, range(img_begin, img_end)) varList = [] this_image = EMData() for index_of_particle in xrange(img_begin, img_end): this_image.read_image(stack, index_of_particle) varList.append( image_decimate_window_xform_ctf(this_image, options.decimate, options.window, options.CTF)) else: from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2 from utilities import model_blank, nearest_proj, model_circle from applications import pca from statistics import avgvar, avgvar_ctf, ccc from filter import filt_tanl from morphology import threshold, square_root from projection import project, prep_vol, prgs from sets import Set if myid == main_node: t1 = time() proj_angles = [] aveList = [] tab = EMUtil.get_all_attributes(stack, 'xform.projection') for i in xrange(nima): t = tab[i].get_params('spider') phi = t['phi'] theta = t['theta'] psi = t['psi'] x = theta if x > 90.0: x = 180.0 - x x = x * 10000 + psi proj_angles.append([x, t['phi'], t['theta'], t['psi'], i]) t2 = time() line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = "%-70s: %d\n" % ("Number of neighboring projections", img_per_grp) log_main.add(msg) print(line, msg) msg = "...... Finding neighboring projections\n" log_main.add(msg) print(line, msg) if options.VERBOSE: msg = "Number of images per group: %d" % img_per_grp log_main.add(msg) print(line, msg) msg = "Now grouping projections" log_main.add(msg) print(line, msg) proj_angles.sort() proj_angles_list = [0.0] * (nima * 4) if myid == main_node: for i in xrange(nima): proj_angles_list[i * 4] = proj_angles[i][1] proj_angles_list[i * 4 + 1] = proj_angles[i][2] proj_angles_list[i * 4 + 2] = proj_angles[i][3] proj_angles_list[i * 4 + 3] = proj_angles[i][4] proj_angles_list = bcast_list_to_all(proj_angles_list, myid, main_node) proj_angles = [] for i in xrange(nima): proj_angles.append([ proj_angles_list[i * 4], proj_angles_list[i * 4 + 1], proj_angles_list[i * 4 + 2], int(proj_angles_list[i * 4 + 3]) ]) del proj_angles_list proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end)) all_proj = Set() for im in proj_list: for jm in im: all_proj.add(proj_angles[jm][3]) all_proj = list(all_proj) if options.VERBOSE: print("On node %2d, number of images needed to be read = %5d" % (myid, len(all_proj))) index = {} for i in xrange(len(all_proj)): index[all_proj[i]] = i mpi_barrier(MPI_COMM_WORLD) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("%-70s: %.2f\n" % ("Finding neighboring projections lasted [s]", time() - t2)) log_main.add(msg) print(msg) msg = ("%-70s: %d\n" % ("Number of groups processed on the main node", len(proj_list))) log_main.add(msg) print(line, msg) if options.VERBOSE: print("Grouping projections took: ", (time() - t2) / 60, "[min]") print("Number of groups on main node: ", len(proj_list)) mpi_barrier(MPI_COMM_WORLD) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("...... calculating the stack of 2D variances \n") log_main.add(msg) print(line, msg) if options.VERBOSE: print("Now calculating the stack of 2D variances") proj_params = [0.0] * (nima * 5) aveList = [] varList = [] if nvec > 0: eigList = [[] for i in xrange(nvec)] if options.VERBOSE: print("Begin to read images on processor %d" % (myid)) ttt = time() #imgdata = EMData.read_images(stack, all_proj) imgdata = [] for index_of_proj in xrange(len(all_proj)): #img = EMData() #img.read_image(stack, all_proj[index_of_proj]) dmg = image_decimate_window_xform_ctf( get_im(stack, all_proj[index_of_proj]), options.decimate, options.window, options.CTF) #print dmg.get_xsize(), "init" imgdata.append(dmg) if options.VERBOSE: print("Reading images on processor %d done, time = %.2f" % (myid, time() - ttt)) print("On processor %d, we got %d images" % (myid, len(imgdata))) mpi_barrier(MPI_COMM_WORLD) ''' imgdata2 = EMData.read_images(stack, range(img_begin, img_end)) if options.fl > 0.0: for k in xrange(len(imgdata2)): imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) if myid == main_node: vol.write_image("vol_ctf.hdf") print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf")) del vol, imgdata2 mpi_barrier(MPI_COMM_WORLD) ''' from applications import prepare_2d_forPCA from utilities import model_blank for i in xrange(len(proj_list)): ki = proj_angles[proj_list[i][0]][3] if ki >= symbaselen: continue mi = index[ki] phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi]) grp_imgdata = [] for j in xrange(img_per_grp): mj = index[proj_angles[proj_list[i][j]][3]] phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj]) alpha, sx, sy, mirror = params_3D_2D_NEW( phi, theta, psi, s2x, s2y, mirror_list[i][j]) if thetaM <= 90: if mirror == 0: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, phiM - phi, 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, 180 - (phiM - phi), 0.0, 0.0, 1.0) else: if mirror == 0: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, -(phiM - phi), 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, -(180 - (phiM - phi)), 0.0, 0.0, 1.0) set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0]) grp_imgdata.append(imgdata[mj]) #print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize() if not options.no_norm: #print grp_imgdata[j].get_xsize() mask = model_circle(nx / 2 - 2, nx, nx) for k in xrange(img_per_grp): ave, std, minn, maxx = Util.infomask( grp_imgdata[k], mask, False) grp_imgdata[k] -= ave grp_imgdata[k] /= std del mask if options.fl > 0.0: from filter import filt_ctf, filt_table from fundamentals import fft, window2d nx2 = 2 * nx ny2 = 2 * ny if options.CTF: from utilities import pad for k in xrange(img_per_grp): grp_imgdata[k] = window2d( fft( filt_tanl( filt_ctf( fft( pad(grp_imgdata[k], nx2, ny2, 1, 0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa)), nx, ny) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) else: for k in xrange(img_per_grp): grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) else: from utilities import pad, read_text_file from filter import filt_ctf, filt_table from fundamentals import fft, window2d nx2 = 2 * nx ny2 = 2 * ny if options.CTF: from utilities import pad for k in xrange(img_per_grp): grp_imgdata[k] = window2d( fft( filt_ctf(fft( pad(grp_imgdata[k], nx2, ny2, 1, 0.0)), grp_imgdata[k].get_attr("ctf"), binary=1)), nx, ny) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) ''' if i < 10 and myid == main_node: for k in xrange(10): grp_imgdata[k].write_image("grp%03d.hdf"%i, k) ''' """ if myid == main_node and i==0: for pp in xrange(len(grp_imgdata)): grp_imgdata[pp].write_image("pp.hdf", pp) """ ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata) """ if myid == main_node and i==0: for pp in xrange(len(grp_imgdata)): grp_imgdata[pp].write_image("qq.hdf", pp) """ var = model_blank(nx, ny) for q in grp_imgdata: Util.add_img2(var, q) Util.mul_scalar(var, 1.0 / (len(grp_imgdata) - 1)) # Switch to std dev var = square_root(threshold(var)) #if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a") #else: ave, var = avgvar(grp_imgdata, mode="a") """ if myid == main_node: ave.write_image("avgv.hdf",i) var.write_image("varv.hdf",i) """ set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0]) set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0]) aveList.append(ave) varList.append(var) if options.VERBOSE: print("%5.2f%% done on processor %d" % (i * 100.0 / len(proj_list), myid)) if nvec > 0: eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True) for k in xrange(nvec): set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0]) eigList[k].append(eig[k]) """ if myid == 0 and i == 0: for k in xrange(nvec): eig[k].write_image("eig.hdf", k) """ del imgdata # To this point, all averages, variances, and eigenvectors are computed if options.ave2D: from fundamentals import fpol if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node: for im in xrange(len(aveList)): aveList[im].write_image( os.path.join(options.output_dir, options.ave2D), km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im + i + 70000) """ nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('members', map(int, members)) members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('pix_err', map(float, members)) members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('refprojdir', map(float, members)) """ tmpvol = fpol(ave, Tracker["nx"], Tracker["nx"], 1) tmpvol.write_image( os.path.join(options.output_dir, options.ave2D), km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in xrange(len(aveList)): send_EMData(aveList[im], main_node, im + myid + 70000) """ members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) members = aveList[im].get_attr('pix_err') mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) except: mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) """ if options.ave3D: from fundamentals import fpol if options.VERBOSE: print("Reconstructing 3D average volume") ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(ave3D, myid) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" ave3D = fpol(ave3D, Tracker["nx"], Tracker["nx"], Tracker["nx"]) ave3D.write_image( os.path.join(options.output_dir, options.ave3D)) msg = ("%-70s: %s\n" % ( "Writing to the disk volume reconstructed from averages as", options.ave3D)) log_main.add(msg) print(line, msg) del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList if nvec > 0: for k in xrange(nvec): if options.VERBOSE: print("Reconstruction eigenvolumes", k) cont = True ITER = 0 mask2d = model_circle(radiuspca, nx, nx) while cont: #print "On node %d, iteration %d"%(myid, ITER) eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(eig3D, myid, main_node) if options.fl > 0.0: eig3D = filt_tanl(eig3D, options.fl, options.aa) if myid == main_node: eig3D.write_image( os.path.join(options.outpout_dir, "eig3d_%03d.hdf" % (k, ITER))) Util.mul_img(eig3D, model_circle(radiuspca, nx, nx, nx)) eig3Df, kb = prep_vol(eig3D) del eig3D cont = False icont = 0 for l in xrange(len(eigList[k])): phi, theta, psi, s2x, s2y = get_params_proj( eigList[k][l]) proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y]) cl = ccc(proj, eigList[k][l], mask2d) if cl < 0.0: icont += 1 cont = True eigList[k][l] *= -1.0 u = int(cont) u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD) icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" u = int(u[0]) msg = (" Eigenvector: ", k, " number changed ", int(icont[0])) log_main.add(msg) print(line, msg) else: u = 0 u = bcast_number_to_all(u, main_node) cont = bool(u) ITER += 1 del eig3Df, kb mpi_barrier(MPI_COMM_WORLD) del eigList, mask2d if options.ave3D: del ave3D if options.var2D: from fundamentals import fpol if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node: for im in xrange(len(varList)): tmpvol = fpol(varList[im], Tracker["nx"], Tracker["nx"], 1) tmpvol.write_image( os.path.join(options.output_dir, options.var2D), km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im + i + 70000) tmpvol = fpol(ave, Tracker["nx"], Tracker["nx"], 1) tmpvol.write_image( os.path.join(options.output_dir, options.var2D, km)) km += 1 else: mpi_send(len(varList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in xrange(len(varList)): send_EMData(varList[im], main_node, im + myid + 70000) # What with the attributes?? mpi_barrier(MPI_COMM_WORLD) if options.var3D: if myid == main_node and options.VERBOSE: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("Reconstructing 3D variability volume") log_main.add(msg) print(line, msg) t6 = time() # radiusvar = options.radius # if( radiusvar < 0 ): radiusvar = nx//2 -3 res = recons3d_4nn_MPI(myid, varList, symmetry=options.sym, npad=options.npad) #res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ) if myid == main_node: from fundamentals import fpol res = fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"]) res.write_image(os.path.join(options.output_dir, options.var3D)) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("%-70s: %.2f\n" % ("Reconstructing 3D variability took [s]", time() - t6)) log_main.add(msg) print(line, msg) if options.VERBOSE: print("Reconstruction took: %.2f [min]" % ((time() - t6) / 60)) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("%-70s: %.2f\n" % ("Total time for these computations [s]", time() - t0)) print(line, msg) log_main.add(msg) if options.VERBOSE: print("Total time for these computations: %.2f [min]" % ((time() - t0) / 60)) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("sx3dvariability") print(line, msg) log_main.add(msg) from mpi import mpi_finalize mpi_finalize() if RUNNING_UNDER_MPI: global_def.MPI = False global_def.BATCH = False
def main(): from optparse import OptionParser from global_def import SPARXVERSION from EMAN2 import EMData from logger import Logger, BaseLogger_Files import sys, os, time global Tracker, Blockdata from global_def import ERROR progname = os.path.basename(sys.argv[0]) usage = progname + " --output_dir=output_dir --isac_dir=output_dir_of_isac " parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--adjust_to_analytic_model", action="store_true", default=False, help="adjust power spectrum of 2-D averages to an analytic model ") parser.add_option( "--adjust_to_given_pw2", action="store_true", default=False, help="adjust power spectrum to 2-D averages to given 1D power spectrum" ) parser.add_option("--B_enhance", action="store_true", default=False, help="using B-factor to enhance 2-D averages") parser.add_option("--no_adjustment", action="store_true", default=False, help="No power spectrum adjustment") options_list = [] adjust_to_analytic_model = False for q in sys.argv[1:]: if (q[:26] == "--adjust_to_analytic_model"): adjust_to_analytic_model = True options_list.append(q) break adjust_to_given_pw2 = False for q in sys.argv[1:]: if (q[:21] == "--adjust_to_given_pw2"): adjust_to_given_pw2 = True options_list.append(q) break B_enhance = False for q in sys.argv[1:]: if (q[:11] == "--B_enhance"): B_enhance = True options_list.append(q) break no_adjustment = False for q in sys.argv[1:]: if (q[:15] == "--no_adjustment"): no_adjustment = True options_list.append(q) break if len(options_list) == 0: if (Blockdata["myid"] == Blockdata["main_node"]): print( "specify one of the following options to start: 1. adjust_to_analytic_model; 2. adjust_to_given_pw2; 3. B_enhance; 4. no_adjustment" ) if len(options_list) > 1: ERROR( "The specified options are exclusive. Use only one of them to start", "sxcompute_isac_avg.py", 1, Blockdata["myid"]) # options in common parser.add_option( "--isac_dir", type="string", default='', help="ISAC run output directory, input directory for this command") parser.add_option( "--output_dir", type="string", default='', help="output directory where computed averages are saved") parser.add_option("--pixel_size", type="float", default=-1.0, help="pixel_size of raw images") parser.add_option( "--fl", type="float", default=-1.0, help= "low pass filter, =-1, not applied; =1, using FH1 (initial resolution), =2 using FH2 (resolution after local alignment), or user provided value" ) parser.add_option("--stack", type="string", default="", help="data stack used in ISAC") parser.add_option("--radius", type="int", default=-1, help="radius") parser.add_option("--xr", type="float", default=-1.0, help="local alignment search range") parser.add_option("--ts", type="float", default=1.0, help="local alignment search step") parser.add_option("--fh", type="float", default=-1., help="local alignment high frequencies limit") parser.add_option("--maxit", type="int", default=5, help="local alignment iterations") parser.add_option("--navg", type="int", default=-1, help="number of aveages") parser.add_option("--skip_local_alignment", action="store_true", default=False, help="skip local alignment") parser.add_option( "--noctf", action="store_true", default=False, help= "no ctf correction, useful for negative stained data. always ctf for cryo data" ) if B_enhance: parser.add_option( "--B_start", type="float", default=10.0, help= "start frequency (1./Angstrom) of power spectrum for B_factor estimation" ) parser.add_option( "--Bfactor", type="float", default=-1.0, help= "User defined bactors (e.g. 45.0[A^2]). By default, the program automatically estimates B-factor. " ) if adjust_to_given_pw2: parser.add_option("--modelpw", type="string", default='', help="1-D reference power spectrum") checking_flag = 0 if (Blockdata["myid"] == Blockdata["main_node"]): if not os.path.exists(options.modelpw): checking_flag = 1 checking_flag = bcast_number_to_all(checking_flag, Blockdata["main_node"], MPI_COMM_WORLD) if checking_flag == 1: ERROR("User provided power spectrum does not exist", "sxcompute_isac_avg.py", 1, Blockdata["myid"]) (options, args) = parser.parse_args(sys.argv[1:]) Tracker = {} Constants = {} Constants["isac_dir"] = options.isac_dir Constants["masterdir"] = options.output_dir Constants["pixel_size"] = options.pixel_size Constants["orgstack"] = options.stack Constants["radius"] = options.radius Constants["xrange"] = options.xr Constants["xstep"] = options.ts Constants["FH"] = options.fh Constants["maxit"] = options.maxit Constants["navg"] = options.navg Constants["low_pass_filter"] = options.fl if B_enhance: Constants["B_start"] = options.B_start Constants["Bfactor"] = options.Bfactor if adjust_to_given_pw2: Constants["modelpw"] = options.modelpw Tracker["constants"] = Constants # ------------------------------------------------------------- # # Create and initialize Tracker dictionary with input options # State Variables #<<<---------------------->>>imported functions<<<--------------------------------------------- from utilities import get_im, bcast_number_to_all, write_text_file, read_text_file, wrap_mpi_bcast, write_text_row from utilities import cmdexecute from filter import filt_tanl from time import sleep from logger import Logger, BaseLogger_Files import user_functions import string from string import split, atoi, atof import json #x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1) #y_range = x_range ####----------------------------------------------------------- # Create Master directory line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]: masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen") else: masterdir = Tracker["constants"]["masterdir"] if (Blockdata["myid"] == Blockdata["main_node"]): msg = "Postprocessing ISAC 2D averages starts" print(line, "Postprocessing ISAC 2D averages starts") if not masterdir: timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime()) masterdir = "sharpen_" + Tracker["constants"]["isac_dir"] os.mkdir(masterdir) else: if os.path.exists(masterdir): print("%s already exists" % masterdir) else: os.mkdir(masterdir) li = len(masterdir) else: li = 0 li = mpi_bcast(li, 1, MPI_INT, Blockdata["main_node"], MPI_COMM_WORLD)[0] masterdir = mpi_bcast(masterdir, li, MPI_CHAR, Blockdata["main_node"], MPI_COMM_WORLD) masterdir = string.join(masterdir, "") Tracker["constants"]["masterdir"] = masterdir log_main = Logger(BaseLogger_Files()) log_main.prefix = Tracker["constants"]["masterdir"] + "/" while not os.path.exists(Tracker["constants"]["masterdir"]): print("Node ", Blockdata["myid"], " waiting...", Tracker["constants"]["masterdir"]) sleep(1) mpi_barrier(MPI_COMM_WORLD) if (Blockdata["myid"] == Blockdata["main_node"]): init_dict = {} print(Tracker["constants"]["isac_dir"]) Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"], "2dalignment") core = read_text_row( os.path.join(Tracker["directory"], "initial2Dparams.txt")) for im in xrange(len(core)): init_dict[im] = core[im] del core else: init_dict = 0 init_dict = wrap_mpi_bcast(init_dict, Blockdata["main_node"], communicator=MPI_COMM_WORLD) ### if (Blockdata["myid"] == Blockdata["main_node"]): #Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack") image = get_im(Tracker["constants"]["orgstack"], 0) Tracker["constants"]["nnxo"] = image.get_xsize() try: ctf_params = image.get_attr("ctf") if Tracker["constants"]["pixel_size"] == -1.: Tracker["constants"]["pixel_size"] = ctf_params.apix except: print("pixel size value is not given.") Tracker["ini_shrink"] = float( get_im(os.path.join(Tracker["directory"], "aqfinal.hdf"), 0).get_xsize()) / Tracker["constants"]["nnxo"] else: Tracker["ini_shrink"] = 0 Tracker = wrap_mpi_bcast(Tracker, Blockdata["main_node"], communicator=MPI_COMM_WORLD) #print(Tracker["constants"]["pixel_size"], "pixel_size") x_range = max(Tracker["constants"]["xrange"], int(1. / Tracker["ini_shrink"]) + 1) y_range = x_range if (Blockdata["myid"] == Blockdata["main_node"]): parameters = read_text_row( os.path.join(Tracker["constants"]["isac_dir"], "all_parameters.txt")) else: parameters = 0 parameters = wrap_mpi_bcast(parameters, Blockdata["main_node"], communicator=MPI_COMM_WORLD) params_dict = {} list_dict = {} #parepare params_dict if Tracker["constants"]["navg"] < 0: navg = EMUtil.get_image_count( os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf")) else: navg = min( Tracker["constants"]["navg"], EMUtil.get_image_count( os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf"))) global_dict = {} ptl_list = [] memlist = [] if (Blockdata["myid"] == Blockdata["main_node"]): for iavg in xrange(navg): params_of_this_average = [] image = get_im( os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf"), iavg) members = image.get_attr("members") memlist.append(members) for im in xrange(len(members)): abs_id = members[im] global_dict[abs_id] = [iavg, im] P = combine_params2( init_dict[abs_id][0], init_dict[abs_id][1], init_dict[abs_id][2], init_dict[abs_id][3], \ parameters[abs_id][0], parameters[abs_id][1]/Tracker["ini_shrink"], parameters[abs_id][2]/Tracker["ini_shrink"], parameters[abs_id][3]) if parameters[abs_id][3] == -1: print("wrong one") params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0]) ptl_list.append(abs_id) params_dict[iavg] = params_of_this_average list_dict[iavg] = members write_text_row( params_of_this_average, os.path.join(Tracker["constants"]["masterdir"], "params_avg_%03d.txt" % iavg)) ptl_list.sort() init_params = [None for im in xrange(len(ptl_list))] for im in xrange(len(ptl_list)): init_params[im] = [ptl_list[im]] + params_dict[global_dict[ ptl_list[im]][0]][global_dict[ptl_list[im]][1]] write_text_row( init_params, os.path.join(Tracker["constants"]["masterdir"], "init_isac_params.txt")) else: params_dict = 0 list_dict = 0 memlist = 0 params_dict = wrap_mpi_bcast(params_dict, Blockdata["main_node"], communicator=MPI_COMM_WORLD) list_dict = wrap_mpi_bcast(list_dict, Blockdata["main_node"], communicator=MPI_COMM_WORLD) memlist = wrap_mpi_bcast(memlist, Blockdata["main_node"], communicator=MPI_COMM_WORLD) # Now computing! del init_dict tag_sharpen_avg = 1000 ## always apply low pass filter to B_enhanced images to suppress noise in high frequencies enforced_to_H1 = False if options.B_enhance: if Tracker["constants"]["low_pass_filter"] == -1: print("User does not provide low pass filter") enforced_to_H1 = True if navg < Blockdata["nproc"]: # Each CPU do one average FH_list = [None for im in xrange(navg)] for iavg in xrange(navg): if Blockdata["myid"] == iavg: mlist = [None for i in xrange(len(list_dict[iavg]))] for im in xrange(len(mlist)): mlist[im] = get_im(Tracker["constants"]["orgstack"], list_dict[iavg][im]) set_params2D(mlist[im], params_dict[iavg][im], xform="xform.align2d") if options.noctf: new_avg, frc, plist = compute_average_noctf( mlist, Tracker["constants"]["radius"]) else: new_avg, frc, plist = compute_average_ctf( mlist, Tracker["constants"]["radius"]) FH1 = get_optimistic_res(frc) #write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d_before_ali.txt"%iavg)) if not options.skip_local_alignment: new_average1 = within_group_refinement([mlist[kik] for kik in xrange(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \ ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \ dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1) new_average2 = within_group_refinement([mlist[kik] for kik in xrange(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \ ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \ dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1) if options.noctf: new_avg, frc, plist = compute_average_noctf( mlist, Tracker["constants"]["radius"]) else: new_avg, frc, plist = compute_average_ctf( mlist, Tracker["constants"]["radius"]) FH2 = get_optimistic_res(frc) #write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg)) #if Tracker["constants"]["nopwadj"]: # pw adjustment, 1. analytic model 2. PDB model 3. B-facttor enhancement else: FH2 = 0.0 FH_list[iavg] = [FH1, FH2] if options.B_enhance: new_avg, gb = apply_enhancement( new_avg, Tracker["constants"]["B_start"], Tracker["constants"]["pixel_size"], Tracker["constants"]["Bfactor"]) print("Process avg %d %f %f %f" % (iavg, gb, FH1, FH2)) elif options.adjust_to_given_pw2: roo = read_text_file(Tracker["constants"]["modelpw"], -1) roo = roo[0] # always put pw in the first column new_avg = adjust_pw_to_model( new_avg, Tracker["constants"]["pixel_size"], roo) elif options.adjust_to_analytic_model: new_avg = adjust_pw_to_model( new_avg, Tracker["constants"]["pixel_size"], None) elif options.no_adjustment: pass print("Process avg %d %f %f" % (iavg, FH1, FH2)) if Tracker["constants"]["low_pass_filter"] != -1.: if Tracker["constants"]["low_pass_filter"] == 1.: low_pass_filter = FH1 elif Tracker["constants"]["low_pass_filter"] == 2.: low_pass_filter = FH2 if options.skip_local_alignment: low_pass_filter = FH1 else: low_pass_filter = Tracker["constants"][ "low_pass_filter"] if low_pass_filter >= 0.45: low_pass_filter = 0.45 new_avg = filt_tanl(new_avg, low_pass_filter, 0.1) new_avg.set_attr("members", list_dict[iavg]) new_avg.set_attr("n_objects", len(list_dict[iavg])) mpi_barrier(MPI_COMM_WORLD) for im in xrange(navg): # avg if im == Blockdata[ "myid"] and Blockdata["myid"] != Blockdata["main_node"]: send_EMData(new_avg, Blockdata["main_node"], tag_sharpen_avg) elif Blockdata["myid"] == Blockdata["main_node"]: if im != Blockdata["main_node"]: new_avg_other_cpu = recv_EMData(im, tag_sharpen_avg) new_avg_other_cpu.set_attr("members", memlist[im]) new_avg_other_cpu.write_image( os.path.join(Tracker["constants"]["masterdir"], "class_averages.hdf"), im) else: new_avg.write_image( os.path.join(Tracker["constants"]["masterdir"], "class_averages.hdf"), im) if not options.skip_local_alignment: if im == Blockdata["myid"]: write_text_row( plist, os.path.join(Tracker["constants"]["masterdir"], "ali2d_local_params_avg_%03d.txt" % im)) if Blockdata["myid"] == im and Blockdata["myid"] != Blockdata[ "main_node"]: wrap_mpi_send(plist_dict[im], Blockdata["main_node"], MPI_COMM_WORLD) elif im != Blockdata["main_node"] and Blockdata[ "myid"] == Blockdata["main_node"]: dummy = wrap_mpi_recv(im, MPI_COMM_WORLD) plist_dict[im] = dummy if im == Blockdata["myid"] and im != Blockdata["main_node"]: wrap_mpi_send(FH_list[im], Blockdata["main_node"], MPI_COMM_WORLD) elif im != Blockdata["main_node"] and Blockdata[ "myid"] == Blockdata["main_node"]: dummy = wrap_mpi_recv(im, MPI_COMM_WORLD) FH_list[im] = dummy else: if im == Blockdata["myid"] and im != Blockdata["main_node"]: wrap_mpi_send(FH_list, Blockdata["main_node"], MPI_COMM_WORLD) elif im != Blockdata["main_node"] and Blockdata[ "myid"] == Blockdata["main_node"]: dummy = wrap_mpi_recv(im, MPI_COMM_WORLD) FH_list[im] = dummy[im] mpi_barrier(MPI_COMM_WORLD) else: FH_list = [[0, 0.0, 0.0] for im in xrange(navg)] image_start, image_end = MPI_start_end(navg, Blockdata["nproc"], Blockdata["myid"]) if Blockdata["myid"] == Blockdata["main_node"]: cpu_dict = {} for iproc in xrange(Blockdata["nproc"]): local_image_start, local_image_end = MPI_start_end( navg, Blockdata["nproc"], iproc) for im in xrange(local_image_start, local_image_end): cpu_dict[im] = iproc else: cpu_dict = 0 cpu_dict = wrap_mpi_bcast(cpu_dict, Blockdata["main_node"], communicator=MPI_COMM_WORLD) slist = [None for im in xrange(navg)] ini_list = [None for im in xrange(navg)] avg1_list = [None for im in xrange(navg)] avg2_list = [None for im in xrange(navg)] plist_dict = {} data_list = [None for im in xrange(navg)] if Blockdata["myid"] == Blockdata["main_node"]: print("read data") for iavg in xrange(image_start, image_end): mlist = [None for i in xrange(len(list_dict[iavg]))] for im in xrange(len(mlist)): mlist[im] = get_im(Tracker["constants"]["orgstack"], list_dict[iavg][im]) set_params2D(mlist[im], params_dict[iavg][im], xform="xform.align2d") data_list[iavg] = mlist print("read data done %d" % Blockdata["myid"]) #if Blockdata["myid"] == Blockdata["main_node"]: print("start to compute averages") for iavg in xrange(image_start, image_end): mlist = data_list[iavg] if options.noctf: new_avg, frc, plist = compute_average_noctf( mlist, Tracker["constants"]["radius"]) else: new_avg, frc, plist = compute_average_ctf( mlist, Tracker["constants"]["radius"]) FH1 = get_optimistic_res(frc) #write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d_before_ali.txt"%iavg)) if not options.skip_local_alignment: new_average1 = within_group_refinement([mlist[kik] for kik in xrange(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \ ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \ dst=0.0, maxit=Tracker["constants"]["maxit"], FH=max(Tracker["constants"]["FH"], FH1), FF=0.1) new_average2 = within_group_refinement([mlist[kik] for kik in xrange(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \ ou= Tracker["constants"]["radius"], rs=1.0, xrng=[ x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \ dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1) if options.noctf: new_avg, frc, plist = compute_average_noctf( mlist, Tracker["constants"]["radius"]) else: new_avg, frc, plist = compute_average_ctf( mlist, Tracker["constants"]["radius"]) plist_dict[iavg] = plist FH2 = get_optimistic_res(frc) else: FH2 = 0.0 #write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg)) FH_list[iavg] = [iavg, FH1, FH2] if options.B_enhance: new_avg, gb = apply_enhancement( new_avg, Tracker["constants"]["B_start"], Tracker["constants"]["pixel_size"], Tracker["constants"]["Bfactor"]) print("Process avg %d %f %f %f" % (iavg, gb, FH1, FH2)) elif options.adjust_to_given_pw2: roo = read_text_file(Tracker["constants"]["modelpw"], -1) roo = roo[0] # always on the first column new_avg = adjust_pw_to_model( new_avg, Tracker["constants"]["pixel_size"], roo) print("Process avg %d %f %f" % (iavg, FH1, FH2)) elif adjust_to_analytic_model: new_avg = adjust_pw_to_model( new_avg, Tracker["constants"]["pixel_size"], None) print("Process avg %d %f %f" % (iavg, FH1, FH2)) elif options.no_adjustment: pass if Tracker["constants"]["low_pass_filter"] != -1.: new_avg = filt_tanl(new_avg, Tracker["constants"]["low_pass_filter"], 0.1) if Tracker["constants"]["low_pass_filter"] != -1.: if Tracker["constants"]["low_pass_filter"] == 1.: low_pass_filter = FH1 elif Tracker["constants"]["low_pass_filter"] == 2.: low_pass_filter = FH2 if options.skip_local_alignment: low_pass_filter = FH1 else: low_pass_filter = Tracker["constants"]["low_pass_filter"] if low_pass_filter >= 0.45: low_pass_filter = 0.45 new_avg = filt_tanl(new_avg, low_pass_filter, 0.1) else: if enforced_to_H1: new_avg = filt_tanl(new_avg, FH1, 0.1) if options.B_enhance: new_avg = fft(new_avg) new_avg.set_attr("members", list_dict[iavg]) new_avg.set_attr("n_objects", len(list_dict[iavg])) slist[iavg] = new_avg ## send to main node to write mpi_barrier(MPI_COMM_WORLD) for im in xrange(navg): # avg if cpu_dict[im] == Blockdata[ "myid"] and Blockdata["myid"] != Blockdata["main_node"]: send_EMData(slist[im], Blockdata["main_node"], tag_sharpen_avg) elif cpu_dict[im] == Blockdata["myid"] and Blockdata[ "myid"] == Blockdata["main_node"]: slist[im].set_attr("members", memlist[im]) slist[im].write_image( os.path.join(Tracker["constants"]["masterdir"], "class_averages.hdf"), im) elif cpu_dict[im] != Blockdata["myid"] and Blockdata[ "myid"] == Blockdata["main_node"]: new_avg_other_cpu = recv_EMData(cpu_dict[im], tag_sharpen_avg) new_avg_other_cpu.set_attr("members", memlist[im]) new_avg_other_cpu.write_image( os.path.join(Tracker["constants"]["masterdir"], "class_averages.hdf"), im) if not options.skip_local_alignment: if cpu_dict[im] == Blockdata["myid"]: write_text_row( plist_dict[im], os.path.join(Tracker["constants"]["masterdir"], "ali2d_local_params_avg_%03d.txt" % im)) if cpu_dict[im] == Blockdata[ "myid"] and cpu_dict[im] != Blockdata["main_node"]: wrap_mpi_send(plist_dict[im], Blockdata["main_node"], MPI_COMM_WORLD) wrap_mpi_send(FH_list, Blockdata["main_node"], MPI_COMM_WORLD) elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[ "myid"] == Blockdata["main_node"]: dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD) plist_dict[im] = dummy dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD) FH_list[im] = dummy[im] else: if cpu_dict[im] == Blockdata[ "myid"] and cpu_dict[im] != Blockdata["main_node"]: wrap_mpi_send(FH_list, Blockdata["main_node"], MPI_COMM_WORLD) elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[ "myid"] == Blockdata["main_node"]: dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD) FH_list[im] = dummy[im] mpi_barrier(MPI_COMM_WORLD) mpi_barrier(MPI_COMM_WORLD) if not options.skip_local_alignment: if Blockdata["myid"] == Blockdata["main_node"]: ali3d_local_params = [None for im in xrange(len(ptl_list))] for im in xrange(len(ptl_list)): ali3d_local_params[im] = [ptl_list[im]] + plist_dict[ global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]] write_text_row( ali3d_local_params, os.path.join(Tracker["constants"]["masterdir"], "ali2d_local_params.txt")) write_text_row( FH_list, os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt")) else: if Blockdata["myid"] == Blockdata["main_node"]: write_text_row( FH_list, os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt")) mpi_barrier(MPI_COMM_WORLD) target_xr = 3 target_yr = 3 if (Blockdata["myid"] == 0): cmd = "{} {} {} {} {} {} {} {} {} {}".format("sxchains.py", os.path.join(Tracker["constants"]["masterdir"],"class_averages.hdf"),\ os.path.join(Tracker["constants"]["masterdir"],"junk.hdf"),os.path.join(Tracker["constants"]["masterdir"],"ordered_class_averages.hdf"),\ "--circular","--radius=%d"%Tracker["constants"]["radius"] , "--xr=%d"%(target_xr+1),"--yr=%d"%(target_yr+1),"--align", ">/dev/null") junk = cmdexecute(cmd) cmd = "{} {}".format( "rm -rf", os.path.join(Tracker["constants"]["masterdir"], "junk.hdf")) junk = cmdexecute(cmd) from mpi import mpi_finalize mpi_finalize() exit()
def main(): from logger import Logger, BaseLogger_Files import user_functions from optparse import OptionParser, SUPPRESS_HELP from global_def import SPARXVERSION from EMAN2 import EMData main_node = 0 mpi_init(0, []) mpi_comm = MPI_COMM_WORLD myid = mpi_comm_rank(MPI_COMM_WORLD) mpi_size = mpi_comm_size(MPI_COMM_WORLD) # Total number of processes, passed by --np option. # mpi_barrier(mpi_comm) # from mpi import mpi_finalize # mpi_finalize() # print "mpi finalize" # from sys import exit # exit() progname = os.path.basename(sys.argv[0]) usage = progname + " stack [output_directory] --ir=inner_radius --radius=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --center=center_type --maxit1=max_iter1 --maxit2=max_iter2 --L2threshold=0.1 --fl --aa --ref_a=S --sym=c1" usage += """ stack 2D images in a stack file: (default required string) output_directory: directory name into which the output files will be written. If it does not exist, the directory will be created. If it does exist, the program will continue executing from where it stopped (if it did not already reach the end). The "--use_latest_master_directory" option can be used to choose the most recent directory that starts with "master". """ parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--radius", type="int", help="radius of the particle: has to be less than < int(nx/2)-1 (default required int)") parser.add_option("--ir", type="int", default=1, help="inner radius for rotational search: > 0 (default 1)") parser.add_option("--rs", type="int", default=1, help="step between rings in rotational search: >0 (default 1)") parser.add_option("--xr", type="string", default='0', help="range for translation search in x direction: search is +/xr in pixels (default '0')") parser.add_option("--yr", type="string", default='0', help="range for translation search in y direction: if omitted will be set to xr, search is +/yr in pixels (default '0')") parser.add_option("--ts", type="string", default='1.0', help="step size of the translation search in x-y directions: search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default '1.0')") parser.add_option("--delta", type="string", default='2.0', help="angular step of reference projections: (default '2.0')") #parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches (phi and theta)") parser.add_option("--center", type="float", default=-1.0, help="centering of 3D template: average shift method; 0: no centering; 1: center of gravity (default -1.0)") parser.add_option("--maxit1", type="int", default=400, help="maximum number of iterations performed for the GA part: (default 400)") parser.add_option("--maxit2", type="int", default=50, help="maximum number of iterations performed for the finishing up part: (default 50)") parser.add_option("--L2threshold", type="float", default=0.03, help="stopping criterion of GA: given as a maximum relative dispersion of volumes' L2 norms: (default 0.03)") parser.add_option("--doga", type="float", default=0.1, help="do GA when fraction of orientation changes less than 1.0 degrees is at least doga: (default 0.1)") parser.add_option("--n_shc_runs", type="int", default=4, help="number of quasi-independent shc runs (same as '--nruns' parameter from sxviper.py): (default 4)") parser.add_option("--n_rv_runs", type="int", default=10, help="number of rviper iterations: (default 10)") parser.add_option("--n_v_runs", type="int", default=3, help="number of viper runs for each r_viper cycle: (default 3)") parser.add_option("--outlier_percentile", type="float", default=95.0, help="percentile above which outliers are removed every rviper iteration: (default 95.0)") parser.add_option("--iteration_start", type="int", default=0, help="starting iteration for rviper: 0 means go to the most recent one (default 0)") #parser.add_option("--CTF", action="store_true", default=False, help="NOT IMPLEMENTED Consider CTF correction during the alignment ") #parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data (default 1.0)") parser.add_option("--ref_a", type="string", default='S', help="method for generating the quasi-uniformly distributed projection directions: (default S)") parser.add_option("--sym", type="string", default='c1', help="point-group symmetry of the structure: (default c1)") # parser.add_option("--function", type="string", default="ref_ali3d", help="name of the reference preparation function (ref_ali3d by default)") ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--function", type="string", default="ref_ali3d", help=SUPPRESS_HELP) parser.add_option("--npad", type="int", default=2, help="padding size for 3D reconstruction: (default 2)") # parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default 2)") #options introduced for the do_volume function parser.add_option("--fl", type="float", default=0.25, help="cut-off frequency applied to the template volume: using a hyperbolic tangent low-pass filter (default 0.25)") parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass filter: (default 0.1)") parser.add_option("--pwreference", type="string", default='', help="text file with a reference power spectrum: (default none)") parser.add_option("--mask3D", type="string", default=None, help="3D mask file: (default sphere)") parser.add_option("--moon_elimination", type="string", default='', help="elimination of disconnected pieces: two arguments: mass in KDa and pixel size in px/A separated by comma, no space (default none)") # used for debugging, help is supressed with SUPPRESS_HELP ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--my_random_seed", type="int", default=123, help = SUPPRESS_HELP) ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--run_get_already_processed_viper_runs", action="store_true", dest="run_get_already_processed_viper_runs", default=False, help = SUPPRESS_HELP) ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False, help = SUPPRESS_HELP) parser.add_option("--criterion_name", type="string", default='80th percentile',help="criterion deciding if volumes have a core set of stable projections: '80th percentile', other options:'fastest increase in the last quartile' (default '80th percentile')") parser.add_option("--outlier_index_threshold_method",type="string", default='discontinuity_in_derivative',help="method that decides which images to keep: discontinuity_in_derivative, other options:percentile, angle_measure (default discontinuity_in_derivative)") parser.add_option("--angle_threshold", type="int", default=30, help="angle threshold for projection removal if using 'angle_measure': (default 30)") required_option_list = ['radius'] (options, args) = parser.parse_args(sys.argv[1:]) options.CTF = False options.snr = 1.0 options.an = -1 if options.moon_elimination == "": options.moon_elimination = [] else: options.moon_elimination = map(float, options.moon_elimination.split(",")) # Making sure all required options appeared. for required_option in required_option_list: if not options.__dict__[required_option]: print "\n ==%s== mandatory option is missing.\n"%required_option print "Please run '" + progname + " -h' for detailed options" return 1 mpi_barrier(MPI_COMM_WORLD) if(myid == main_node): print "****************************************************************" Util.version() print "****************************************************************" sys.stdout.flush() mpi_barrier(MPI_COMM_WORLD) # this is just for benefiting from a user friendly parameter name options.ou = options.radius my_random_seed = options.my_random_seed criterion_name = options.criterion_name outlier_index_threshold_method = options.outlier_index_threshold_method use_latest_master_directory = options.use_latest_master_directory iteration_start_default = options.iteration_start number_of_rrr_viper_runs = options.n_rv_runs no_of_viper_runs_analyzed_together_from_user_options = options.n_v_runs no_of_shc_runs_analyzed_together = options.n_shc_runs outlier_percentile = options.outlier_percentile angle_threshold = options.angle_threshold run_get_already_processed_viper_runs = options.run_get_already_processed_viper_runs get_already_processed_viper_runs(run_get_already_processed_viper_runs) import random random.seed(my_random_seed) if len(args) < 1 or len(args) > 3: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" return 1 # if len(args) > 2: # ref_vol = get_im(args[2]) # else: ref_vol = None # error_status = None # if myid == 0: # number_of_images = EMUtil.get_image_count(args[0]) # if mpi_size > number_of_images: # error_status = ('Number of processes supplied by --np in mpirun needs to be less than or equal to %d (total number of images) ' % number_of_images, getframeinfo(currentframe())) # if_error_then_all_processes_exit_program(error_status) bdb_stack_location = "" masterdir = "" if len(args) == 2: masterdir = args[1] if masterdir[-1] != DIR_DELIM: masterdir += DIR_DELIM elif len(args) == 1: if use_latest_master_directory: all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)] import re; r = re.compile("^master.*$") all_dirs = filter(r.match, all_dirs) if len(all_dirs)>0: # all_dirs = max(all_dirs, key=os.path.getctime) masterdir = max(all_dirs, key=os.path.getmtime) masterdir += DIR_DELIM log = Logger(BaseLogger_Files()) error_status = 0 if mpi_size % no_of_shc_runs_analyzed_together != 0: ERROR('Number of processes needs to be a multiple of the number of quasi-independent runs (shc) within each viper run. ' 'Total quasi-independent runs by default are 3, you can change it by specifying ' '--n_shc_runs option (in sxviper this option is called --nruns). Also, to improve communication time it is recommended that ' 'the number of processes divided by the number of quasi-independent runs is a power ' 'of 2 (e.g. 2, 4, 8 or 16 depending on how many physical cores each node has).', 'sxviper', 1) error_status = 1 if_error_then_all_processes_exit_program(error_status) #Create folder for all results or check if there is one created already if(myid == main_node): #cmd = "{}".format("Rmycounter ccc") #cmdexecute(cmd) if( masterdir == ""): timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime()) masterdir = "master"+timestring if not os.path.exists(masterdir): cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) if ':' in args[0]: bdb_stack_location = args[0].split(":")[0] + ":" + masterdir + args[0].split(":")[1] org_stack_location = args[0] if(not os.path.exists(os.path.join(masterdir,"EMAN2DB" + DIR_DELIM))): # cmd = "{} {}".format("cp -rp EMAN2DB", masterdir, "EMAN2DB" DIR_DELIM) # cmdexecute(cmd) cmd = "{} {} {}".format("e2bdb.py", org_stack_location,"--makevstack=" + bdb_stack_location + "_000") cmdexecute(cmd) from applications import header try: header(bdb_stack_location + "_000", params='original_image_index', fprint=True) print "Images were already indexed!" except KeyError: print "Indexing images" header(bdb_stack_location + "_000", params='original_image_index', consecutive=True) else: filename = os.path.basename(args[0]) bdb_stack_location = "bdb:" + masterdir + os.path.splitext(filename)[0] if(not os.path.exists(os.path.join(masterdir,"EMAN2DB" + DIR_DELIM))): cmd = "{} {} {}".format("sxcpy.py ", args[0], bdb_stack_location + "_000") cmdexecute(cmd) from applications import header try: header(bdb_stack_location + "_000", params='original_image_index', fprint=True) print "Images were already indexed!" except KeyError: print "Indexing images" header(bdb_stack_location + "_000", params='original_image_index', consecutive=True) # send masterdir to all processes dir_len = len(masterdir)*int(myid == main_node) dir_len = mpi_bcast(dir_len,1,MPI_INT,0,MPI_COMM_WORLD)[0] masterdir = mpi_bcast(masterdir,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD) masterdir = string.join(masterdir,"") if masterdir[-1] != DIR_DELIM: masterdir += DIR_DELIM global_def.LOGFILE = os.path.join(masterdir, global_def.LOGFILE) print_program_start_information() # mpi_barrier(mpi_comm) # from mpi import mpi_finalize # mpi_finalize() # print "mpi finalize" # from sys import exit # exit() # send bdb_stack_location to all processes dir_len = len(bdb_stack_location)*int(myid == main_node) dir_len = mpi_bcast(dir_len,1,MPI_INT,0,MPI_COMM_WORLD)[0] bdb_stack_location = mpi_bcast(bdb_stack_location,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD) bdb_stack_location = string.join(bdb_stack_location,"") iteration_start = get_latest_directory_increment_value(masterdir, "main") if (myid == main_node): if (iteration_start < iteration_start_default): ERROR('Starting iteration provided is greater than last iteration performed. Quiting program', 'sxviper', 1) error_status = 1 if iteration_start_default!=0: iteration_start = iteration_start_default if (myid == main_node): if (number_of_rrr_viper_runs < iteration_start): ERROR('Please provide number of rviper runs (--n_rv_runs) greater than number of iterations already performed.', 'sxviper', 1) error_status = 1 if_error_then_all_processes_exit_program(error_status) for rviper_iter in range(iteration_start, number_of_rrr_viper_runs + 1): if(myid == main_node): all_projs = EMData.read_images(bdb_stack_location + "_%03d"%(rviper_iter - 1)) print "XXXXXXXXXXXXXXXXX" print "Number of projections (in loop): " + str(len(all_projs)) print "XXXXXXXXXXXXXXXXX" subset = range(len(all_projs)) else: all_projs = None subset = None runs_iter = get_latest_directory_increment_value(masterdir + NAME_OF_MAIN_DIR + "%03d"%rviper_iter, DIR_DELIM + NAME_OF_RUN_DIR, start_value=0) - 1 no_of_viper_runs_analyzed_together = max(runs_iter + 2, no_of_viper_runs_analyzed_together_from_user_options) first_time_entering_the_loop_need_to_do_full_check_up = True while True: runs_iter += 1 if not first_time_entering_the_loop_need_to_do_full_check_up: if runs_iter >= no_of_viper_runs_analyzed_together: break first_time_entering_the_loop_need_to_do_full_check_up = False this_run_is_NOT_complete = 0 if (myid == main_node): independent_run_dir = masterdir + DIR_DELIM + NAME_OF_MAIN_DIR + ('%03d' + DIR_DELIM + NAME_OF_RUN_DIR + "%03d" + DIR_DELIM)%(rviper_iter, runs_iter) if run_get_already_processed_viper_runs: cmd = "{} {}".format("mkdir -p", masterdir + DIR_DELIM + NAME_OF_MAIN_DIR + ('%03d' + DIR_DELIM)%(rviper_iter)); cmdexecute(cmd) cmd = "{} {}".format("rm -rf", independent_run_dir); cmdexecute(cmd) cmd = "{} {}".format("cp -r", get_already_processed_viper_runs() + " " + independent_run_dir); cmdexecute(cmd) if os.path.exists(independent_run_dir + "log.txt") and (string_found_in_file("Finish VIPER2", independent_run_dir + "log.txt")): this_run_is_NOT_complete = 0 else: this_run_is_NOT_complete = 1 cmd = "{} {}".format("rm -rf", independent_run_dir); cmdexecute(cmd) cmd = "{} {}".format("mkdir -p", independent_run_dir); cmdexecute(cmd) this_run_is_NOT_complete = mpi_bcast(this_run_is_NOT_complete,1,MPI_INT,main_node,MPI_COMM_WORLD)[0] dir_len = len(independent_run_dir) dir_len = mpi_bcast(dir_len,1,MPI_INT,main_node,MPI_COMM_WORLD)[0] independent_run_dir = mpi_bcast(independent_run_dir,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD) independent_run_dir = string.join(independent_run_dir,"") else: this_run_is_NOT_complete = mpi_bcast(this_run_is_NOT_complete,1,MPI_INT,main_node,MPI_COMM_WORLD)[0] dir_len = 0 independent_run_dir = "" dir_len = mpi_bcast(dir_len,1,MPI_INT,main_node,MPI_COMM_WORLD)[0] independent_run_dir = mpi_bcast(independent_run_dir,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD) independent_run_dir = string.join(independent_run_dir,"") if this_run_is_NOT_complete: mpi_barrier(MPI_COMM_WORLD) if independent_run_dir[-1] != DIR_DELIM: independent_run_dir += DIR_DELIM log.prefix = independent_run_dir options.user_func = user_functions.factory[options.function] # for debugging purposes #if (myid == main_node): #cmd = "{} {}".format("cp ~/log.txt ", independent_run_dir) #cmdexecute(cmd) #cmd = "{} {}{}".format("cp ~/paramdir/params$(mycounter ccc).txt ", independent_run_dir, "param%03d.txt"%runs_iter) #cmd = "{} {}{}".format("cp ~/paramdir/params$(mycounter ccc).txt ", independent_run_dir, "params.txt") #cmdexecute(cmd) if (myid == main_node): store_value_of_simple_vars_in_json_file(masterdir + 'program_state_stack.json', locals(), exclude_list_of_vars=["usage"], vars_that_will_show_only_size = ["subset"]) store_value_of_simple_vars_in_json_file(masterdir + 'program_state_stack.json', options.__dict__, write_or_append='a') # mpi_barrier(mpi_comm) # from mpi import mpi_finalize # mpi_finalize() # print "mpi finalize" # from sys import exit # exit() out_params, out_vol, out_peaks = multi_shc(all_projs, subset, no_of_shc_runs_analyzed_together, options, mpi_comm=mpi_comm, log=log, ref_vol=ref_vol) # end of: if this_run_is_NOT_complete: if runs_iter >= (no_of_viper_runs_analyzed_together_from_user_options - 1): increment_for_current_iteration = identify_outliers(myid, main_node, rviper_iter, no_of_viper_runs_analyzed_together, no_of_viper_runs_analyzed_together_from_user_options, masterdir, bdb_stack_location, outlier_percentile, criterion_name, outlier_index_threshold_method, angle_threshold) if increment_for_current_iteration == MUST_END_PROGRAM_THIS_ITERATION: break no_of_viper_runs_analyzed_together += increment_for_current_iteration # end of independent viper loop calculate_volumes_after_rotation_and_save_them(options, rviper_iter, masterdir, bdb_stack_location, myid, mpi_size, no_of_viper_runs_analyzed_together, no_of_viper_runs_analyzed_together_from_user_options) if increment_for_current_iteration == MUST_END_PROGRAM_THIS_ITERATION: if (myid == main_node): print "RVIPER found a core set of stable projections for the current RVIPER iteration (%d), the maximum angle difference between corresponding projections from different VIPER volumes is less than %.2f. Finishing."%(rviper_iter, ANGLE_ERROR_THRESHOLD) break else: if (myid == main_node): print "After running the last iteration (%d), RVIPER did not find a set of projections with the maximum angle difference between corresponding projections from different VIPER volumes less than %.2f Finishing."%(rviper_iter, ANGLE_ERROR_THRESHOLD) # end of RVIPER loop #mpi_finalize() #sys.exit() mpi_barrier(MPI_COMM_WORLD) mpi_finalize()
def main(): from logger import Logger, BaseLogger_Files arglist = [] i = 0 while( i < len(sys.argv) ): if sys.argv[i]=='-p4pg': i = i+2 elif sys.argv[i]=='-p4wd': i = i+2 else: arglist.append( sys.argv[i] ) i = i+1 progname = os.path.basename(arglist[0]) usage = progname + " stack outdir <mask> --focus=3Dmask --radius=outer_radius --delta=angular_step" +\ "--an=angular_neighborhood --maxit=max_iter --CTF --sym=c1 --function=user_function --independent=indenpendent_runs --number_of_images_per_group=number_of_images_per_group --low_pass_frequency=.25 --seed=random_seed" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--focus", type ="string", default ='', help="bineary 3D mask for focused clustering ") parser.add_option("--ir", type = "int", default =1, help="inner radius for rotational correlation > 0 (set to 1)") parser.add_option("--radius", type = "int", default =-1, help="particle radius in pixel for rotational correlation <nx-1 (set to the radius of the particle)") parser.add_option("--maxit", type = "int", default =25, help="maximum number of iteration") parser.add_option("--rs", type = "int", default =1, help="step between rings in rotational correlation >0 (set to 1)" ) parser.add_option("--xr", type ="string", default ='1', help="range for translation search in x direction, search is +/-xr ") parser.add_option("--yr", type ="string", default ='-1', help="range for translation search in y direction, search is +/-yr (default = same as xr)") parser.add_option("--ts", type ="string", default ='0.25', help="step size of the translation search in both directions direction, search is -xr, -xr+ts, 0, xr-ts, xr ") parser.add_option("--delta", type ="string", default ='2', help="angular step of reference projections") parser.add_option("--an", type ="string", default ='-1', help="angular neighborhood for local searches") parser.add_option("--center", type ="int", default =0, help="0 - if you do not want the volume to be centered, 1 - center the volume using cog (default=0)") parser.add_option("--nassign", type ="int", default =1, help="number of reassignment iterations performed for each angular step (set to 3) ") parser.add_option("--nrefine", type ="int", default =0, help="number of alignment iterations performed for each angular step (set to 0)") parser.add_option("--CTF", action ="store_true", default =False, help="do CTF correction during clustring") parser.add_option("--stoprnct", type ="float", default =3.0, help="Minimum percentage of assignment change to stop the program") parser.add_option("--sym", type ="string", default ='c1', help="symmetry of the structure ") parser.add_option("--function", type ="string", default ='do_volume_mrk05', help="name of the reference preparation function") parser.add_option("--independent", type ="int", default = 3, help="number of independent run") parser.add_option("--number_of_images_per_group", type ="int", default =1000, help="number of groups") parser.add_option("--low_pass_filter", type ="float", default =-1.0, help="absolute frequency of low-pass filter for 3d sorting on the original image size" ) parser.add_option("--nxinit", type ="int", default =64, help="initial image size for sorting" ) parser.add_option("--unaccounted", action ="store_true", default =False, help="reconstruct the unaccounted images") parser.add_option("--seed", type ="int", default =-1, help="random seed for create initial random assignment for EQ Kmeans") parser.add_option("--smallest_group", type ="int", default =500, help="minimum members for identified group") parser.add_option("--sausage", action ="store_true", default =False, help="way of filter volume") parser.add_option("--chunkdir", type ="string", default ='', help="chunkdir for computing margin of error") parser.add_option("--PWadjustment", type ="string", default ='', help="1-D power spectrum of PDB file used for EM volume power spectrum correction") parser.add_option("--protein_shape", type ="string", default ='g', help="protein shape. It defines protein preferred orientation angles. Currently it has g and f two types ") parser.add_option("--upscale", type ="float", default =0.5, help=" scaling parameter to adjust the power spectrum of EM volumes") parser.add_option("--wn", type ="int", default =0, help="optimal window size for data processing") parser.add_option("--interpolation", type ="string", default ="4nn", help="3-d reconstruction interpolation method, two options trl and 4nn") (options, args) = parser.parse_args(arglist[1:]) if len(args) < 1 or len(args) > 4: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" else: if len(args)>2: mask_file = args[2] else: mask_file = None orgstack =args[0] masterdir =args[1] global_def.BATCH = True #---initialize MPI related variables from mpi import mpi_init, mpi_comm_size, MPI_COMM_WORLD, mpi_comm_rank,mpi_barrier,mpi_bcast, mpi_bcast, MPI_INT,MPI_CHAR sys.argv = mpi_init(len(sys.argv),sys.argv) nproc = mpi_comm_size(MPI_COMM_WORLD) myid = mpi_comm_rank(MPI_COMM_WORLD) mpi_comm = MPI_COMM_WORLD main_node= 0 # import some utilities from utilities import get_im,bcast_number_to_all,cmdexecute,write_text_file,read_text_file,wrap_mpi_bcast, get_params_proj, write_text_row from applications import recons3d_n_MPI, mref_ali3d_MPI, Kmref_ali3d_MPI from statistics import k_means_match_clusters_asg_new,k_means_stab_bbenum from applications import mref_ali3d_EQ_Kmeans, ali3d_mref_Kmeans_MPI # Create the main log file from logger import Logger,BaseLogger_Files if myid ==main_node: log_main=Logger(BaseLogger_Files()) log_main.prefix = masterdir+"/" else: log_main =None #--- fill input parameters into dictionary named after Constants Constants ={} Constants["stack"] = args[0] Constants["masterdir"] = masterdir Constants["mask3D"] = mask_file Constants["focus3Dmask"] = options.focus Constants["indep_runs"] = options.independent Constants["stoprnct"] = options.stoprnct Constants["number_of_images_per_group"] = options.number_of_images_per_group Constants["CTF"] = options.CTF Constants["maxit"] = options.maxit Constants["ir"] = options.ir Constants["radius"] = options.radius Constants["nassign"] = options.nassign Constants["rs"] = options.rs Constants["xr"] = options.xr Constants["yr"] = options.yr Constants["ts"] = options.ts Constants["delta"] = options.delta Constants["an"] = options.an Constants["sym"] = options.sym Constants["center"] = options.center Constants["nrefine"] = options.nrefine #Constants["fourvar"] = options.fourvar Constants["user_func"] = options.function Constants["low_pass_filter"] = options.low_pass_filter # enforced low_pass_filter #Constants["debug"] = options.debug Constants["main_log_prefix"] = args[1] #Constants["importali3d"] = options.importali3d Constants["myid"] = myid Constants["main_node"] = main_node Constants["nproc"] = nproc Constants["log_main"] = log_main Constants["nxinit"] = options.nxinit Constants["unaccounted"] = options.unaccounted Constants["seed"] = options.seed Constants["smallest_group"] = options.smallest_group Constants["sausage"] = options.sausage Constants["chunkdir"] = options.chunkdir Constants["PWadjustment"] = options.PWadjustment Constants["upscale"] = options.upscale Constants["wn"] = options.wn Constants["3d-interpolation"] = options.interpolation Constants["protein_shape"] = options.protein_shape # ----------------------------------------------------- # # Create and initialize Tracker dictionary with input options Tracker = {} Tracker["constants"] = Constants Tracker["maxit"] = Tracker["constants"]["maxit"] Tracker["radius"] = Tracker["constants"]["radius"] #Tracker["xr"] = "" #Tracker["yr"] = "-1" # Do not change! #Tracker["ts"] = 1 #Tracker["an"] = "-1" #Tracker["delta"] = "2.0" #Tracker["zoom"] = True #Tracker["nsoft"] = 0 #Tracker["local"] = False #Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"] Tracker["upscale"] = Tracker["constants"]["upscale"] #Tracker["upscale"] = 0.5 Tracker["applyctf"] = False # Should the data be premultiplied by the CTF. Set to False for local continuous. #Tracker["refvol"] = None Tracker["nxinit"] = Tracker["constants"]["nxinit"] #Tracker["nxstep"] = 32 Tracker["icurrentres"] = -1 #Tracker["ireachedres"] = -1 #Tracker["lowpass"] = 0.4 #Tracker["falloff"] = 0.2 #Tracker["inires"] = options.inires # Now in A, convert to absolute before using Tracker["fuse_freq"] = 50 # Now in A, convert to absolute before using #Tracker["delpreviousmax"] = False #Tracker["anger"] = -1.0 #Tracker["shifter"] = -1.0 #Tracker["saturatecrit"] = 0.95 #Tracker["pixercutoff"] = 2.0 #Tracker["directory"] = "" #Tracker["previousoutputdir"] = "" #Tracker["eliminated-outliers"] = False #Tracker["mainiteration"] = 0 #Tracker["movedback"] = False #Tracker["state"] = Tracker["constants"]["states"][0] #Tracker["global_resolution"] =0.0 Tracker["orgstack"] = orgstack #-------------------------------------------------------------------- # import from utilities from utilities import sample_down_1D_curve,get_initial_ID,remove_small_groups,print_upper_triangular_matrix,print_a_line_with_timestamp from utilities import print_dict,get_resolution_mrk01,partition_to_groups,partition_independent_runs,get_outliers from utilities import merge_groups, save_alist, margin_of_error, get_margin_of_error, do_two_way_comparison, select_two_runs, get_ali3d_params from utilities import counting_projections, unload_dict, load_dict, get_stat_proj, create_random_list, get_number_of_groups, recons_mref from utilities import apply_low_pass_filter, get_groups_from_partition, get_number_of_groups, get_complementary_elements_total, update_full_dict from utilities import count_chunk_members, set_filter_parameters_from_adjusted_fsc, adjust_fsc_down, get_two_chunks_from_stack ####------------------------------------------------------------------ # # Get the pixel size; if none, set to 1.0, and the original image size from utilities import get_shrink_data_huang if(myid == main_node): line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print(line+"Initialization of 3-D sorting") a = get_im(orgstack) nnxo = a.get_xsize() if( Tracker["nxinit"] > nnxo ): ERROR("Image size less than minimum permitted $d"%Tracker["nxinit"],"sxsort3d.py",1) nnxo = -1 else: if Tracker["constants"]["CTF"]: i = a.get_attr('ctf') pixel_size = i.apix fq = pixel_size/Tracker["fuse_freq"] else: pixel_size = 1.0 # No pixel size, fusing computed as 5 Fourier pixels fq = 5.0/nnxo del a else: nnxo = 0 fq = 0.0 pixel_size = 1.0 nnxo = bcast_number_to_all(nnxo, source_node = main_node) if( nnxo < 0 ): mpi_finalize() exit() pixel_size = bcast_number_to_all(pixel_size, source_node = main_node) fq = bcast_number_to_all(fq, source_node = main_node) if Tracker["constants"]["wn"]==0: Tracker["constants"]["nnxo"] = nnxo else: Tracker["constants"]["nnxo"] = Tracker["constants"]["wn"] nnxo = Tracker["constants"]["nnxo"] Tracker["constants"]["pixel_size"] = pixel_size Tracker["fuse_freq"] = fq del fq, nnxo, pixel_size if(Tracker["constants"]["radius"] < 1): Tracker["constants"]["radius"] = Tracker["constants"]["nnxo"]//2-2 elif((2*Tracker["constants"]["radius"] +2) > Tracker["constants"]["nnxo"]): ERROR("Particle radius set too large!","sxsort3d.py",1,myid) ####----------------------------------------------------------------------------------------- # Master directory if myid == main_node: if masterdir =="": timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime()) masterdir ="master_sort3d"+timestring li =len(masterdir) cmd="{} {}".format("mkdir", masterdir) os.system(cmd) else: li=0 li = mpi_bcast(li,1,MPI_INT,main_node,MPI_COMM_WORLD)[0] if li>0: masterdir = mpi_bcast(masterdir,li,MPI_CHAR,main_node,MPI_COMM_WORLD) import string masterdir = string.join(masterdir,"") if myid ==main_node: print_dict(Tracker["constants"],"Permanent settings of 3-D sorting program") ######### create a vstack from input stack to the local stack in masterdir # stack name set to default Tracker["constants"]["stack"] = "bdb:"+masterdir+"/rdata" Tracker["constants"]["ali3d"] = os.path.join(masterdir, "ali3d_init.txt") Tracker["constants"]["ctf_params"] = os.path.join(masterdir, "ctf_params.txt") Tracker["constants"]["partstack"] = Tracker["constants"]["ali3d"] # also serves for refinement if myid == main_node: total_stack = EMUtil.get_image_count(Tracker["orgstack"]) else: total_stack = 0 total_stack = bcast_number_to_all(total_stack, source_node = main_node) mpi_barrier(MPI_COMM_WORLD) from time import sleep while not os.path.exists(masterdir): print "Node ",myid," waiting..." sleep(5) mpi_barrier(MPI_COMM_WORLD) if myid == main_node: log_main.add("Sphire sort3d ") log_main.add("the sort3d master directory is "+masterdir) ##### ###---------------------------------------------------------------------------------- # Initial data analysis and handle two chunk files from random import shuffle # Compute the resolution #### make chunkdir dictionary for computing margin of error import user_functions user_func = user_functions.factory[Tracker["constants"]["user_func"]] chunk_dict = {} chunk_list = [] if myid == main_node: chunk_one = read_text_file(os.path.join(Tracker["constants"]["chunkdir"],"chunk0.txt")) chunk_two = read_text_file(os.path.join(Tracker["constants"]["chunkdir"],"chunk1.txt")) else: chunk_one = 0 chunk_two = 0 chunk_one = wrap_mpi_bcast(chunk_one, main_node) chunk_two = wrap_mpi_bcast(chunk_two, main_node) mpi_barrier(MPI_COMM_WORLD) ######################## Read/write bdb: data on main node ############################ if myid==main_node: if(orgstack[:4] == "bdb:"): cmd = "{} {} {}".format("e2bdb.py", orgstack,"--makevstack="+Tracker["constants"]["stack"]) else: cmd = "{} {} {}".format("sxcpy.py", orgstack, Tracker["constants"]["stack"]) cmdexecute(cmd) cmd = "{} {} {}".format("sxheader.py --params=xform.projection", "--export="+Tracker["constants"]["ali3d"],orgstack) cmdexecute(cmd) cmd = "{} {} {}".format("sxheader.py --params=ctf", "--export="+Tracker["constants"]["ctf_params"],orgstack) cmdexecute(cmd) mpi_barrier(MPI_COMM_WORLD) ########----------------------------------------------------------------------------- Tracker["total_stack"] = total_stack Tracker["constants"]["total_stack"] = total_stack Tracker["shrinkage"] = float(Tracker["nxinit"])/Tracker["constants"]["nnxo"] Tracker["radius"] = Tracker["constants"]["radius"]*Tracker["shrinkage"] if Tracker["constants"]["mask3D"]: Tracker["mask3D"] = os.path.join(masterdir,"smask.hdf") else: Tracker["mask3D"] = None if Tracker["constants"]["focus3Dmask"]: Tracker["focus3D"] = os.path.join(masterdir,"sfocus.hdf") else: Tracker["focus3D"] = None if myid == main_node: if Tracker["constants"]["mask3D"]: mask_3D = get_shrink_3dmask(Tracker["nxinit"],Tracker["constants"]["mask3D"]) mask_3D.write_image(Tracker["mask3D"]) if Tracker["constants"]["focus3Dmask"]: mask_3D = get_shrink_3dmask(Tracker["nxinit"],Tracker["constants"]["focus3Dmask"]) st = Util.infomask(mask_3D, None, True) if( st[0] == 0.0 ): ERROR("sxrsort3d","incorrect focused mask, after binarize all values zero",1) mask_3D.write_image(Tracker["focus3D"]) del mask_3D if Tracker["constants"]["PWadjustment"] !='': PW_dict = {} nxinit_pwsp = sample_down_1D_curve(Tracker["constants"]["nxinit"],Tracker["constants"]["nnxo"],Tracker["constants"]["PWadjustment"]) Tracker["nxinit_PW"] = os.path.join(masterdir,"spwp.txt") if myid == main_node: write_text_file(nxinit_pwsp,Tracker["nxinit_PW"]) PW_dict[Tracker["constants"]["nnxo"]] = Tracker["constants"]["PWadjustment"] PW_dict[Tracker["constants"]["nxinit"]] = Tracker["nxinit_PW"] Tracker["PW_dict"] = PW_dict mpi_barrier(MPI_COMM_WORLD) #-----------------------From two chunks to FSC, and low pass filter-----------------------------------------### for element in chunk_one: chunk_dict[element] = 0 for element in chunk_two: chunk_dict[element] = 1 chunk_list =[chunk_one, chunk_two] Tracker["chunk_dict"] = chunk_dict Tracker["P_chunk0"] = len(chunk_one)/float(total_stack) Tracker["P_chunk1"] = len(chunk_two)/float(total_stack) ### create two volumes to estimate resolution if myid == main_node: for index in xrange(2): write_text_file(chunk_list[index],os.path.join(masterdir,"chunk%01d.txt"%index)) mpi_barrier(MPI_COMM_WORLD) vols = [] for index in xrange(2): data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nxinit"], os.path.join(masterdir,"chunk%01d.txt"%index), Tracker["constants"]["partstack"],myid,main_node,nproc,preshift=True) vol = recons3d_4nn_ctf_MPI(myid=myid, prjlist=data,symmetry=Tracker["constants"]["sym"], finfo=None) if myid == main_node: vol.write_image(os.path.join(masterdir, "vol%d.hdf"%index)) vols.append(vol) mpi_barrier(MPI_COMM_WORLD) if myid ==main_node: low_pass, falloff,currentres = get_resolution_mrk01(vols,Tracker["constants"]["radius"],Tracker["constants"]["nxinit"],masterdir,Tracker["mask3D"]) if low_pass >Tracker["constants"]["low_pass_filter"]: low_pass= Tracker["constants"]["low_pass_filter"] else: low_pass =0.0 falloff =0.0 currentres =0.0 bcast_number_to_all(currentres,source_node = main_node) bcast_number_to_all(low_pass,source_node = main_node) bcast_number_to_all(falloff,source_node = main_node) Tracker["currentres"] = currentres Tracker["falloff"] = falloff if Tracker["constants"]["low_pass_filter"] ==-1.0: Tracker["low_pass_filter"] = min(.45,low_pass/Tracker["shrinkage"]) # no better than .45 else: Tracker["low_pass_filter"] = min(.45,Tracker["constants"]["low_pass_filter"]/Tracker["shrinkage"]) Tracker["lowpass"] = Tracker["low_pass_filter"] Tracker["falloff"] =.1 Tracker["global_fsc"] = os.path.join(masterdir, "fsc.txt") ############################################################################################ if myid == main_node: log_main.add("The command-line inputs are as following:") log_main.add("**********************************************************") for a in sys.argv: if myid == main_node:log_main.add(a) if myid == main_node: log_main.add("number of cpus used in this run is %d"%Tracker["constants"]["nproc"]) log_main.add("**********************************************************") from filter import filt_tanl ### START 3-D sorting if myid ==main_node: log_main.add("----------3-D sorting program------- ") log_main.add("current resolution %6.3f for images of original size in terms of absolute frequency"%Tracker["currentres"]) log_main.add("equivalent to %f Angstrom resolution"%(Tracker["constants"]["pixel_size"]/Tracker["currentres"]/Tracker["shrinkage"])) log_main.add("the user provided enforced low_pass_filter is %f"%Tracker["constants"]["low_pass_filter"]) #log_main.add("equivalent to %f Angstrom resolution"%(Tracker["constants"]["pixel_size"]/Tracker["constants"]["low_pass_filter"])) for index in xrange(2): filt_tanl(get_im(os.path.join(masterdir,"vol%01d.hdf"%index)), Tracker["low_pass_filter"],Tracker["falloff"]).write_image(os.path.join(masterdir, "volf%01d.hdf"%index)) mpi_barrier(MPI_COMM_WORLD) from utilities import get_input_from_string delta = get_input_from_string(Tracker["constants"]["delta"]) delta = delta[0] from utilities import even_angles n_angles = even_angles(delta, 0, 180) this_ali3d = Tracker["constants"]["ali3d"] sampled = get_stat_proj(Tracker,delta,this_ali3d) if myid ==main_node: nc = 0 for a in sampled: if len(sampled[a])>0: nc += 1 log_main.add("total sampled direction %10d at angle step %6.3f"%(len(n_angles), delta)) log_main.add("captured sampled directions %10d percentage covered by data %6.3f"%(nc,float(nc)/len(n_angles)*100)) number_of_images_per_group = Tracker["constants"]["number_of_images_per_group"] if myid ==main_node: log_main.add("user provided number_of_images_per_group %d"%number_of_images_per_group) Tracker["number_of_images_per_group"] = number_of_images_per_group number_of_groups = get_number_of_groups(total_stack,number_of_images_per_group) Tracker["number_of_groups"] = number_of_groups generation =0 partition_dict ={} full_dict ={} workdir =os.path.join(masterdir,"generation%03d"%generation) Tracker["this_dir"] = workdir if myid ==main_node: log_main.add("---- generation %5d"%generation) log_main.add("number of images per group is set as %d"%number_of_images_per_group) log_main.add("the initial number of groups is %10d "%number_of_groups) cmd="{} {}".format("mkdir",workdir) os.system(cmd) mpi_barrier(MPI_COMM_WORLD) list_to_be_processed = range(Tracker["constants"]["total_stack"]) Tracker["this_data_list"] = list_to_be_processed create_random_list(Tracker) ################################# full_dict ={} for iptl in xrange(Tracker["constants"]["total_stack"]): full_dict[iptl] = iptl Tracker["full_ID_dict"] = full_dict ################################# for indep_run in xrange(Tracker["constants"]["indep_runs"]): Tracker["this_particle_list"] = Tracker["this_indep_list"][indep_run] ref_vol = recons_mref(Tracker) if myid == main_node: log_main.add("independent run %10d"%indep_run) mpi_barrier(MPI_COMM_WORLD) Tracker["this_data_list"] = list_to_be_processed Tracker["total_stack"] = len(Tracker["this_data_list"]) Tracker["this_particle_text_file"] = os.path.join(workdir,"independent_list_%03d.txt"%indep_run) # for get_shrink_data if myid == main_node: write_text_file(Tracker["this_data_list"], Tracker["this_particle_text_file"]) mpi_barrier(MPI_COMM_WORLD) outdir = os.path.join(workdir, "EQ_Kmeans%03d"%indep_run) ref_vol = apply_low_pass_filter(ref_vol,Tracker) mref_ali3d_EQ_Kmeans(ref_vol, outdir, Tracker["this_particle_text_file"], Tracker) partition_dict[indep_run]=Tracker["this_partition"] Tracker["partition_dict"] = partition_dict Tracker["total_stack"] = len(Tracker["this_data_list"]) Tracker["this_total_stack"] = Tracker["total_stack"] ############################### do_two_way_comparison(Tracker) ############################### ref_vol_list = [] from time import sleep number_of_ref_class = [] for igrp in xrange(len(Tracker["two_way_stable_member"])): Tracker["this_data_list"] = Tracker["two_way_stable_member"][igrp] Tracker["this_data_list_file"] = os.path.join(workdir,"stable_class%d.txt"%igrp) if myid == main_node: write_text_file(Tracker["this_data_list"], Tracker["this_data_list_file"]) data,old_shifts = get_shrink_data_huang(Tracker,Tracker["nxinit"], Tracker["this_data_list_file"], Tracker["constants"]["partstack"], myid, main_node, nproc, preshift = True) volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"], finfo = None) ref_vol_list.append(volref) number_of_ref_class.append(len(Tracker["this_data_list"])) if myid == main_node: log_main.add("group %d members %d "%(igrp,len(Tracker["this_data_list"]))) Tracker["number_of_ref_class"] = number_of_ref_class nx_of_image = ref_vol_list[0].get_xsize() if Tracker["constants"]["PWadjustment"]: Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image] else: Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"] # no PW adjustment if myid == main_node: for iref in xrange(len(ref_vol_list)): refdata = [None]*4 refdata[0] = ref_vol_list[iref] refdata[1] = Tracker refdata[2] = Tracker["constants"]["myid"] refdata[3] = Tracker["constants"]["nproc"] volref = user_func(refdata) volref.write_image(os.path.join(workdir,"volf_stable.hdf"),iref) mpi_barrier(MPI_COMM_WORLD) Tracker["this_data_list"] = Tracker["this_accounted_list"] outdir = os.path.join(workdir,"Kmref") empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI(ref_vol_list,outdir,Tracker["this_accounted_text"],Tracker) Tracker["this_unaccounted_list"] = get_complementary_elements(list_to_be_processed,final_list) if myid == main_node: log_main.add("the number of particles not processed is %d"%len(Tracker["this_unaccounted_list"])) write_text_file(Tracker["this_unaccounted_list"],Tracker["this_unaccounted_text"]) update_full_dict(Tracker["this_unaccounted_list"], Tracker) ####################################### number_of_groups = len(res_groups) vol_list = [] number_of_ref_class = [] for igrp in xrange(number_of_groups): data,old_shifts = get_shrink_data_huang(Tracker, Tracker["constants"]["nnxo"], os.path.join(outdir,"Class%d.txt"%igrp), Tracker["constants"]["partstack"],myid,main_node,nproc,preshift = True) volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"], finfo=None) vol_list.append(volref) if( myid == main_node ): npergroup = len(read_text_file(os.path.join(outdir,"Class%d.txt"%igrp))) else: npergroup = 0 npergroup = bcast_number_to_all(npergroup, main_node ) number_of_ref_class.append(npergroup) Tracker["number_of_ref_class"] = number_of_ref_class mpi_barrier(MPI_COMM_WORLD) nx_of_image = vol_list[0].get_xsize() if Tracker["constants"]["PWadjustment"]: Tracker["PWadjustment"]=Tracker["PW_dict"][nx_of_image] else: Tracker["PWadjustment"]=Tracker["constants"]["PWadjustment"] if myid == main_node: for ivol in xrange(len(vol_list)): refdata =[None]*4 refdata[0] = vol_list[ivol] refdata[1] = Tracker refdata[2] = Tracker["constants"]["myid"] refdata[3] = Tracker["constants"]["nproc"] volref = user_func(refdata) volref.write_image(os.path.join(workdir,"volf_of_Classes.hdf"),ivol) log_main.add("number of unaccounted particles %10d"%len(Tracker["this_unaccounted_list"])) log_main.add("number of accounted particles %10d"%len(Tracker["this_accounted_list"])) Tracker["this_data_list"] = Tracker["this_unaccounted_list"] # reset parameters for the next round calculation Tracker["total_stack"] = len(Tracker["this_unaccounted_list"]) Tracker["this_total_stack"] = Tracker["total_stack"] number_of_groups = get_number_of_groups(len(Tracker["this_unaccounted_list"]),number_of_images_per_group) Tracker["number_of_groups"] = number_of_groups while number_of_groups >= 2 : generation +=1 partition_dict ={} workdir =os.path.join(masterdir,"generation%03d"%generation) Tracker["this_dir"] = workdir if myid ==main_node: log_main.add("*********************************************") log_main.add("----- generation %5d "%generation) log_main.add("number of images per group is set as %10d "%number_of_images_per_group) log_main.add("the number of groups is %10d "%number_of_groups) log_main.add(" number of particles for clustering is %10d"%Tracker["total_stack"]) cmd ="{} {}".format("mkdir",workdir) os.system(cmd) mpi_barrier(MPI_COMM_WORLD) create_random_list(Tracker) for indep_run in xrange(Tracker["constants"]["indep_runs"]): Tracker["this_particle_list"] = Tracker["this_indep_list"][indep_run] ref_vol = recons_mref(Tracker) if myid == main_node: log_main.add("independent run %10d"%indep_run) outdir = os.path.join(workdir, "EQ_Kmeans%03d"%indep_run) Tracker["this_data_list"] = Tracker["this_unaccounted_list"] #ref_vol=apply_low_pass_filter(ref_vol,Tracker) mref_ali3d_EQ_Kmeans(ref_vol,outdir,Tracker["this_unaccounted_text"],Tracker) partition_dict[indep_run] = Tracker["this_partition"] Tracker["this_data_list"] = Tracker["this_unaccounted_list"] Tracker["total_stack"] = len(Tracker["this_unaccounted_list"]) Tracker["partition_dict"] = partition_dict Tracker["this_total_stack"] = Tracker["total_stack"] total_list_of_this_run = Tracker["this_unaccounted_list"] ############################### do_two_way_comparison(Tracker) ############################### ref_vol_list = [] number_of_ref_class = [] for igrp in xrange(len(Tracker["two_way_stable_member"])): Tracker["this_data_list"] = Tracker["two_way_stable_member"][igrp] Tracker["this_data_list_file"] = os.path.join(workdir,"stable_class%d.txt"%igrp) if myid == main_node: write_text_file(Tracker["this_data_list"], Tracker["this_data_list_file"]) mpi_barrier(MPI_COMM_WORLD) data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nxinit"],Tracker["this_data_list_file"],Tracker["constants"]["partstack"],myid,main_node,nproc,preshift = True) volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"],finfo= None) #volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1) if myid == main_node:volref.write_image(os.path.join(workdir,"vol_stable.hdf"),iref) #volref = resample(volref,Tracker["shrinkage"]) ref_vol_list.append(volref) number_of_ref_class.append(len(Tracker["this_data_list"])) mpi_barrier(MPI_COMM_WORLD) Tracker["number_of_ref_class"] = number_of_ref_class Tracker["this_data_list"] = Tracker["this_accounted_list"] outdir = os.path.join(workdir,"Kmref") empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI(ref_vol_list,outdir,Tracker["this_accounted_text"],Tracker) # calculate the 3-D structure of original image size for each group number_of_groups = len(res_groups) Tracker["this_unaccounted_list"] = get_complementary_elements(total_list_of_this_run,final_list) if myid == main_node: log_main.add("the number of particles not processed is %d"%len(Tracker["this_unaccounted_list"])) write_text_file(Tracker["this_unaccounted_list"],Tracker["this_unaccounted_text"]) mpi_barrier(MPI_COMM_WORLD) update_full_dict(Tracker["this_unaccounted_list"],Tracker) vol_list = [] for igrp in xrange(number_of_groups): data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nnxo"], os.path.join(outdir,"Class%d.txt"%igrp), Tracker["constants"]["partstack"], myid, main_node, nproc,preshift = True) volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"],finfo= None) vol_list.append(volref) mpi_barrier(MPI_COMM_WORLD) nx_of_image=ref_vol_list[0].get_xsize() if Tracker["constants"]["PWadjustment"]: Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image] else: Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"] if myid == main_node: for ivol in xrange(len(vol_list)): refdata = [None]*4 refdata[0] = vol_list[ivol] refdata[1] = Tracker refdata[2] = Tracker["constants"]["myid"] refdata[3] = Tracker["constants"]["nproc"] volref = user_func(refdata) volref.write_image(os.path.join(workdir, "volf_of_Classes.hdf"),ivol) log_main.add("number of unaccounted particles %10d"%len(Tracker["this_unaccounted_list"])) log_main.add("number of accounted particles %10d"%len(Tracker["this_accounted_list"])) del vol_list mpi_barrier(MPI_COMM_WORLD) number_of_groups = get_number_of_groups(len(Tracker["this_unaccounted_list"]),number_of_images_per_group) Tracker["number_of_groups"] = number_of_groups Tracker["this_data_list"] = Tracker["this_unaccounted_list"] Tracker["total_stack"] = len(Tracker["this_unaccounted_list"]) if Tracker["constants"]["unaccounted"]: data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nnxo"],Tracker["this_unaccounted_text"],Tracker["constants"]["partstack"],myid,main_node,nproc,preshift = True) volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"],finfo= None) nx_of_image = volref.get_xsize() if Tracker["constants"]["PWadjustment"]: Tracker["PWadjustment"]=Tracker["PW_dict"][nx_of_image] else: Tracker["PWadjustment"]=Tracker["constants"]["PWadjustment"] if( myid == main_node ): refdata = [None]*4 refdata[0] = volref refdata[1] = Tracker refdata[2] = Tracker["constants"]["myid"] refdata[3] = Tracker["constants"]["nproc"] volref = user_func(refdata) #volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1) volref.write_image(os.path.join(workdir,"volf_unaccounted.hdf")) # Finish program if myid ==main_node: log_main.add("sxsort3d finishes") mpi_barrier(MPI_COMM_WORLD) from mpi import mpi_finalize mpi_finalize() exit()
def main(args): from utilities import if_error_then_all_processes_exit_program, write_text_row, drop_image, model_gauss_noise, get_im, set_params_proj, wrap_mpi_bcast, model_circle from logger import Logger, BaseLogger_Files from mpi import mpi_init, mpi_finalize, MPI_COMM_WORLD, mpi_comm_rank, mpi_comm_size, mpi_barrier import user_functions import sys import os from applications import MPI_start_end from optparse import OptionParser, SUPPRESS_HELP from global_def import SPARXVERSION from EMAN2 import EMData from multi_shc import multi_shc progname = os.path.basename(sys.argv[0]) usage = progname + " stack [output_directory] --ir=inner_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --center=center_type --maxit1=max_iter1 --maxit2=max_iter2 --L2threshold=0.1 --ref_a=S --sym=c1" usage += """ stack 2D images in a stack file: (default required string) directory output directory name: into which the results will be written (if it does not exist, it will be created, if it does exist, the results will be written possibly overwriting previous results) (default required string) """ parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--radius", type="int", help= "radius of the particle: has to be less than < int(nx/2)-1 (default required int)" ) parser.add_option( "--xr", type="string", default='0', help= "range for translation search in x direction: search is +/xr in pixels (default '0')" ) parser.add_option( "--yr", type="string", default='0', help= "range for translation search in y direction: if omitted will be set to xr, search is +/yr in pixels (default '0')" ) parser.add_option("--mask3D", type="string", default=None, help="3D mask file: (default sphere)") parser.add_option( "--moon_elimination", type="string", default='', help= "elimination of disconnected pieces: two arguments: mass in KDa and pixel size in px/A separated by comma, no space (default none)" ) parser.add_option( "--ir", type="int", default=1, help="inner radius for rotational search: > 0 (default 1)") # 'radius' and 'ou' are the same as per Pawel's request; 'ou' is hidden from the user # the 'ou' variable is not changed to 'radius' in the 'sparx' program. This change is at interface level only for sxviper. ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--ou", type="int", default=-1, help=SUPPRESS_HELP) parser.add_option( "--rs", type="int", default=1, help="step between rings in rotational search: >0 (default 1)") parser.add_option( "--ts", type="string", default='1.0', help= "step size of the translation search in x-y directions: search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default '1.0')" ) parser.add_option( "--delta", type="string", default='2.0', help="angular step of reference projections: (default '2.0')") parser.add_option( "--center", type="float", default=-1.0, help= "centering of 3D template: average shift method; 0: no centering; 1: center of gravity (default -1.0)" ) parser.add_option( "--maxit1", type="int", default=400, help= "maximum number of iterations performed for the GA part: (default 400)" ) parser.add_option( "--maxit2", type="int", default=50, help= "maximum number of iterations performed for the finishing up part: (default 50)" ) parser.add_option( "--L2threshold", type="float", default=0.03, help= "stopping criterion of GA: given as a maximum relative dispersion of volumes' L2 norms: (default 0.03)" ) parser.add_option( "--ref_a", type="string", default='S', help= "method for generating the quasi-uniformly distributed projection directions: (default S)" ) parser.add_option( "--sym", type="string", default='c1', help="point-group symmetry of the structure: (default c1)") # parser.add_option("--function", type="string", default="ref_ali3d", help="name of the reference preparation function (ref_ali3d by default)") ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--function", type="string", default="ref_ali3d", help=SUPPRESS_HELP) parser.add_option( "--nruns", type="int", default=6, help= "GA population: aka number of quasi-independent volumes (default 6)") parser.add_option( "--doga", type="float", default=0.1, help= "do GA when fraction of orientation changes less than 1.0 degrees is at least doga: (default 0.1)" ) ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--npad", type="int", default=2, help="padding size for 3D reconstruction (default=2)") parser.add_option( "--fl", type="float", default=0.25, help= "cut-off frequency applied to the template volume: using a hyperbolic tangent low-pass filter (default 0.25)" ) parser.add_option( "--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass filter: (default 0.1)") parser.add_option( "--pwreference", type="string", default='', help="text file with a reference power spectrum: (default none)") parser.add_option("--debug", action="store_true", default=False, help="debug info printout: (default False)") ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help=SUPPRESS_HELP) #parser.add_option("--an", type="string", default= "-1", help="NOT USED angular neighborhood for local searches (phi and theta)") #parser.add_option("--CTF", action="store_true", default=False, help="NOT USED Consider CTF correction during the alignment ") #parser.add_option("--snr", type="float", default= 1.0, help="NOT USED Signal-to-Noise Ratio of the data (default 1.0)") # (options, args) = parser.parse_args(sys.argv[1:]) required_option_list = ['radius'] (options, args) = parser.parse_args(args) # option_dict = vars(options) # print parser if options.return_options: return parser if options.moon_elimination == "": options.moon_elimination = [] else: options.moon_elimination = map(float, options.moon_elimination.split(",")) # Making sure all required options appeared. for required_option in required_option_list: if not options.__dict__[required_option]: print "\n ==%s== mandatory option is missing.\n" % required_option print "Please run '" + progname + " -h' for detailed options" return 1 if len(args) < 2 or len(args) > 3: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" return 1 mpi_init(0, []) log = Logger(BaseLogger_Files()) # 'radius' and 'ou' are the same as per Pawel's request; 'ou' is hidden from the user # the 'ou' variable is not changed to 'radius' in the 'sparx' program. This change is at interface level only for sxviper. options.ou = options.radius runs_count = options.nruns mpi_rank = mpi_comm_rank(MPI_COMM_WORLD) mpi_size = mpi_comm_size( MPI_COMM_WORLD) # Total number of processes, passed by --np option. if mpi_rank == 0: all_projs = EMData.read_images(args[0]) subset = range(len(all_projs)) # if mpi_size > len(all_projs): # ERROR('Number of processes supplied by --np needs to be less than or equal to %d (total number of images) ' % len(all_projs), 'sxviper', 1) # mpi_finalize() # return else: all_projs = None subset = None outdir = args[1] if mpi_rank == 0: if mpi_size % options.nruns != 0: ERROR( 'Number of processes needs to be a multiple of total number of runs. Total runs by default are 3, you can change it by specifying --nruns option.', 'sxviper', 1) mpi_finalize() return if os.path.exists(outdir): ERROR( 'Output directory exists, please change the name and restart the program', "sxviper", 1) mpi_finalize() return os.mkdir(outdir) import global_def global_def.LOGFILE = os.path.join(outdir, global_def.LOGFILE) mpi_barrier(MPI_COMM_WORLD) if outdir[-1] != "/": outdir += "/" log.prefix = outdir # if len(args) > 2: # ref_vol = get_im(args[2]) # else: ref_vol = None options.user_func = user_functions.factory[options.function] options.CTF = False options.snr = 1.0 options.an = -1.0 from multi_shc import multi_shc out_params, out_vol, out_peaks = multi_shc(all_projs, subset, runs_count, options, mpi_comm=MPI_COMM_WORLD, log=log, ref_vol=ref_vol) mpi_finalize()
def main(): def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror): # the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D if mirror: m = 1 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0) else: m = 0 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0) return alpha, sx, sy, m progname = os.path.basename(sys.argv[0]) usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl= --aa= --sym=symmetry --CTF" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--output_dir", type="string" , default="./", help="Output directory") parser.add_option("--ave2D", type="string" , default=False, help="Write to the disk a stack of 2D averages") parser.add_option("--var2D", type="string" , default=False, help="Write to the disk a stack of 2D variances") parser.add_option("--ave3D", type="string" , default=False, help="Write to the disk reconstructed 3D average") parser.add_option("--var3D", type="string" , default=False, help="Compute 3D variability (time consuming!)") parser.add_option("--img_per_grp", type="int" , default=100, help="Number of neighbouring projections.(Default is 100)") parser.add_option("--no_norm", action="store_true", default=False, help="Do not use normalization.(Default is to apply normalization)") #parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" ) parser.add_option("--npad", type="int" , default=2 , help="Number of time to pad the original images.(Default is 2 times padding)") parser.add_option("--sym" , type="string" , default="c1", help="Symmetry. (Default is no symmetry)") parser.add_option("--fl", type="float" , default=0.0, help="Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)") parser.add_option("--aa", type="float" , default=0.02 , help="Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)") parser.add_option("--CTF", action="store_true", default=False, help="Use CFT correction.(Default is no CTF correction)") #parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version") #parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" ) #parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" ) #parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" ) #parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" ) parser.add_option("--VAR" , action="store_true", default=False, help="Stack of input consists of 2D variances (Default False)") parser.add_option("--decimate", type ="float", default=0.25, help="Image decimate rate, a number less than 1. (Default is 0.25)") parser.add_option("--window", type ="int", default=0, help="Target image size relative to original image size. (Default value is zero.)") #parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)") #parser.add_option("--nvec", type="int" , default=0 , help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)") parser.add_option("--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)") parser.add_option("--overhead", type ="float", default=0.5, help="python overhead per CPU.") (options,args) = parser.parse_args() ##### from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX #from mpi import * from applications import MPI_start_end from reconstruction import recons3d_em, recons3d_em_MPI from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI from utilities import print_begin_msg, print_end_msg, print_msg from utilities import read_text_row, get_image, get_im, wrap_mpi_send, wrap_mpi_recv from utilities import bcast_EMData_to_all, bcast_number_to_all from utilities import get_symt # This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015 from EMAN2db import db_open_dict # Set up global variables related to bdb cache if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() # Set up global variables related to ERROR function global_def.BATCH = True # detect if program is running under MPI RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ if RUNNING_UNDER_MPI: global_def.MPI = True if options.output_dir =="./": current_output_dir = os.path.abspath(options.output_dir) else: current_output_dir = options.output_dir if options.symmetrize : if RUNNING_UNDER_MPI: try: sys.argv = mpi_init(len(sys.argv), sys.argv) try: number_of_proc = mpi_comm_size(MPI_COMM_WORLD) if( number_of_proc > 1 ): ERROR("Cannot use more than one CPU for symmetry preparation","sx3dvariability",1) except: pass except: pass if not os.path.exists(current_output_dir): os.mkdir(current_output_dir) # Input #instack = "Clean_NORM_CTF_start_wparams.hdf" #instack = "bdb:data" from logger import Logger,BaseLogger_Files if os.path.exists(os.path.join(current_output_dir, "log.txt")): os.remove(os.path.join(current_output_dir, "log.txt")) log_main=Logger(BaseLogger_Files()) log_main.prefix = os.path.join(current_output_dir, "./") instack = args[0] sym = options.sym.lower() if( sym == "c1" ): ERROR("There is no need to symmetrize stack for C1 symmetry","sx3dvariability",1) line ="" for a in sys.argv: line +=" "+a log_main.add(line) if(instack[:4] !="bdb:"): #if output_dir =="./": stack = "bdb:data" stack = "bdb:"+current_output_dir+"/data" delete_bdb(stack) junk = cmdexecute("sxcpy.py "+instack+" "+stack) else: stack = instack qt = EMUtil.get_all_attributes(stack,'xform.projection') na = len(qt) ts = get_symt(sym) ks = len(ts) angsa = [None]*na for k in range(ks): #Qfile = "Q%1d"%k #if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k) Qfile = os.path.join(current_output_dir, "Q%1d"%k) #delete_bdb("bdb:Q%1d"%k) delete_bdb("bdb:"+Qfile) #junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k) junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:"+Qfile) #DB = db_open_dict("bdb:Q%1d"%k) DB = db_open_dict("bdb:"+Qfile) for i in range(na): ut = qt[i]*ts[k] DB.set_attr(i, "xform.projection", ut) #bt = ut.get_params("spider") #angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]] #write_text_row(angsa, 'ptsma%1d.txt'%k) #junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k) #junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k)) DB.close() #if options.output_dir =="./": delete_bdb("bdb:sdata") delete_bdb("bdb:" + current_output_dir + "/"+"sdata") #junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q") sdata = "bdb:"+current_output_dir+"/"+"sdata" print(sdata) junk = cmdexecute("e2bdb.py " + current_output_dir +" --makevstack="+sdata +" --filt=Q") #junk = cmdexecute("ls EMAN2DB/sdata*") #a = get_im("bdb:sdata") a = get_im(sdata) a.set_attr("variabilitysymmetry",sym) #a.write_image("bdb:sdata") a.write_image(sdata) else: from fundamentals import window2d sys.argv = mpi_init(len(sys.argv), sys.argv) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) main_node = 0 shared_comm = mpi_comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL) myid_on_node = mpi_comm_rank(shared_comm) no_of_processes_per_group = mpi_comm_size(shared_comm) masters_from_groups_vs_everything_else_comm = mpi_comm_split(MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node) color, no_of_groups, balanced_processor_load_on_nodes = get_colors_and_subsets(main_node, MPI_COMM_WORLD, myid, \ shared_comm, myid_on_node, masters_from_groups_vs_everything_else_comm) overhead_loading = options.overhead*number_of_proc #memory_per_node = options.memory_per_node #if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group keepgoing = 1 current_window = options.window current_decimate = options.decimate if len(args) == 1: stack = args[0] else: print(( "usage: " + usage)) print(( "Please run '" + progname + " -h' for detailed options")) return 1 t0 = time() # obsolete flags options.MPI = True #options.nvec = 0 options.radiuspca = -1 options.iter = 40 options.abs = 0.0 options.squ = 0.0 if options.fl > 0.0 and options.aa == 0.0: ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid) #if options.VAR and options.SND: # ERROR("Only one of var and SND can be set!", "sx3dvariability", myid) if options.VAR and (options.ave2D or options.ave3D or options.var2D): ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid) #if options.SND and (options.ave2D or options.ave3D): # ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid) #if options.nvec > 0 : # ERROR("PCA option not implemented", "sx3dvariability", 1, myid) #if options.nvec > 0 and options.ave3D == None: # ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", 1, myid) if current_decimate>1.0 or current_decimate<0.0: ERROR("Decimate rate should be a value between 0.0 and 1.0", "sx3dvariability", 1, myid) if current_window < 0.0: ERROR("Target window size should be always larger than zero", "sx3dvariability", 1, myid) if myid == main_node: img = get_image(stack, 0) nx = img.get_xsize() ny = img.get_ysize() if(min(nx, ny) < current_window): keepgoing = 0 keepgoing = bcast_number_to_all(keepgoing, main_node, MPI_COMM_WORLD) if keepgoing == 0: ERROR("The target window size cannot be larger than the size of decimated image", "sx3dvariability", 1, myid) import string options.sym = options.sym.lower() # if global_def.CACHE_DISABLE: # from utilities import disable_bdb_cache # disable_bdb_cache() # global_def.BATCH = True if myid == main_node: if not os.path.exists(current_output_dir): os.mkdir(current_output_dir)# Never delete output_dir in the program! img_per_grp = options.img_per_grp #nvec = options.nvec radiuspca = options.radiuspca from logger import Logger,BaseLogger_Files #if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt")) log_main=Logger(BaseLogger_Files()) log_main.prefix = os.path.join(current_output_dir, "./") if myid == main_node: line = "" for a in sys.argv: line +=" "+a log_main.add(line) log_main.add("-------->>>Settings given by all options<<<-------") log_main.add("Symmetry : %s"%options.sym) log_main.add("Input stack : %s"%stack) log_main.add("Output_dir : %s"%current_output_dir) if options.ave3D: log_main.add("Ave3d : %s"%options.ave3D) if options.var3D: log_main.add("Var3d : %s"%options.var3D) if options.ave2D: log_main.add("Ave2D : %s"%options.ave2D) if options.var2D: log_main.add("Var2D : %s"%options.var2D) if options.VAR: log_main.add("VAR : True") else: log_main.add("VAR : False") if options.CTF: log_main.add("CTF correction : True ") else: log_main.add("CTF correction : False ") log_main.add("Image per group : %5d"%options.img_per_grp) log_main.add("Image decimate rate : %4.3f"%current_decimate) log_main.add("Low pass filter : %4.3f"%options.fl) current_fl = options.fl if current_fl == 0.0: current_fl = 0.5 log_main.add("Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"%round((current_fl*current_decimate),3)) log_main.add("Window size : %5d "%current_window) log_main.add("sx3dvariability begins") symbaselen = 0 if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() nnxo = nx nnyo = ny if options.sym != "c1" : imgdata = get_im(stack) try: i = imgdata.get_attr("variabilitysymmetry").lower() if(i != options.sym): ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", 1, myid) except: ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", 1, myid) from utilities import get_symt i = len(get_symt(options.sym)) if((nima/i)*i != nima): ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", 1, myid) symbaselen = nima/i else: symbaselen = nima else: nima = 0 nx = 0 ny = 0 nnxo = 0 nnyo = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) nnxo = bcast_number_to_all(nnxo) nnyo = bcast_number_to_all(nnyo) if current_window > max(nx, ny): ERROR("Window size is larger than the original image size", "sx3dvariability", 1) if current_decimate == 1.: if current_window !=0: nx = current_window ny = current_window else: if current_window == 0: nx = int(nx*current_decimate+0.5) ny = int(ny*current_decimate+0.5) else: nx = int(current_window*current_decimate+0.5) ny = nx symbaselen = bcast_number_to_all(symbaselen) # check FFT prime number from fundamentals import smallprime is_fft_friendly = (nx == smallprime(nx)) if not is_fft_friendly: if myid == main_node: log_main.add("The target image size is not a product of small prime numbers") log_main.add("Program adjusts the input settings!") ### two cases if current_decimate == 1.: nx = smallprime(nx) ny = nx current_window = nx # update if myid == main_node: log_main.add("The window size is updated to %d."%current_window) else: if current_window == 0: nx = smallprime(int(nx*current_decimate+0.5)) current_decimate = float(nx)/nnxo ny = nx if (myid == main_node): log_main.add("The decimate rate is updated to %f."%current_decimate) else: nx = smallprime(int(current_window*current_decimate+0.5)) ny = nx current_window = int(nx/current_decimate+0.5) if (myid == main_node): log_main.add("The window size is updated to %d."%current_window) if myid == main_node: log_main.add("The target image size is %d"%nx) if radiuspca == -1: radiuspca = nx/2-2 if myid == main_node: log_main.add("%-70s: %d\n"%("Number of projection", nima)) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) """ if options.SND: from projection import prep_vol, prgs from statistics import im_diff from utilities import get_im, model_circle, get_params_proj, set_params_proj from utilities import get_ctf, generate_ctf from filter import filt_ctf imgdata = EMData.read_images(stack, range(img_begin, img_end)) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) bcast_EMData_to_all(vol, myid) volft, kb = prep_vol(vol) mask = model_circle(nx/2-2, nx, ny) varList = [] for i in xrange(img_begin, img_end): phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin]) ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y]) if options.CTF: ctf_params = get_ctf(imgdata[i-img_begin]) ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params)) diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask) diff2 = diff*diff set_params_proj(diff2, [phi, theta, psi, s2x, s2y]) varList.append(diff2) mpi_barrier(MPI_COMM_WORLD) """ if options.VAR: # 2D variance images have no shifts #varList = EMData.read_images(stack, range(img_begin, img_end)) from EMAN2 import Region for index_of_particle in range(img_begin,img_end): image = get_im(stack, index_of_proj) if current_window > 0: varList.append(fdecimate(window2d(image,current_window,current_window), nx,ny)) else: varList.append(fdecimate(image, nx,ny)) else: from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2 from utilities import model_blank, nearest_proj, model_circle, write_text_row, wrap_mpi_gatherv from applications import pca from statistics import avgvar, avgvar_ctf, ccc from filter import filt_tanl from morphology import threshold, square_root from projection import project, prep_vol, prgs from sets import Set from utilities import wrap_mpi_recv, wrap_mpi_bcast, wrap_mpi_send import numpy as np if myid == main_node: t1 = time() proj_angles = [] aveList = [] tab = EMUtil.get_all_attributes(stack, 'xform.projection') for i in range(nima): t = tab[i].get_params('spider') phi = t['phi'] theta = t['theta'] psi = t['psi'] x = theta if x > 90.0: x = 180.0 - x x = x*10000+psi proj_angles.append([x, t['phi'], t['theta'], t['psi'], i]) t2 = time() log_main.add( "%-70s: %d\n"%("Number of neighboring projections", img_per_grp)) log_main.add("...... Finding neighboring projections\n") log_main.add( "Number of images per group: %d"%img_per_grp) log_main.add( "Now grouping projections") proj_angles.sort() proj_angles_list = np.full((nima, 4), 0.0, dtype=np.float32) for i in range(nima): proj_angles_list[i][0] = proj_angles[i][1] proj_angles_list[i][1] = proj_angles[i][2] proj_angles_list[i][2] = proj_angles[i][3] proj_angles_list[i][3] = proj_angles[i][4] else: proj_angles_list = 0 proj_angles_list = wrap_mpi_bcast(proj_angles_list, main_node, MPI_COMM_WORLD) proj_angles = [] for i in range(nima): proj_angles.append([proj_angles_list[i][0], proj_angles_list[i][1], proj_angles_list[i][2], int(proj_angles_list[i][3])]) del proj_angles_list proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end)) all_proj = Set() for im in proj_list: for jm in im: all_proj.add(proj_angles[jm][3]) all_proj = list(all_proj) index = {} for i in range(len(all_proj)): index[all_proj[i]] = i mpi_barrier(MPI_COMM_WORLD) if myid == main_node: log_main.add("%-70s: %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2)) log_main.add("%-70s: %d\n"%("Number of groups processed on the main node", len(proj_list))) log_main.add("Grouping projections took: %12.1f [m]"%((time()-t2)/60.)) log_main.add("Number of groups on main node: ", len(proj_list)) mpi_barrier(MPI_COMM_WORLD) if myid == main_node: log_main.add("...... Calculating the stack of 2D variances \n") # Memory estimation. There are two memory consumption peaks # peak 1. Compute ave, var; # peak 2. Var volume reconstruction; # proj_params = [0.0]*(nima*5) aveList = [] varList = [] #if nvec > 0: eigList = [[] for i in range(nvec)] dnumber = len(all_proj)# all neighborhood set for assigned to myid pnumber = len(proj_list)*2. + img_per_grp # aveList and varList tnumber = dnumber+pnumber vol_size2 = nx**3*4.*8/1.e9 vol_size1 = 2.*nnxo**3*4.*8/1.e9 proj_size = nnxo*nnyo*len(proj_list)*4.*2./1.e9 # both aveList and varList orig_data_size = nnxo*nnyo*4.*tnumber/1.e9 reduced_data_size = nx*nx*4.*tnumber/1.e9 full_data = np.full((number_of_proc, 2), -1., dtype=np.float16) full_data[myid] = orig_data_size, reduced_data_size if myid != main_node: wrap_mpi_send(full_data, main_node, MPI_COMM_WORLD) if myid == main_node: for iproc in range(number_of_proc): if iproc != main_node: dummy = wrap_mpi_recv(iproc, MPI_COMM_WORLD) full_data[np.where(dummy>-1)] = dummy[np.where(dummy>-1)] del dummy mpi_barrier(MPI_COMM_WORLD) full_data = wrap_mpi_bcast(full_data, main_node, MPI_COMM_WORLD) # find the CPU with heaviest load minindx = np.argsort(full_data, 0) heavy_load_myid = minindx[-1][1] total_mem = sum(full_data) if myid == main_node: if current_window == 0: log_main.add("Nx: current image size = %d. Decimated by %f from %d"%(nx, current_decimate, nnxo)) else: log_main.add("Nx: current image size = %d. Windowed to %d, and decimated by %f from %d"%(nx, current_window, current_decimate, nnxo)) log_main.add("Nproj: number of particle images.") log_main.add("Navg: number of 2D average images.") log_main.add("Nvar: number of 2D variance images.") log_main.add("Img_per_grp: user defined image per group for averaging = %d"%img_per_grp) log_main.add("Overhead: total python overhead memory consumption = %f"%overhead_loading) log_main.add("Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"%\ (total_mem[1] + overhead_loading)) del full_data mpi_barrier(MPI_COMM_WORLD) if myid == heavy_load_myid: log_main.add("Begin reading and preprocessing images on processor. Wait... ") ttt = time() #imgdata = EMData.read_images(stack, all_proj) imgdata = [ None for im in range(len(all_proj))] for index_of_proj in range(len(all_proj)): #image = get_im(stack, all_proj[index_of_proj]) if( current_window > 0): imgdata[index_of_proj] = fdecimate(window2d(get_im(stack, all_proj[index_of_proj]),current_window,current_window), nx, ny) else: imgdata[index_of_proj] = fdecimate(get_im(stack, all_proj[index_of_proj]), nx, ny) if (current_decimate> 0.0 and options.CTF): ctf = imgdata[index_of_proj].get_attr("ctf") ctf.apix = ctf.apix/current_decimate imgdata[index_of_proj].set_attr("ctf", ctf) if myid == heavy_load_myid and index_of_proj%100 == 0: log_main.add(" ...... %6.2f%% "%(index_of_proj/float(len(all_proj))*100.)) mpi_barrier(MPI_COMM_WORLD) if myid == heavy_load_myid: log_main.add("All_proj preprocessing cost %7.2f m"%((time()-ttt)/60.)) log_main.add("Wait untill reading on all CPUs done...") ''' imgdata2 = EMData.read_images(stack, range(img_begin, img_end)) if options.fl > 0.0: for k in xrange(len(imgdata2)): imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) if myid == main_node: vol.write_image("vol_ctf.hdf") print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf")) del vol, imgdata2 mpi_barrier(MPI_COMM_WORLD) ''' from applications import prepare_2d_forPCA from utilities import model_blank from EMAN2 import Transform if not options.no_norm: mask = model_circle(nx/2-2, nx, nx) if options.CTF: from utilities import pad from filter import filt_ctf from filter import filt_tanl if myid == heavy_load_myid: log_main.add("Start computing 2D aveList and varList. Wait...") ttt = time() inner=nx//2-4 outer=inner+2 xform_proj_for_2D = [ None for i in range(len(proj_list))] for i in range(len(proj_list)): ki = proj_angles[proj_list[i][0]][3] if ki >= symbaselen: continue mi = index[ki] dpar = Util.get_transform_params(imgdata[mi], "xform.projection", "spider") phiM, thetaM, psiM, s2xM, s2yM = dpar["phi"],dpar["theta"],dpar["psi"],-dpar["tx"]*current_decimate,-dpar["ty"]*current_decimate grp_imgdata = [] for j in range(img_per_grp): mj = index[proj_angles[proj_list[i][j]][3]] cpar = Util.get_transform_params(imgdata[mj], "xform.projection", "spider") alpha, sx, sy, mirror = params_3D_2D_NEW(cpar["phi"], cpar["theta"],cpar["psi"], -cpar["tx"]*current_decimate, -cpar["ty"]*current_decimate, mirror_list[i][j]) if thetaM <= 90: if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM - cpar["phi"]), 0.0, 0.0, 1.0) else: if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM- cpar["phi"]), 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM - cpar["phi"])), 0.0, 0.0, 1.0) imgdata[mj].set_attr("xform.align2d", Transform({"type":"2D","alpha":alpha,"tx":sx,"ty":sy,"mirror":mirror,"scale":1.0})) grp_imgdata.append(imgdata[mj]) if not options.no_norm: for k in range(img_per_grp): ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False) grp_imgdata[k] -= ave grp_imgdata[k] /= std if options.fl > 0.0: for k in range(img_per_grp): grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) # Because of background issues, only linear option works. if options.CTF: ave, var = aves_wiener(grp_imgdata, SNR = 1.0e5, interpolation_method = "linear") else: ave, var = ave_var(grp_imgdata) # Switch to std dev # threshold is not really needed,it is just in case due to numerical accuracy something turns out negative. var = square_root(threshold(var)) set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0]) set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0]) aveList.append(ave) varList.append(var) xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0] ''' if nvec > 0: eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True) for k in range(nvec): set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0]) eigList[k].append(eig[k]) """ if myid == 0 and i == 0: for k in xrange(nvec): eig[k].write_image("eig.hdf", k) """ ''' if (myid == heavy_load_myid) and (i%100 == 0): log_main.add(" ......%6.2f%% "%(i/float(len(proj_list))*100.)) del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index if not options.no_norm: del mask if myid == main_node: del tab # At this point, all averages and variances are computed mpi_barrier(MPI_COMM_WORLD) if (myid == heavy_load_myid): log_main.add("Computing aveList and varList took %12.1f [m]"%((time()-ttt)/60.)) xform_proj_for_2D = wrap_mpi_gatherv(xform_proj_for_2D, main_node, MPI_COMM_WORLD) if (myid == main_node): write_text_row(xform_proj_for_2D, os.path.join(current_output_dir, "params.txt")) del xform_proj_for_2D mpi_barrier(MPI_COMM_WORLD) if options.ave2D: from fundamentals import fpol from applications import header if myid == main_node: log_main.add("Compute ave2D ... ") km = 0 for i in range(number_of_proc): if i == main_node : for im in range(len(aveList)): aveList[im].write_image(os.path.join(current_output_dir, options.ave2D), km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in range(nl): ave = recv_EMData(i, im+i+70000) """ nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('members', map(int, members)) members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('pix_err', map(float, members)) members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('refprojdir', map(float, members)) """ tmpvol=fpol(ave, nx, nx,1) tmpvol.write_image(os.path.join(current_output_dir, options.ave2D), km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in range(len(aveList)): send_EMData(aveList[im], main_node,im+myid+70000) """ members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) members = aveList[im].get_attr('pix_err') mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) except: mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) """ if myid == main_node: header(os.path.join(current_output_dir, options.ave2D), params='xform.projection', fimport = os.path.join(current_output_dir, "params.txt")) mpi_barrier(MPI_COMM_WORLD) if options.ave3D: from fundamentals import fpol t5 = time() if myid == main_node: log_main.add("Reconstruct ave3D ... ") ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(ave3D, myid) if myid == main_node: if current_decimate != 1.0: ave3D = resample(ave3D, 1./current_decimate) ave3D = fpol(ave3D, nnxo, nnxo, nnxo) # always to the orignal image size set_pixel_size(ave3D, 1.0) ave3D.write_image(os.path.join(current_output_dir, options.ave3D)) log_main.add("Ave3D reconstruction took %12.1f [m]"%((time()-t5)/60.0)) log_main.add("%-70s: %s\n"%("The reconstructed ave3D is saved as ", options.ave3D)) mpi_barrier(MPI_COMM_WORLD) del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList ''' if nvec > 0: for k in range(nvec): if myid == main_node:log_main.add("Reconstruction eigenvolumes", k) cont = True ITER = 0 mask2d = model_circle(radiuspca, nx, nx) while cont: #print "On node %d, iteration %d"%(myid, ITER) eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(eig3D, myid, main_node) if options.fl > 0.0: eig3D = filt_tanl(eig3D, options.fl, options.aa) if myid == main_node: eig3D.write_image(os.path.join(options.outpout_dir, "eig3d_%03d.hdf"%(k, ITER))) Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) ) eig3Df, kb = prep_vol(eig3D) del eig3D cont = False icont = 0 for l in range(len(eigList[k])): phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l]) proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y]) cl = ccc(proj, eigList[k][l], mask2d) if cl < 0.0: icont += 1 cont = True eigList[k][l] *= -1.0 u = int(cont) u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD) icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: u = int(u[0]) log_main.add(" Eigenvector: ",k," number changed ",int(icont[0])) else: u = 0 u = bcast_number_to_all(u, main_node) cont = bool(u) ITER += 1 del eig3Df, kb mpi_barrier(MPI_COMM_WORLD) del eigList, mask2d ''' if options.ave3D: del ave3D if options.var2D: from fundamentals import fpol from applications import header if myid == main_node: log_main.add("Compute var2D...") km = 0 for i in range(number_of_proc): if i == main_node : for im in range(len(varList)): tmpvol=fpol(varList[im], nx, nx,1) tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in range(nl): ave = recv_EMData(i, im+i+70000) tmpvol=fpol(ave, nx, nx,1) tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km) km += 1 else: mpi_send(len(varList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in range(len(varList)): send_EMData(varList[im], main_node, im+myid+70000)# What with the attributes?? mpi_barrier(MPI_COMM_WORLD) if myid == main_node: from applications import header header(os.path.join(current_output_dir, options.var2D), params = 'xform.projection',fimport = os.path.join(current_output_dir, "params.txt")) mpi_barrier(MPI_COMM_WORLD) if options.var3D: if myid == main_node: log_main.add("Reconstruct var3D ...") t6 = time() # radiusvar = options.radius # if( radiusvar < 0 ): radiusvar = nx//2 -3 res = recons3d_4nn_MPI(myid, varList, symmetry = options.sym, npad=options.npad) #res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ) if myid == main_node: from fundamentals import fpol if current_decimate != 1.0: res = resample(res, 1./current_decimate) res = fpol(res, nnxo, nnxo, nnxo) set_pixel_size(res, 1.0) res.write_image(os.path.join(current_output_dir, options.var3D)) log_main.add("%-70s: %s\n"%("The reconstructed var3D is saved as ", options.var3D)) log_main.add("Var3D reconstruction took %f12.1 [m]"%((time()-t6)/60.0)) log_main.add("Total computation time %f12.1 [m]"%((time()-t0)/60.0)) log_main.add("sx3dvariability finishes") from mpi import mpi_finalize mpi_finalize() if RUNNING_UNDER_MPI: global_def.MPI = False global_def.BATCH = False
def main(): from optparse import OptionParser from global_def import SPARXVERSION from EMAN2 import EMData from logger import Logger, BaseLogger_Files import sys, os, time global Tracker, Blockdata from global_def import ERROR progname = os.path.basename(sys.argv[0]) usage = progname + " --output_dir=output_dir --isac_dir=output_dir_of_isac " parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--pw_adjustment", type ="string", default ='analytical_model', \ help="adjust power spectrum of 2-D averages to an analytic model. Other opions: no_adjustment; bfactor; a text file of 1D rotationally averaged PW") #### Four options for --pw_adjustment: # 1> analytical_model(default); # 2> no_adjustment; # 3> bfactor; # 4> adjust_to_given_pw2(user has to provide a text file that contains 1D rotationally averaged PW) # options in common parser.add_option( "--isac_dir", type="string", default='', help="ISAC run output directory, input directory for this command") parser.add_option( "--output_dir", type="string", default='', help="output directory where computed averages are saved") parser.add_option( "--pixel_size", type="float", default=-1.0, help= "pixel_size of raw images. one can put 1.0 in case of negative stain data" ) parser.add_option( "--fl", type="float", default=-1.0, help= "low pass filter, = -1.0, not applied; =0.0, using FH1 (initial resolution), = 1.0 using FH2 (resolution after local alignment), or user provided value in absolute freqency [0.0:0.5]" ) parser.add_option("--stack", type="string", default="", help="data stack used in ISAC") parser.add_option("--radius", type="int", default=-1, help="radius") parser.add_option("--xr", type="float", default=-1.0, help="local alignment search range") #parser.add_option("--ts", type ="float", default =1.0, help= "local alignment search step") parser.add_option("--fh", type="float", default=-1.0, help="local alignment high frequencies limit") #parser.add_option("--maxit", type ="int", default =5, help= "local alignment iterations") parser.add_option("--navg", type="int", default=1000000, help="number of aveages") parser.add_option("--local_alignment", action="store_true", default=False, help="do local alignment") parser.add_option( "--noctf", action="store_true", default=False, help= "no ctf correction, useful for negative stained data. always ctf for cryo data" ) parser.add_option( "--B_start", type="float", default=45.0, help= "start frequency (Angstrom) of power spectrum for B_factor estimation") parser.add_option( "--Bfactor", type="float", default=-1.0, help= "User defined bactors (e.g. 25.0[A^2]). By default, the program automatically estimates B-factor. " ) (options, args) = parser.parse_args(sys.argv[1:]) adjust_to_analytic_model = False adjust_to_given_pw2 = False B_enhance = False no_adjustment = False if options.pw_adjustment == 'analytical_model': adjust_to_analytic_model = True elif options.pw_adjustment == 'no_adjustment': no_adjustment = True elif options.pw_adjustment == 'bfactor': B_enhance = True else: adjust_to_given_pw2 = True from utilities import get_im, bcast_number_to_all, write_text_file, read_text_file, wrap_mpi_bcast, write_text_row from utilities import cmdexecute from filter import filt_tanl from logger import Logger, BaseLogger_Files import user_functions import string from string import split, atoi, atof import json mpi_init(0, []) nproc = mpi_comm_size(MPI_COMM_WORLD) myid = mpi_comm_rank(MPI_COMM_WORLD) Blockdata = {} # MPI stuff Blockdata["nproc"] = nproc Blockdata["myid"] = myid Blockdata["main_node"] = 0 Blockdata["shared_comm"] = mpi_comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL) Blockdata["myid_on_node"] = mpi_comm_rank(Blockdata["shared_comm"]) Blockdata["no_of_processes_per_group"] = mpi_comm_size( Blockdata["shared_comm"]) masters_from_groups_vs_everything_else_comm = mpi_comm_split( MPI_COMM_WORLD, Blockdata["main_node"] == Blockdata["myid_on_node"], Blockdata["myid_on_node"]) Blockdata["color"], Blockdata["no_of_groups"], balanced_processor_load_on_nodes = get_colors_and_subsets(Blockdata["main_node"], MPI_COMM_WORLD, Blockdata["myid"], \ Blockdata["shared_comm"], Blockdata["myid_on_node"], masters_from_groups_vs_everything_else_comm) # We need two nodes for processing of volumes Blockdata["node_volume"] = [ Blockdata["no_of_groups"] - 3, Blockdata["no_of_groups"] - 2, Blockdata["no_of_groups"] - 1 ] # For 3D stuff take three last nodes # We need two CPUs for processing of volumes, they are taken to be main CPUs on each volume # We have to send the two myids to all nodes so we can identify main nodes on two selected groups. Blockdata["nodes"] = [Blockdata["node_volume"][0]*Blockdata["no_of_processes_per_group"],Blockdata["node_volume"][1]*Blockdata["no_of_processes_per_group"], \ Blockdata["node_volume"][2]*Blockdata["no_of_processes_per_group"]] # End of Blockdata: sorting requires at least three nodes, and the used number of nodes be integer times of three global_def.BATCH = True global_def.MPI = True if adjust_to_given_pw2: checking_flag = 0 if (Blockdata["myid"] == Blockdata["main_node"]): if not os.path.exists(options.pw_adjustment): checking_flag = 1 checking_flag = bcast_number_to_all(checking_flag, Blockdata["main_node"], MPI_COMM_WORLD) if checking_flag == 1: ERROR("User provided power spectrum does not exist", "sxcompute_isac_avg.py", 1, Blockdata["myid"]) Tracker = {} Constants = {} Constants["isac_dir"] = options.isac_dir Constants["masterdir"] = options.output_dir Constants["pixel_size"] = options.pixel_size Constants["orgstack"] = options.stack Constants["radius"] = options.radius Constants["xrange"] = options.xr Constants["FH"] = options.fh Constants["low_pass_filter"] = options.fl #Constants["maxit"] = options.maxit Constants["navg"] = options.navg Constants["B_start"] = options.B_start Constants["Bfactor"] = options.Bfactor if adjust_to_given_pw2: Constants["modelpw"] = options.pw_adjustment Tracker["constants"] = Constants # ------------------------------------------------------------- # # Create and initialize Tracker dictionary with input options # State Variables #<<<---------------------->>>imported functions<<<--------------------------------------------- #x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1) #y_range = x_range ####----------------------------------------------------------- # Create Master directory and associated subdirectories line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]: masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen") else: masterdir = Tracker["constants"]["masterdir"] if (Blockdata["myid"] == Blockdata["main_node"]): msg = "Postprocessing ISAC 2D averages starts" print(line, "Postprocessing ISAC 2D averages starts") if not masterdir: timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime()) masterdir = "sharpen_" + Tracker["constants"]["isac_dir"] os.mkdir(masterdir) else: if os.path.exists(masterdir): print("%s already exists" % masterdir) else: os.mkdir(masterdir) subdir_path = os.path.join(masterdir, "ali2d_local_params_avg") if not os.path.exists(subdir_path): os.mkdir(subdir_path) subdir_path = os.path.join(masterdir, "params_avg") if not os.path.exists(subdir_path): os.mkdir(subdir_path) li = len(masterdir) else: li = 0 li = mpi_bcast(li, 1, MPI_INT, Blockdata["main_node"], MPI_COMM_WORLD)[0] masterdir = mpi_bcast(masterdir, li, MPI_CHAR, Blockdata["main_node"], MPI_COMM_WORLD) masterdir = string.join(masterdir, "") Tracker["constants"]["masterdir"] = masterdir log_main = Logger(BaseLogger_Files()) log_main.prefix = Tracker["constants"]["masterdir"] + "/" while not os.path.exists(Tracker["constants"]["masterdir"]): print("Node ", Blockdata["myid"], " waiting...", Tracker["constants"]["masterdir"]) sleep(1) mpi_barrier(MPI_COMM_WORLD) if (Blockdata["myid"] == Blockdata["main_node"]): init_dict = {} print(Tracker["constants"]["isac_dir"]) Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"], "2dalignment") core = read_text_row( os.path.join(Tracker["directory"], "initial2Dparams.txt")) for im in range(len(core)): init_dict[im] = core[im] del core else: init_dict = 0 init_dict = wrap_mpi_bcast(init_dict, Blockdata["main_node"], communicator=MPI_COMM_WORLD) ### do_ctf = True if options.noctf: do_ctf = False if (Blockdata["myid"] == Blockdata["main_node"]): if do_ctf: print("CTF correction is on") else: print("CTF correction is off") if options.local_alignment: print("local refinement is on") else: print("local refinement is off") if B_enhance: print("Bfactor is to be applied on averages") elif adjust_to_given_pw2: print("PW of averages is adjusted to a given 1D PW curve") elif adjust_to_analytic_model: print("PW of averages is adjusted to analytical model") else: print("PW of averages is not adjusted") #Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack") image = get_im(Tracker["constants"]["orgstack"], 0) Tracker["constants"]["nnxo"] = image.get_xsize() if Tracker["constants"]["pixel_size"] == -1.0: print( "Pixel size value is not provided by user. extracting it from ctf header entry of the original stack." ) try: ctf_params = image.get_attr("ctf") Tracker["constants"]["pixel_size"] = ctf_params.apix except: ERROR( "Pixel size could not be extracted from the original stack.", "sxcompute_isac_avg.py", 1, Blockdata["myid"]) # action=1 - fatal error, exit ## Now fill in low-pass filter isac_shrink_path = os.path.join(Tracker["constants"]["isac_dir"], "README_shrink_ratio.txt") if not os.path.exists(isac_shrink_path): ERROR( "%s does not exist in the specified ISAC run output directory" % (isac_shrink_path), "sxcompute_isac_avg.py", 1, Blockdata["myid"]) # action=1 - fatal error, exit isac_shrink_file = open(isac_shrink_path, "r") isac_shrink_lines = isac_shrink_file.readlines() isac_shrink_ratio = float( isac_shrink_lines[5] ) # 6th line: shrink ratio (= [target particle radius]/[particle radius]) used in the ISAC run isac_radius = float( isac_shrink_lines[6] ) # 7th line: particle radius at original pixel size used in the ISAC run isac_shrink_file.close() print("Extracted parameter values") print("ISAC shrink ratio : {0}".format(isac_shrink_ratio)) print("ISAC particle radius : {0}".format(isac_radius)) Tracker["ini_shrink"] = isac_shrink_ratio else: Tracker["ini_shrink"] = 0.0 Tracker = wrap_mpi_bcast(Tracker, Blockdata["main_node"], communicator=MPI_COMM_WORLD) #print(Tracker["constants"]["pixel_size"], "pixel_size") x_range = max(Tracker["constants"]["xrange"], int(1. / Tracker["ini_shrink"] + 0.99999)) a_range = y_range = x_range if (Blockdata["myid"] == Blockdata["main_node"]): parameters = read_text_row( os.path.join(Tracker["constants"]["isac_dir"], "all_parameters.txt")) else: parameters = 0 parameters = wrap_mpi_bcast(parameters, Blockdata["main_node"], communicator=MPI_COMM_WORLD) params_dict = {} list_dict = {} #parepare params_dict #navg = min(Tracker["constants"]["navg"]*Blockdata["nproc"], EMUtil.get_image_count(os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf"))) navg = min( Tracker["constants"]["navg"], EMUtil.get_image_count( os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf"))) global_dict = {} ptl_list = [] memlist = [] if (Blockdata["myid"] == Blockdata["main_node"]): print("Number of averages computed in this run is %d" % navg) for iavg in range(navg): params_of_this_average = [] image = get_im( os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf"), iavg) members = sorted(image.get_attr("members")) memlist.append(members) for im in range(len(members)): abs_id = members[im] global_dict[abs_id] = [iavg, im] P = combine_params2( init_dict[abs_id][0], init_dict[abs_id][1], init_dict[abs_id][2], init_dict[abs_id][3], \ parameters[abs_id][0], parameters[abs_id][1]/Tracker["ini_shrink"], parameters[abs_id][2]/Tracker["ini_shrink"], parameters[abs_id][3]) if parameters[abs_id][3] == -1: print( "WARNING: Image #{0} is an unaccounted particle with invalid 2D alignment parameters and should not be the member of any classes. Please check the consitency of input dataset." .format(abs_id) ) # How to check what is wrong about mirror = -1 (Toshio 2018/01/11) params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0]) ptl_list.append(abs_id) params_dict[iavg] = params_of_this_average list_dict[iavg] = members write_text_row( params_of_this_average, os.path.join(Tracker["constants"]["masterdir"], "params_avg", "params_avg_%03d.txt" % iavg)) ptl_list.sort() init_params = [None for im in range(len(ptl_list))] for im in range(len(ptl_list)): init_params[im] = [ptl_list[im]] + params_dict[global_dict[ ptl_list[im]][0]][global_dict[ptl_list[im]][1]] write_text_row( init_params, os.path.join(Tracker["constants"]["masterdir"], "init_isac_params.txt")) else: params_dict = 0 list_dict = 0 memlist = 0 params_dict = wrap_mpi_bcast(params_dict, Blockdata["main_node"], communicator=MPI_COMM_WORLD) list_dict = wrap_mpi_bcast(list_dict, Blockdata["main_node"], communicator=MPI_COMM_WORLD) memlist = wrap_mpi_bcast(memlist, Blockdata["main_node"], communicator=MPI_COMM_WORLD) # Now computing! del init_dict tag_sharpen_avg = 1000 ## always apply low pass filter to B_enhanced images to suppress noise in high frequencies enforced_to_H1 = False if B_enhance: if Tracker["constants"]["low_pass_filter"] == -1.0: enforced_to_H1 = True if navg < Blockdata["nproc"]: # Each CPU do one average ERROR("number of nproc is larger than number of averages", "sxcompute_isac_avg.py", 1, Blockdata["myid"]) else: FH_list = [[0, 0.0, 0.0] for im in range(navg)] image_start, image_end = MPI_start_end(navg, Blockdata["nproc"], Blockdata["myid"]) if Blockdata["myid"] == Blockdata["main_node"]: cpu_dict = {} for iproc in range(Blockdata["nproc"]): local_image_start, local_image_end = MPI_start_end( navg, Blockdata["nproc"], iproc) for im in range(local_image_start, local_image_end): cpu_dict[im] = iproc else: cpu_dict = 0 cpu_dict = wrap_mpi_bcast(cpu_dict, Blockdata["main_node"], communicator=MPI_COMM_WORLD) slist = [None for im in range(navg)] ini_list = [None for im in range(navg)] avg1_list = [None for im in range(navg)] avg2_list = [None for im in range(navg)] plist_dict = {} data_list = [None for im in range(navg)] if Blockdata["myid"] == Blockdata["main_node"]: if B_enhance: print( "Avg ID B-factor FH1(Res before ali) FH2(Res after ali)" ) else: print("Avg ID FH1(Res before ali) FH2(Res after ali)") for iavg in range(image_start, image_end): mlist = EMData.read_images(Tracker["constants"]["orgstack"], list_dict[iavg]) for im in range(len(mlist)): #mlist[im]= get_im(Tracker["constants"]["orgstack"], list_dict[iavg][im]) set_params2D(mlist[im], params_dict[iavg][im], xform="xform.align2d") if options.local_alignment: """ new_average1 = within_group_refinement([mlist[kik] for kik in range(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \ ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \ dst=0.0, maxit=Tracker["constants"]["maxit"], FH=max(Tracker["constants"]["FH"], FH1), FF=0.02, method="") new_average2 = within_group_refinement([mlist[kik] for kik in range(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \ ou= Tracker["constants"]["radius"], rs=1.0, xrng=[ x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \ dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.02, method="") new_avg, frc, plist = compute_average(mlist, Tracker["constants"]["radius"], do_ctf) """ new_avg, plist, FH2 = refinement_2d_local( mlist, Tracker["constants"]["radius"], a_range, x_range, y_range, CTF=do_ctf, SNR=1.0e10) plist_dict[iavg] = plist FH1 = -1.0 else: new_avg, frc, plist = compute_average( mlist, Tracker["constants"]["radius"], do_ctf) FH1 = get_optimistic_res(frc) FH2 = -1.0 #write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg)) FH_list[iavg] = [iavg, FH1, FH2] if B_enhance: new_avg, gb = apply_enhancement( new_avg, Tracker["constants"]["B_start"], Tracker["constants"]["pixel_size"], Tracker["constants"]["Bfactor"]) print(" %6d %6.3f %4.3f %4.3f" % (iavg, gb, FH1, FH2)) elif adjust_to_given_pw2: roo = read_text_file(Tracker["constants"]["modelpw"], -1) roo = roo[0] # always on the first column new_avg = adjust_pw_to_model( new_avg, Tracker["constants"]["pixel_size"], roo) print(" %6d %4.3f %4.3f " % (iavg, FH1, FH2)) elif adjust_to_analytic_model: new_avg = adjust_pw_to_model( new_avg, Tracker["constants"]["pixel_size"], None) print(" %6d %4.3f %4.3f " % (iavg, FH1, FH2)) elif no_adjustment: pass if Tracker["constants"]["low_pass_filter"] != -1.0: if Tracker["constants"]["low_pass_filter"] == 0.0: low_pass_filter = FH1 elif Tracker["constants"]["low_pass_filter"] == 1.0: low_pass_filter = FH2 if not options.local_alignment: low_pass_filter = FH1 else: low_pass_filter = Tracker["constants"]["low_pass_filter"] if low_pass_filter >= 0.45: low_pass_filter = 0.45 new_avg = filt_tanl(new_avg, low_pass_filter, 0.02) else: # No low pass filter but if enforced if enforced_to_H1: new_avg = filt_tanl(new_avg, FH1, 0.02) if B_enhance: new_avg = fft(new_avg) new_avg.set_attr("members", list_dict[iavg]) new_avg.set_attr("n_objects", len(list_dict[iavg])) slist[iavg] = new_avg print( strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>", "Refined average %7d" % iavg) ## send to main node to write mpi_barrier(MPI_COMM_WORLD) for im in range(navg): # avg if cpu_dict[im] == Blockdata[ "myid"] and Blockdata["myid"] != Blockdata["main_node"]: send_EMData(slist[im], Blockdata["main_node"], tag_sharpen_avg) elif cpu_dict[im] == Blockdata["myid"] and Blockdata[ "myid"] == Blockdata["main_node"]: slist[im].set_attr("members", memlist[im]) slist[im].set_attr("n_objects", len(memlist[im])) slist[im].write_image( os.path.join(Tracker["constants"]["masterdir"], "class_averages.hdf"), im) elif cpu_dict[im] != Blockdata["myid"] and Blockdata[ "myid"] == Blockdata["main_node"]: new_avg_other_cpu = recv_EMData(cpu_dict[im], tag_sharpen_avg) new_avg_other_cpu.set_attr("members", memlist[im]) new_avg_other_cpu.set_attr("n_objects", len(memlist[im])) new_avg_other_cpu.write_image( os.path.join(Tracker["constants"]["masterdir"], "class_averages.hdf"), im) if options.local_alignment: if cpu_dict[im] == Blockdata["myid"]: write_text_row( plist_dict[im], os.path.join(Tracker["constants"]["masterdir"], "ali2d_local_params_avg", "ali2d_local_params_avg_%03d.txt" % im)) if cpu_dict[im] == Blockdata[ "myid"] and cpu_dict[im] != Blockdata["main_node"]: wrap_mpi_send(plist_dict[im], Blockdata["main_node"], MPI_COMM_WORLD) wrap_mpi_send(FH_list, Blockdata["main_node"], MPI_COMM_WORLD) elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[ "myid"] == Blockdata["main_node"]: dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD) plist_dict[im] = dummy dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD) FH_list[im] = dummy[im] else: if cpu_dict[im] == Blockdata[ "myid"] and cpu_dict[im] != Blockdata["main_node"]: wrap_mpi_send(FH_list, Blockdata["main_node"], MPI_COMM_WORLD) elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[ "myid"] == Blockdata["main_node"]: dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD) FH_list[im] = dummy[im] mpi_barrier(MPI_COMM_WORLD) mpi_barrier(MPI_COMM_WORLD) if options.local_alignment: if Blockdata["myid"] == Blockdata["main_node"]: ali3d_local_params = [None for im in range(len(ptl_list))] for im in range(len(ptl_list)): ali3d_local_params[im] = [ptl_list[im]] + plist_dict[ global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]] write_text_row( ali3d_local_params, os.path.join(Tracker["constants"]["masterdir"], "ali2d_local_params.txt")) write_text_row( FH_list, os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt")) else: if Blockdata["myid"] == Blockdata["main_node"]: write_text_row( FH_list, os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt")) mpi_barrier(MPI_COMM_WORLD) target_xr = 3 target_yr = 3 if (Blockdata["myid"] == 0): cmd = "{} {} {} {} {} {} {} {} {} {}".format("sxchains.py", os.path.join(Tracker["constants"]["masterdir"],"class_averages.hdf"),\ os.path.join(Tracker["constants"]["masterdir"],"junk.hdf"),os.path.join(Tracker["constants"]["masterdir"],"ordered_class_averages.hdf"),\ "--circular","--radius=%d"%Tracker["constants"]["radius"] , "--xr=%d"%(target_xr+1),"--yr=%d"%(target_yr+1),"--align", ">/dev/null") junk = cmdexecute(cmd) cmd = "{} {}".format( "rm -rf", os.path.join(Tracker["constants"]["masterdir"], "junk.hdf")) junk = cmdexecute(cmd) from mpi import mpi_finalize mpi_finalize() exit()
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " stack outdir <maskfile> --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translation_step --dst=delta --center=center --maxit=max_iteration --CTF --snr=SNR --Fourvar=Fourier_variance --Ng=group_number --Function=user_function_name --CUDA --GPUID --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--ir", type="float", default=1, help="inner radius for rotational correlation > 0 (set to 1)") parser.add_option( "--ou", type="float", default=-1, help= "outer radius for rotational correlation < nx/2-1 (set to the radius of the particle)" ) parser.add_option( "--rs", type="float", default=1, help="step between rings in rotational correlation > 0 (set to 1)") parser.add_option( "--xr", type="string", default="4 2 1 1", help="range for translation search in x direction, search is +/xr ") parser.add_option( "--yr", type="string", default="-1", help="range for translation search in y direction, search is +/yr ") parser.add_option("--ts", type="string", default="2 1 0.5 0.25", help="step of translation search in both directions") parser.add_option( "--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)") parser.add_option("--dst", type="float", default=0.0, help="delta") parser.add_option( "--center", type="float", default=-1, help= "-1.average center method; 0.not centered; 1.phase approximation; 2.cc with Gaussian function; 3.cc with donut-shaped image 4.cc with user-defined reference 5.cc with self-rotated average" ) parser.add_option( "--maxit", type="float", default=0, help= "maximum number of iterations (0 means the maximum iterations is 10, but it will automatically stop should the criterion falls" ) parser.add_option("--CTF", action="store_true", default=False, help="use CTF correction during alignment") parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio of the data (set to 1.0)") parser.add_option("--Fourvar", action="store_true", default=False, help="compute Fourier variance") #parser.add_option("--Ng", type="int", default=-1, help="number of groups in the new CTF filteration") parser.add_option( "--function", type="string", default="ref_ali2d", help="name of the reference preparation function (default ref_ali2d)") #parser.add_option("--CUDA", action="store_true", default=False, help="use CUDA program") #parser.add_option("--GPUID", type="string", default="", help="ID of GPUs available") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version ") parser.add_option( "--rotational", action="store_true", default=False, help= "rotational alignment with optional limited in-plane angle, the parameters are: ir, ou, rs, psi_max, mode(F or H), maxit, orient, randomize" ) parser.add_option("--psi_max", type="float", default=180.0, help="psi_max") parser.add_option("--mode", type="string", default="F", help="Full or Half rings, default F") parser.add_option( "--randomize", action="store_true", default=False, help="randomize initial rotations (suboption of friedel, default False)" ) parser.add_option( "--orient", action="store_true", default=False, help= "orient images such that the average is symmetric about x-axis, for layer lines (suboption of friedel, default False)" ) parser.add_option( "--template", type="string", default=None, help= "2D alignment will be initialized using the template provided (only non-MPI version, default None)" ) parser.add_option("--random_method", type="string", default="", help="use SHC or SCF (default standard method)") (options, args) = parser.parse_args() if len(args) < 2 or len(args) > 3: print("usage: " + usage) print("Please run '" + progname + " -h' for detailed options") elif (options.rotational): from applications import ali2d_rotationaltop global_def.BATCH = True ali2d_rotationaltop(args[1], args[0], options.randomize, options.orient, options.ir, options.ou, options.rs, options.psi_max, options.mode, options.maxit) else: if args[1] == 'None': outdir = None else: outdir = args[1] if len(args) == 2: mask = None else: mask = args[2] if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True if options.MPI: from applications import ali2d_base from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD sys.argv = mpi_init(len(sys.argv), sys.argv) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) myid = mpi_comm_rank(MPI_COMM_WORLD) main_node = 0 if (myid == main_node): import subprocess from logger import Logger, BaseLogger_Files # Create output directory log = Logger(BaseLogger_Files()) log.prefix = os.path.join(outdir) cmd = "mkdir " + log.prefix outcome = subprocess.call(cmd, shell=True) log.prefix += "/" else: outcome = 0 log = None from utilities import bcast_number_to_all outcome = bcast_number_to_all(outcome, source_node=main_node) if (outcome == 1): ERROR( 'Output directory exists, please change the name and restart the program', "ali2d_MPI", 1, myid) dummy = ali2d_base(args[0], outdir, mask, options.ir, options.ou, options.rs, options.xr, options.yr, \ options.ts, options.nomirror, options.dst, \ options.center, options.maxit, options.CTF, options.snr, options.Fourvar, \ options.function, random_method = options.random_method, log = log, \ number_of_proc = number_of_proc, myid = myid, main_node = main_node, mpi_comm = MPI_COMM_WORLD,\ write_headers = True) else: print(" Non-MPI is no more in use, try MPI option, please.") """ from applications import ali2d ali2d(args[0], outdir, mask, options.ir, options.ou, options.rs, options.xr, options.yr, \ options.ts, options.nomirror, options.dst, \ options.center, options.maxit, options.CTF, options.snr, options.Fourvar, \ -1, options.function, False, "", options.MPI, \ options.template, random_method = options.random_method) """ global_def.BATCH = False if options.MPI: from mpi import mpi_finalize mpi_finalize()
def main(args): progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" + " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" + " --generation=generation --rand_seed=rand_seed>") parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--radius", type="int", help= "particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)" ) parser.add_option( "--target_radius", type="int", default=29, help= "target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)" ) parser.add_option( "--target_nx", type="int", default=76, help= "target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1' (default 76)" ) parser.add_option( "--img_per_grp", type="int", default=100, help= "number of images per class: in the ideal case (essentially maximum size of class) (default 100)" ) parser.add_option( "--CTF", action="store_true", default=False, help= "apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)" ) parser.add_option( "--ir", type="int", default=1, help= "inner ring: of the resampling to polar coordinates. units - pixels (default 1)" ) parser.add_option( "--rs", type="int", default=1, help= "ring step: of the resampling to polar coordinates. units - pixels (default 1)" ) parser.add_option( "--xr", type="int", default=1, help= "x range: of translational search. By default, set by the program. (default 1)" ) parser.add_option( "--yr", type="int", default=-1, help= "y range: of translational search. By default, same as xr. (default -1)" ) parser.add_option( "--ts", type="float", default=1.0, help= "search step: of translational search: units - pixels (default 1.0)") parser.add_option( "--maxit", type="int", default=30, help="number of iterations for reference-free alignment: (default 30)") #parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)") parser.add_option( "--center_method", type="int", default=-1, help= "method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)" ) parser.add_option( "--dst", type="float", default=90.0, help="discrete angle used in within group alignment: (default 90.0)") parser.add_option( "--FL", type="float", default=0.2, help= "lowest stopband: frequency used in the tangent filter (default 0.2)") parser.add_option( "--FH", type="float", default=0.3, help= "highest stopband: frequency used in the tangent filter (default 0.3)") parser.add_option("--FF", type="float", default=0.2, help="fall-off of the tangent filter: (default 0.2)") parser.add_option( "--init_iter", type="int", default=3, help= "SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)" ) parser.add_option( "--main_iter", type="int", default=3, help= "SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)" ) parser.add_option( "--iter_reali", type="int", default=1, help= "SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)" ) parser.add_option( "--match_first", type="int", default=1, help= "number of iterations to run 2-way matching in the first phase: (default 1)" ) parser.add_option( "--max_round", type="int", default=20, help= "maximum rounds: of generating candidate class averages in the first phase (default 20)" ) parser.add_option( "--match_second", type="int", default=5, help= "number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)" ) parser.add_option( "--stab_ali", type="int", default=5, help="number of alignments when checking stability: (default 5)") parser.add_option( "--thld_err", type="float", default=0.7, help= "threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)" ) parser.add_option( "--indep_run", type="int", default=4, help= "level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching. Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)" ) parser.add_option("--thld_grp", type="int", default=10, help="minimum size of reproducible class (default 10)") parser.add_option( "--n_generations", type="int", default=10, help= "maximum number of generations: program stops when reaching this total number of generations: (default 10)" ) #parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)") parser.add_option( "--rand_seed", type="int", help= "random seed set before calculations: useful for testing purposes. By default, total randomness (type int)" ) parser.add_option("--new", action="store_true", default=False, help="use new code: (default False)") parser.add_option("--debug", action="store_true", default=False, help="debug info printout: (default False)") # must be switched off in production parser.add_option( "--use_latest_master_directory", action="store_true", default=False, help= "use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)" ) parser.add_option( "--restart_section", type="string", default=' ', help= "restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')" ) parser.add_option( "--stop_after_candidates", action="store_true", default=False, help= "stop after candidates: stops after the 'candidate_class_averages' section. (default False)" ) ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help=SUPPRESS_HELP) parser.add_option( "--skip_prealignment", action="store_true", default=False, help= "skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)" ) required_option_list = ['radius'] (options, args) = parser.parse_args(args) if options.return_options: return parser if len(args) > 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" sys.exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True from isac import iter_isac from fundamentals import rot_shift2D, resample from utilities import pad, combine_params2 command_line_provided_stack_filename = args[0] main_node = 0 mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) nproc = mpi_comm_size(MPI_COMM_WORLD) mpi_barrier(MPI_COMM_WORLD) if (myid == main_node): print "****************************************************************" Util.version() print "****************************************************************" sys.stdout.flush() mpi_barrier(MPI_COMM_WORLD) # Making sure all required options appeared. for required_option in required_option_list: if not options.__dict__[required_option]: print "\n ==%s== mandatory option is missing.\n" % required_option print "Please run '" + progname + " -h' for detailed options" return 1 radi = options.radius target_radius = options.target_radius target_nx = options.target_nx center_method = options.center_method if (radi < 1): ERROR("Particle radius has to be provided!", "sxisac", 1, myid) use_latest_master_directory = options.use_latest_master_directory stop_after_candidates = options.stop_after_candidates # program_state_stack.restart_location_title_from_command_line = options.restart_section from utilities import qw program_state_stack.PROGRAM_STATE_VARIABLES = set( qw(""" isac_generation """)) # create or reuse master directory masterdir = "" stack_processed_by_ali2d_base__filename = "" stack_processed_by_ali2d_base__filename__without_master_dir = "" error_status = 0 if len(args) == 2: masterdir = args[1] elif len(args) == 1: if use_latest_master_directory: all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)] import re r = re.compile("^master.*$") all_dirs = filter(r.match, all_dirs) if len(all_dirs) > 0: # all_dirs = max(all_dirs, key=os.path.getctime) masterdir = max(all_dirs, key=os.path.getmtime) #Create folder for all results or check if there is one created already if (myid == main_node): if (masterdir == ""): timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime()) masterdir = "master" + timestring cmd = "{} {}".format("mkdir", masterdir) junk = cmdexecute(cmd) elif not os.path.exists(masterdir): # os.path.exists(masterdir) does not exist masterdir = args[1] cmd = "{} {}".format("mkdir", masterdir) junk = cmdexecute(cmd) if (args[0][:4] == "bdb:"): filename = args[0][4:] else: filename = args[0][:-4] filename = os.path.basename(filename) stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join( masterdir, filename) stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename if_error_then_all_processes_exit_program(error_status) # send masterdir to all processes masterdir = send_string_to_all(masterdir) if myid == 0: if options.restart_section != " ": if os.path.exists(os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)): stored_stack, stored_state = restore_program_stack_and_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) import re if "," in options.restart_section: parsed_restart_section_option = options.restart_section.split( ",") stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % parsed_restart_section_option[0], stored_state[-1]["location_in_program"]) generation_str_format = parsed_restart_section_option[1] if generation_str_format != "": isac_generation_from_command_line = int( generation_str_format) stored_state[-1][ "isac_generation"] = isac_generation_from_command_line else: isac_generation_from_command_line = 1 if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % options.restart_section, stored_state[-1]["location_in_program"]) if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] store_program_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE), stored_state, stored_stack) else: print "Please remove the restart_section option from the command line. The program must be started from the beginning." mpi_finalize() sys.exit() else: isac_generation_from_command_line = -1 program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) stack_processed_by_ali2d_base__filename = send_string_to_all( stack_processed_by_ali2d_base__filename) stack_processed_by_ali2d_base__filename__without_master_dir = \ send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir) # previous code 2016-05-05--20-14-12-153 # # PARAMETERS OF THE PROCEDURE # if( options.xr == -1 ): # # Default values # # target_nx = 76 # # target_radius = 29 # target_xr = 1 # else: # nx//2 # # Check below! # target_xr = options.xr # # target_nx = 76 + target_xr - 1 # subtract one, which is default # target_nx += target_xr - 1 # subtract one, which is default # # target_radius = 29 target_xr = options.xr target_nx += target_xr - 1 # subtract one, which is default if (options.yr == -1): yr = options.xr else: yr = options.yr mpi_barrier(MPI_COMM_WORLD) # Initialization of stacks if (myid == main_node): print "command_line_provided_stack_filename", command_line_provided_stack_filename number_of_images_in_stack = EMUtil.get_image_count( command_line_provided_stack_filename) else: number_of_images_in_stack = 0 number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node=main_node) nxrsteps = 4 init2dir = os.path.join(masterdir, "2dalignment") # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() if not os.path.exists( os.path.join(init2dir, "Finished_initial_2d_alignment.txt")): if (myid == 0): import subprocess from logger import Logger, BaseLogger_Files # Create output directory log2d = Logger(BaseLogger_Files()) log2d.prefix = os.path.join(init2dir) cmd = "mkdir -p " + log2d.prefix outcome = subprocess.call(cmd, shell=True) log2d.prefix += "/" # outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True) else: outcome = 0 log2d = None if (myid == main_node): a = get_im(command_line_provided_stack_filename) nnxo = a.get_xsize() else: nnxo = 0 nnxo = bcast_number_to_all(nnxo, source_node=main_node) image_start, image_end = MPI_start_end(number_of_images_in_stack, nproc, myid) if options.skip_prealignment: params2d = [[0.0, 0.0, 0.0, 0] for i in xrange(image_start, image_end)] else: original_images = EMData.read_images( command_line_provided_stack_filename, range(image_start, image_end)) # We assume the target radius will be 29, and xr = 1. shrink_ratio = float(target_radius) / float(radi) for im in xrange(len(original_images)): if (shrink_ratio != 1.0): original_images[im] = resample(original_images[im], shrink_ratio) nx = original_images[0].get_xsize() # nx = int(nx*shrink_ratio + 0.5) txrm = (nx - 2 * (target_radius + 1)) // 2 if (txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d" % (radi), "sxisac", 1, myid) if (txrm / nxrsteps > 0): tss = "" txr = "" while (txrm / nxrsteps > 0): tts = txrm / nxrsteps tss += " %d" % tts txr += " %d" % (tts * nxrsteps) txrm = txrm // 2 else: tss = "1" txr = "%d" % txrm # print "nx, txr, txrm, tss", nx, txr, txrm, tss # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() # section ali2d_base params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \ False, 90.0, center_method, 14, options.CTF, 1.0, False, \ "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False) del original_images for i in xrange(len(params2d)): alpha, sx, sy, mirror = combine_params2( 0, params2d[i][1], params2d[i][2], 0, -params2d[i][0], 0, 0, 0) sx /= shrink_ratio sy /= shrink_ratio params2d[i][0] = 0.0 params2d[i][1] = sx params2d[i][2] = sy params2d[i][3] = 0 #set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0]) mpi_barrier(MPI_COMM_WORLD) tmp = params2d[:] tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD) if (myid == main_node): if options.skip_prealignment: print "=========================================" print "Even though there is no alignment step, '%s' params are set to zero for later use." % os.path.join( init2dir, "initial2Dparams.txt") print "=========================================" write_text_row(tmp, os.path.join(init2dir, "initial2Dparams.txt")) del tmp mpi_barrier(MPI_COMM_WORLD) # We assume the target image size will be target_nx, radius will be 29, and xr = 1. # Note images can be also padded, in which case shrink_ratio > 1. shrink_ratio = float(target_radius) / float(radi) aligned_images = EMData.read_images( command_line_provided_stack_filename, range(image_start, image_end)) nx = aligned_images[0].get_xsize() nima = len(aligned_images) newx = int(nx * shrink_ratio + 0.5) while not os.path.exists(os.path.join(init2dir, "initial2Dparams.txt")): import time time.sleep(1) mpi_barrier(MPI_COMM_WORLD) params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt")) params = params[image_start:image_end] msk = model_circle(radi, nx, nx) for im in xrange(nima): st = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= st[0] if options.CTF: aligned_images[im] = filt_ctf( aligned_images[im], aligned_images[im].get_attr("ctf"), binary=True) if (shrink_ratio < 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) #aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(newx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio == 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(newx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) #aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio > 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(newx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) del msk gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid) number_of_images_in_stack = bcast_number_to_all( number_of_images_in_stack, source_node=main_node) if (myid == main_node): for i in range(number_of_images_in_stack): aligned_images[i].write_image( stack_processed_by_ali2d_base__filename, i) # It has to be explicitly closed from EMAN2db import db_open_dict DB = db_open_dict(stack_processed_by_ali2d_base__filename) DB.close() fp = open(os.path.join(masterdir, "README_shrink_ratio.txt"), "w") output_text = """ Since, for processing purposes, isac changes the image dimensions, adjustment of pixel size needs to be made in subsequent steps, (e.g. running sxviper.py). The shrink ratio for this particular isac run is -------- %.5f %.5f -------- To get the pixel size for the isac output the user needs to divide the original pixel size by the above value. This info is saved in the following file: README_shrink_ratio.txt """ % (shrink_ratio, radi) fp.write(output_text) fp.flush() fp.close() print output_text fp = open( os.path.join(init2dir, "Finished_initial_2d_alignment.txt"), "w") fp.flush() fp.close() else: if (myid == main_node): print "Skipping 2d alignment since it was already done!" mpi_barrier(MPI_COMM_WORLD) # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() os.chdir(masterdir) if program_state_stack(locals(), getframeinfo(currentframe())): # if 1: pass if (myid == main_node): junk = cmdexecute( "sxheader.py --consecutive --params=originalid %s" % stack_processed_by_ali2d_base__filename__without_master_dir) junk = cmdexecute( "e2bdb.py %s --makevstack=%s_000" % (stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir)) if (myid == main_node): main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d") print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no if isac_generation_from_command_line < 0: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = -1 if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: for i in xrange(isac_generation_from_command_line + 1, main_dir_no + 1): if i == isac_generation_from_command_line + 1: backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) junk = cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i + " 000_backup" + "%05d" % backup_dir_no) junk = cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % i) # it includes both command line and json file my_restart_section = stored_state[-1]["location_in_program"].split( "___")[-1] if "restart" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) junk = cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) junk = cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % isac_generation_from_command_line) elif "candidate_class_averages" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) junk = cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line) # junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line) elif "reproducible_class_averages" in my_restart_section: junk = cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*" % isac_generation_from_command_line) junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*" % isac_generation_from_command_line) else: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 0 isac_generation_from_command_line = mpi_bcast( isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0] isac_generation = isac_generation_from_command_line - 1 if (myid == main_node): if isac_generation == 0: junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) write_text_file( [1], os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_accounted.txt" % isac_generation)) write_text_file( range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_unaccounted.txt" % isac_generation)) # Stopping criterion should be inside the program. while True: isac_generation += 1 if isac_generation > options.n_generations: break data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d" % isac_generation program_state_stack.restart_location_title = "restart" if program_state_stack(locals(), getframeinfo(currentframe())): if (myid == main_node): junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) # reference the original stack list_file = os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1), "generation_%d_unaccounted.txt" % (isac_generation - 1)) junk = cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\ stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file)) mpi_barrier(MPI_COMM_WORLD) os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation) program_state_stack.restart_location_title = "candidate_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new) # program_state_stack.restart_location_title = "stopped_program1" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack.restart_location_title = "stop_after_candidates" program_state_stack(locals(), getframeinfo(currentframe())) if stop_after_candidates: mpi_finalize() sys.exit() exit_program = 0 if (myid == main_node): if not os.path.exists( "class_averages_candidate_generation_%d.hdf" % isac_generation): print "This generation (%d) no class average candidates were generated! Finishing." % isac_generation exit_program = 1 exit_program = int( mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if exit_program: os.chdir("..") break program_state_stack.restart_location_title = "reproducible_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new) pass os.chdir("..") if (myid == main_node): accounted_images = read_text_file( os.path.join(NAME_OF_MAIN_DIR + "%04d" % (isac_generation), "generation_%d_accounted.txt" % (isac_generation))) number_of_accounted_images = len(accounted_images) un_accounted_images = read_text_file( os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation), "generation_%d_unaccounted.txt" % (isac_generation))) number_of_un_accounted_images = len(un_accounted_images) else: number_of_accounted_images = 0 number_of_un_accounted_images = 0 number_of_accounted_images = int( mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) number_of_un_accounted_images = int( mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if number_of_accounted_images == 0: if (myid == main_node): print "This generation (%d) there are no accounted images! Finishing." % isac_generation break while (myid == main_node): def files_are_missing(isac_generation): for i in xrange(1, isac_generation + 1): if not os.path.exists( "generation_%04d/class_averages_generation_%d.hdf" % (i, i)): print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting." % ( i, i) return 1 return 0 if files_are_missing(isac_generation): break junk = cmdexecute("rm -f class_averages.hdf") cpy([ "generation_%04d/class_averages_generation_%d.hdf" % (i, i) for i in xrange(1, isac_generation + 1) ], "class_averages.hdf") break if number_of_un_accounted_images == 0: if (myid == main_node): print "This generation (%d) there are no un accounted images! Finishing." % isac_generation break program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall") mpi_barrier(MPI_COMM_WORLD) mpi_finalize()
def run3Dalignment(paramsdict, partids, partstack, outputdir, procid, myid, main_node, nproc): # Reads from paramsdict["stack"] particles partids set parameters in partstack # and do refinement as specified in paramsdict # # Will create outputdir # Will write to outputdir output parameters: params-chunk0.txt and params-chunk1.txt if(myid == main_node): # Create output directory log = Logger(BaseLogger_Files()) log.prefix = os.path.join(outputdir) #cmd = "mkdir "+log.prefix #junk = cmdexecute(cmd) log.prefix += "/" else: log = None mpi_barrier(MPI_COMM_WORLD) ali3d_options.delta = paramsdict["delta"] ali3d_options.ts = paramsdict["ts"] ali3d_options.xr = paramsdict["xr"] # low pass filter is applied to shrank data, so it has to be adjusted ali3d_options.fl = paramsdict["lowpass"]/paramsdict["shrink"] ali3d_options.initfl = paramsdict["initialfl"]/paramsdict["shrink"] ali3d_options.aa = paramsdict["falloff"] ali3d_options.maxit = paramsdict["maxit"] ali3d_options.mask3D = paramsdict["mask3D"] ali3d_options.an = paramsdict["an"] ali3d_options.ou = paramsdict["radius"] # This is changed in ali3d_base, but the shrank value is needed in vol recons, fixt it! shrinkage = paramsdict["shrink"] projdata = getindexdata(paramsdict["stack"], partids, partstack, myid, nproc) onx = projdata[0].get_xsize() last_ring = ali3d_options.ou if last_ring < 0: last_ring = int(onx/2) - 2 mask2D = model_circle(last_ring,onx,onx) - model_circle(ali3d_options.ir,onx,onx) if(shrinkage < 1.0): # get the new size masks2D = resample(mask2D, shrinkage) nx = masks2D.get_xsize() masks2D = model_circle(int(last_ring*shrinkage+0.5),nx,nx) - model_circle(max(int(ali3d_options.ir*shrinkage+0.5),1),nx,nx) nima = len(projdata) oldshifts = [0.0,0.0]*nima for im in xrange(nima): #data[im].set_attr('ID', list_of_particles[im]) ctf_applied = projdata[im].get_attr_default('ctf_applied', 0) phi,theta,psi,sx,sy = get_params_proj(projdata[im]) projdata[im] = fshift(projdata[im], sx, sy) set_params_proj(projdata[im],[phi,theta,psi,0.0,0.0]) # For local SHC set anchor #if(nsoft == 1 and an[0] > -1): # set_params_proj(data[im],[phi,tetha,psi,0.0,0.0], "xform.anchor") oldshifts[im] = [sx,sy] if ali3d_options.CTF : ctf_params = projdata[im].get_attr("ctf") if ctf_applied == 0: st = Util.infomask(projdata[im], mask2D, False) projdata[im] -= st[0] projdata[im] = filt_ctf(projdata[im], ctf_params) projdata[im].set_attr('ctf_applied', 1) if(shrinkage < 1.0): #phi,theta,psi,sx,sy = get_params_proj(projdata[im]) projdata[im] = resample(projdata[im], shrinkage) st = Util.infomask(projdata[im], None, True) projdata[im] -= st[0] st = Util.infomask(projdata[im], masks2D, True) projdata[im] /= st[1] #sx *= shrinkage #sy *= shrinkage #set_params_proj(projdata[im], [phi,theta,psi,sx,sy]) if ali3d_options.CTF : ctf_params.apix /= shrinkage projdata[im].set_attr('ctf', ctf_params) else: st = Util.infomask(projdata[im], None, True) projdata[im] -= st[0] st = Util.infomask(projdata[im], mask2D, True) projdata[im] /= st[1] del mask2D if(shrinkage < 1.0): del masks2D """ if(paramsdict["delpreviousmax"]): for i in xrange(len(projdata)): try: projdata[i].del_attr("previousmax") except: pass """ if(myid == main_node): print_dict(paramsdict,"3D alignment parameters") print(" => actual lowpass : "******" => actual init lowpass : "******" => PW adjustment : ",ali3d_options.pwreference) print(" => partids : ",partids) print(" => partstack : ",partstack) if(ali3d_options.fl > 0.46): ERROR("Low pass filter in 3D alignment > 0.46 on the scale of shrank data","sxcenter_projections",1,myid) # Run alignment command, it returns params per CPU params = center_projections_3D(projdata, paramsdict["refvol"], \ ali3d_options, onx, shrinkage, \ mpi_comm = MPI_COMM_WORLD, myid = myid, main_node = main_node, log = log ) del log, projdata params = wrap_mpi_gatherv(params, main_node, MPI_COMM_WORLD) # store params if(myid == main_node): for im in xrange(nima): params[im][0] = params[im][0]/shrinkage +oldshifts[im][0] params[im][1] = params[im][1]/shrinkage +oldshifts[im][1] line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print(line,"Executed successfully: ","3D alignment"," number of images:%7d"%len(params)) write_text_row(params, os.path.join(outputdir,"params.txt") )
def main(args): progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" + " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" + " --generation=generation --rand_seed=rand_seed>") parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--radius", type="int", default=-1, help="<Particle radius>, it has to be provided.") parser.add_option( "--img_per_grp", type="int", default=100, help= "<number of images per group> in the ideal case (essentially maximum size of class) (100)" ) parser.add_option("--CTF", action="store_true", default=False, help="<CTF flag>, if set the data will be phase-flipped") parser.add_option( "--ir", type="int", default=1, help="<inner ring> of the resampling to polar coordinates (1)") parser.add_option( "--rs", type="int", default=1, help="<ring step> of the resampling to polar coordinates (1)") parser.add_option( "--xr", type="int", default=-1, help= "<x range> of translational search (By default set by the program) (advanced)" ) parser.add_option( "--yr", type="int", default=-1, help="<y range> of translational search (same as xr) (advanced)") parser.add_option("--ts", type="float", default=1.0, help="<search step> of translational search (1.0)") parser.add_option( "--maxit", type="int", default=30, help="number of iterations for reference-free alignment (30)") #parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)") parser.add_option( "--center_method", type="int", default=7, help= "<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)" ) parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment ") parser.add_option( "--FL", type="float", default=0.2, help="<lowest stopband> frequency used in the tangent filter (0.2)") parser.add_option( "--FH", type="float", default=0.3, help="<highest stopband> frequency used in the tangent filter (0.3)") parser.add_option("--FF", type="float", default=0.2, help="<fall-off of the tangent> filter (0.2)") parser.add_option( "--init_iter", type="int", default=3, help= "<init_iter> number of iterations of ISAC program in initialization (3)" ) parser.add_option( "--main_iter", type="int", default=3, help="<main_iter> number of iterations of ISAC program in main part (3)" ) parser.add_option( "--iter_reali", type="int", default=1, help= "<iter_reali> number of iterations in ISAC before checking stability (1)" ) parser.add_option( "--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase (1)" ) parser.add_option( "--max_round", type="int", default=20, help= "maximum rounds of generating candidate averages in the first phase (20)" ) parser.add_option( "--match_second", type="int", default=5, help= "number of iterations to run 2-way (or 3-way) matching in the second phase (5)" ) parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability (5)") parser.add_option( "--thld_err", type="float", default=0.7, help="the threshold of pixel error when checking stability (0.7)") parser.add_option( "--indep_run", type="int", default=4, help= "number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)" ) parser.add_option("--thld_grp", type="int", default=10, help="minimum size of class (10)") parser.add_option( "--n_generations", type="int", default=100, help= "<n_generations> program stops when reaching this total number of generations (advanced)" ) #parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)") parser.add_option( "--rand_seed", type="int", default=None, help= "random seed set before calculations, useful for testing purposes (default None - total randomness)" ) parser.add_option("--new", action="store_true", default=False, help="use new code (default = False)") parser.add_option("--debug", action="store_true", default=False, help="debug info printout (default = False)") # must be switched off in production parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False) parser.add_option( "--restart_section", type="string", default="", help= "<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)" ) parser.add_option( "--stop_after_candidates", action="store_true", default=False, help= "<stop_after_candidates> stops after the 'candidate_class_averages' section" ) parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help=SUPPRESS_HELP) (options, args) = parser.parse_args(args) if options.return_options: return parser if len(args) > 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" sys.exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() from isac import iter_isac global_def.BATCH = True global_def.BATCH = True command_line_provided_stack_filename = args[0] global_def.BATCH = True main_node = 0 mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) nproc = mpi_comm_size(MPI_COMM_WORLD) radi = options.radius center_method = options.center_method if (radi < 1): ERROR("Particle radius has to be provided!", "sxisac", 1, myid) use_latest_master_directory = options.use_latest_master_directory stop_after_candidates = options.stop_after_candidates program_state_stack.restart_location_title_from_command_line = options.restart_section from utilities import qw program_state_stack.PROGRAM_STATE_VARIABLES = set( qw(""" isac_generation """)) # create or reuse master directory masterdir = "" stack_processed_by_ali2d_base__filename = "" stack_processed_by_ali2d_base__filename__without_master_dir = "" error_status = 0 if len(args) == 2: masterdir = args[1] elif len(args) == 1: if use_latest_master_directory: all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)] import re r = re.compile("^master.*$") all_dirs = filter(r.match, all_dirs) if len(all_dirs) > 0: # all_dirs = max(all_dirs, key=os.path.getctime) masterdir = max(all_dirs, key=os.path.getmtime) #Create folder for all results or check if there is one created already if (myid == main_node): if (masterdir == ""): timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime()) masterdir = "master" + timestring cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) elif not os.path.exists(masterdir): # os.path.exists(masterdir) does not exist masterdir = args[1] cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) if (args[0][:4] == "bdb:"): filename = args[0][4:] else: filename = args[0][:-4] filename = os.path.basename(filename) stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join( masterdir, filename) stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename if_error_all_processes_quit_program(error_status) # send masterdir to all processes masterdir = send_string_to_all(masterdir) if myid == 0: if options.restart_section != "": if os.path.exists(os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)): stored_stack, stored_state = restore_program_stack_and_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) import re if "," in options.restart_section: parsed_restart_section_option = options.restart_section.split( ",") stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % parsed_restart_section_option[0], stored_state[-1]["location_in_program"]) generation_str_format = parsed_restart_section_option[1] if generation_str_format != "": isac_generation_from_command_line = int( generation_str_format) stored_state[-1][ "isac_generation"] = isac_generation_from_command_line else: isac_generation_from_command_line = 1 if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % options.restart_section, stored_state[-1]["location_in_program"]) if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] store_program_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE), stored_state, stored_stack) else: print "Please remove the restart_section option from the command line. The program must be started from the beginning." mpi_finalize() sys.exit() else: isac_generation_from_command_line = -1 program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) stack_processed_by_ali2d_base__filename = send_string_to_all( stack_processed_by_ali2d_base__filename) stack_processed_by_ali2d_base__filename__without_master_dir = \ send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir) # PARAMETERS OF THE PROCEDURE if (options.xr == -1): # Default values target_nx = 76 target_radius = 29 target_xr = 1 else: # nx//2 # Check below! target_xr = options.xr target_nx = 76 + target_xr - 1 # subtract one, which is default target_radius = 29 mpi_barrier(MPI_COMM_WORLD) # Initialization of stacks if (myid == main_node): number_of_images_in_stack = EMUtil.get_image_count( command_line_provided_stack_filename) else: number_of_images_in_stack = 0 number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node=main_node) nxrsteps = 4 init2dir = os.path.join(masterdir, "2dalignment") if (myid == 0): import subprocess from logger import Logger, BaseLogger_Files # Create output directory log2d = Logger(BaseLogger_Files()) log2d.prefix = os.path.join(init2dir) cmd = "mkdir -p " + log2d.prefix outcome = subprocess.call(cmd, shell=True) log2d.prefix += "/" # outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True) else: outcome = 0 log2d = None if (myid == main_node): a = get_im(command_line_provided_stack_filename) nnxo = a.get_xsize() else: nnxo = 0 nnxo = bcast_number_to_all(nnxo, source_node=main_node) txrm = (nnxo - 2 * (radi + 1)) // 2 if (txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d" % (radi), "sxisac", 1, myid) if (txrm / nxrsteps > 0): tss = "" txr = "" while (txrm / nxrsteps > 0): tts = txrm / nxrsteps tss += " %d" % tts txr += " %d" % (tts * nxrsteps) txrm = txrm // 2 else: tss = "1" txr = "%d" % txrm # section ali2d_base # centering method is set to #7 params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \ False, 90.0, center_method, 14, options.CTF, 1.0, False, \ "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False) if (myid == main_node): write_text_row(params2d, os.path.join(init2dir, "initial2Dparams.txt")) del params2d mpi_barrier(MPI_COMM_WORLD) # We assume the target image size will be target_nx, radius will be 29, and xr = 1. # Note images can be also padded, in which case shrink_ratio > 1. shrink_ratio = float(target_radius) / float(radi) nx = aligned_images[0].get_xsize() nima = len(aligned_images) newx = int(nx * shrink_ratio + 0.5) from fundamentals import rot_shift2D, resample from utilities import pad, combine_params2 if (shrink_ratio < 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(nx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio == 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(nx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio > 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(target_radius, nx, nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) del msk gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid) number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node=main_node) if (myid == main_node): for i in range(number_of_images_in_stack): aligned_images[i].write_image( stack_processed_by_ali2d_base__filename, i) # It has to be explicitly closed from EMAN2db import db_open_dict DB = db_open_dict(stack_processed_by_ali2d_base__filename) DB.close() mpi_barrier(MPI_COMM_WORLD) global_def.BATCH = True os.chdir(masterdir) if program_state_stack(locals(), getframeinfo(currentframe())): # if 1: pass if (myid == main_node): cmdexecute( "sxheader.py --consecutive --params=originalid %s" % stack_processed_by_ali2d_base__filename__without_master_dir) cmdexecute( "e2bdb.py %s --makevstack=%s_000" % (stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir)) if (myid == main_node): main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d") print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no if isac_generation_from_command_line < 0: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = -1 if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: for i in xrange(isac_generation_from_command_line + 1, main_dir_no + 1): if i == isac_generation_from_command_line + 1: backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i + " 000_backup" + "%05d" % backup_dir_no) cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % i) # it includes both command line and json file my_restart_section = stored_state[-1]["location_in_program"].split( "___")[-1] if "restart" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % isac_generation_from_command_line) elif "candidate_class_averages" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line) # cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line) elif "reproducible_class_averages" in my_restart_section: cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*" % isac_generation_from_command_line) cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*" % isac_generation_from_command_line) else: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 0 isac_generation_from_command_line = mpi_bcast( isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0] isac_generation = isac_generation_from_command_line - 1 if (myid == main_node): if isac_generation == 0: cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) write_text_file( [1], os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_accounted.txt" % isac_generation)) write_text_file( range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_unaccounted.txt" % isac_generation)) # Stopping criterion should be inside the program. while True: isac_generation += 1 if isac_generation > options.n_generations: break data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d" % isac_generation if (myid == main_node): accounted_images = read_text_file( os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1), "generation_%d_accounted.txt" % (isac_generation - 1))) number_of_accounted_images = len(accounted_images) # unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1))) # number_of_unaccounted_images = len(unaccounted_images) else: number_of_accounted_images = 0 number_of_accounted_images = int( mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if number_of_accounted_images == 0: os.chdir("..") break program_state_stack.restart_location_title = "restart" if program_state_stack(locals(), getframeinfo(currentframe())): if (myid == main_node): cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) # reference the original stack list_file = os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1), "generation_%d_unaccounted.txt" % (isac_generation - 1)) cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\ stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file)) mpi_barrier(MPI_COMM_WORLD) os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation) program_state_stack.restart_location_title = "candidate_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new) # program_state_stack.restart_location_title = "stopped_program1" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack.restart_location_title = "stop_after_candidates" program_state_stack(locals(), getframeinfo(currentframe())) if stop_after_candidates: mpi_finalize() sys.exit() exit_program = 0 if (myid == main_node): if not os.path.exists( "class_averages_candidate_generation_%d.hdf" % isac_generation): print "This generation (%d) no class averages were generated!" % isac_generation exit_program = 1 exit_program = int( mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if exit_program: os.chdir("..") break program_state_stack.restart_location_title = "reproducible_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new) pass os.chdir("..") if (myid == main_node): cmdexecute("rm -f class_averages.hdf") cpy([ "generation_%04d/class_averages_generation_%d.hdf" % (i, i) for i in xrange(1, isac_generation) ], "class_averages.hdf") # program_state_stack.restart_location_title = "stopped_program2" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall") mpi_finalize()
def main(): from EMAN2 import EMData from utilities import write_text_file from mpi import mpi_init, mpi_finalize, MPI_COMM_WORLD, mpi_comm_rank, mpi_comm_size, mpi_comm_split, mpi_barrier from logger import Logger, BaseLogger_Files from air import air import sys import os import user_functions from optparse import OptionParser from global_def import SPARXVERSION progname = os.path.basename(sys.argv[0]) usage = progname + " projections minimal_subset_size target_threshold output_directory --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --center=center_type --maxit=max_iter --CTF --snr=SNR --ref_a=S --sym=c1 --function=user_function --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--ir", type="int", default=1, help="inner radius for rotational correlation > 0 (set to 1)") parser.add_option( "--ou", type="int", default=-1, help= "outer radius for rotational correlation < int(nx/2)-1 (set to the radius of the particle)" ) parser.add_option( "--rs", type="int", default=1, help="step between rings in rotational correlation >0 (set to 1)") parser.add_option( "--xr", type="string", default="0", help="range for translation search in x direction, search is +/xr") parser.add_option( "--yr", type="string", default="-1", help= "range for translation search in y direction, search is +/yr (default = same as xr)" ) parser.add_option( "--ts", type="string", default="1", help= "step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional" ) parser.add_option("--delta", type="string", default="2", help="angular step of reference projections") parser.add_option( "--an", type="string", default="-1", help="angular neighborhood for local searches (phi and theta)") parser.add_option( "--center", type="float", default=-1, help= "-1: average shift method; 0: no centering; 1: center of gravity (default=-1)" ) parser.add_option( "--maxit", type="float", default=50, help= "maximum number of iterations performed for each angular step (set to 50) " ) parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ") parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio of the data") parser.add_option( "--ref_a", type="string", default="S", help= "method for generating the quasi-uniformly distributed projection directions (default S)" ) parser.add_option("--sym", type="string", default="c1", help="symmetry of the refined structure") parser.add_option( "--function", type="string", default="ref_ali3d", help="name of the reference preparation function (ref_ali3d by default)" ) parser.add_option("--npad", type="int", default=2, help="padding size for 3D reconstruction (default=2)") parser.add_option( "--MPI", action="store_true", default=True, help="whether to use MPI version - this is always set to True") parser.add_option( "--proc_mshc", type="int", default=3, help="number of MPI processes per multiSHC, 3 is minimum (default=3)") (options, args) = parser.parse_args(sys.argv[1:]) if len(args) < 4: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" return 1 mpi_init(0, []) mpi_size = mpi_comm_size(MPI_COMM_WORLD) mpi_rank = mpi_comm_rank(MPI_COMM_WORLD) proc_per_mshc = int(options.proc_mshc) if mpi_size < proc_per_mshc: print "Number of processes can't be smaller than value given as the parameter --proc_mshc" mpi_finalize() return log = Logger(BaseLogger_Files()) projs = EMData.read_images(args[0]) minimal_subset_size = int(args[1]) target_threshold = float(args[2]) outdir = args[3] if mpi_rank == 0: if os.path.exists(outdir): ERROR( 'Output directory exists, please change the name and restart the program', "sxmulti_shc", 1) mpi_finalize() return os.mkdir(outdir) import global_def global_def.LOGFILE = os.path.join(outdir, global_def.LOGFILE) mpi_barrier(MPI_COMM_WORLD) if outdir[-1] != "/": outdir += "/" log.prefix = outdir me = wrap_mpi_split(MPI_COMM_WORLD, mpi_size / proc_per_mshc) options.user_func = user_functions.factory[options.function] new_subset, new_threshold = air(projs, minimal_subset_size, target_threshold, options, number_of_runs=6, number_of_winners=3, mpi_env=me, log=log) if mpi_rank == 0: log.add("Output threshold =", new_threshold) log.add("Output subset: ", len(new_subset), new_subset) write_text_file(new_subset, log.prefix + "final_subset.txt") mpi_finalize()
def main(args): from utilities import write_text_row, drop_image, model_gauss_noise, get_im, set_params_proj, wrap_mpi_bcast, model_circle from logger import Logger, BaseLogger_Files from mpi import mpi_init, mpi_finalize, MPI_COMM_WORLD, mpi_comm_rank, mpi_comm_size, mpi_barrier import user_functions import sys import os from applications import MPI_start_end from optparse import OptionParser, SUPPRESS_HELP from global_def import SPARXVERSION from EMAN2 import EMData from multi_shc import multi_shc progname = os.path.basename(sys.argv[0]) usage = progname + " stack output_directory --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --center=center_type --maxit1=max_iter1 --maxit2=max_iter2 --L2threshold=0.1 --ref_a=S --sym=c1" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--ir", type= "int", default= 1, help="<inner radius> for rotational correlation > 0 (set to 1)") parser.add_option("--ou", type= "int", default= -1, help="<outer radius> for rotational correlation < int(nx/2)-1 (set to the radius of the particle)") parser.add_option("--rs", type= "int", default= 1, help="<step between> rings in rotational correlation >0 (set to 1)" ) parser.add_option("--xr", type="string", default= "0", help="<xr range> for translation search in x direction, search is +/xr (default 0)") parser.add_option("--yr", type="string", default= "-1", help="<yr range> for translation search in y direction, search is +/yr (default = same as xr)") parser.add_option("--ts", type="string", default= "1", help="<ts step size> of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional") parser.add_option("--delta", type="string", default= "2", help="<angular step> of reference projections (default 2)") parser.add_option("--center", type="float", default= -1, help="-1: average shift method; 0: no centering; 1: center of gravity (default=-1)") parser.add_option("--maxit1", type="float", default= 400, help="maximum number of iterations performed for the GA part (set to 400) ") parser.add_option("--maxit2", type="float", default= 50, help="maximum number of iterations performed for the finishing up part (set to 50) ") parser.add_option("--L2threshold", type="float", default= 0.03, help="Stopping criterion of GA given as a maximum relative dispersion of L2 norms (set to 0.03) ") parser.add_option("--ref_a", type="string", default= "S", help="method for generating the quasi-uniformly distributed projection directions (default S)") parser.add_option("--sym", type="string", default= "c1", help="<symmetry> of the refined structure") # parser.add_option("--function", type="string", default="ref_ali3d", help="name of the reference preparation function (ref_ali3d by default)") parser.add_option("--function", type="string", default="ref_ali3d", help= SUPPRESS_HELP) parser.add_option("--nruns", type="int", default= 6, help="number of quasi-independent runs (default=6)") parser.add_option("--doga", type="float", default= 0.1, help="do GA when fraction of orientation changes less than 1.0 degrees is at least doga (default=0.1)") parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default=2)") parser.add_option("--fl", type="float", default=0.25, help="<cut-off frequency> of hyperbolic tangent low-pass Fourier filter (default 0.25)") parser.add_option("--aa", type="float", default=0.1, help="<fall-off frequency> of hyperbolic tangent low-pass Fourier filter (default 0.1)") parser.add_option("--pwreference", type="string", default="", help="<power spectrum> reference text file (default no power spectrum adjustment) (advanced)") parser.add_option("--mask3D", type="string", default=None, help="3D mask file (default a sphere)") parser.add_option("--moon_elimination", type="string", default="", help="<moon elimination> mass in KDa and resolution in px/A separated by comma, no space (advanced)") parser.add_option("--debug", action="store_true", default=False, help="<debug> info printout (default = False)") parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP) #parser.add_option("--an", type="string", default= "-1", help="NOT USED angular neighborhood for local searches (phi and theta)") #parser.add_option("--CTF", action="store_true", default=False, help="NOT USED Consider CTF correction during the alignment ") #parser.add_option("--snr", type="float", default= 1.0, help="NOT USED Signal-to-Noise Ratio of the data (default 1.0)") # (options, args) = parser.parse_args(sys.argv[1:]) (options, args) = parser.parse_args(args) # option_dict = vars(options) # print parser if options.return_options: return parser if options.moon_elimination == "": options.moon_elimination = [] else: options.moon_elimination = map(float, options.moon_elimination.split(",")) if len(args) < 2 or len(args) > 3: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" return 1 mpi_init(0, []) log = Logger(BaseLogger_Files()) runs_count = options.nruns mpi_rank = mpi_comm_rank(MPI_COMM_WORLD) mpi_size = mpi_comm_size(MPI_COMM_WORLD) # Total number of processes, passed by --np option. if mpi_rank == 0: all_projs = EMData.read_images(args[0]) subset = range(len(all_projs)) # if mpi_size > len(all_projs): # ERROR('Number of processes supplied by --np needs to be less than or equal to %d (total number of images) ' % len(all_projs), 'sxviper', 1) # mpi_finalize() # return else: all_projs = None subset = None outdir = args[1] if mpi_rank == 0: if mpi_size % options.nruns != 0: ERROR('Number of processes needs to be a multiple of total number of runs. Total runs by default are 3, you can change it by specifying --nruns option.', 'sxviper', 1) mpi_finalize() return if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxviper", 1) mpi_finalize() return os.mkdir(outdir) import global_def global_def.LOGFILE = os.path.join(outdir, global_def.LOGFILE) mpi_barrier(MPI_COMM_WORLD) if outdir[-1] != "/": outdir += "/" log.prefix = outdir # if len(args) > 2: # ref_vol = get_im(args[2]) # else: ref_vol = None options.user_func = user_functions.factory[options.function] options.CTF = False options.snr = 1.0 options.an = -1.0 out_params, out_vol, out_peaks = multi_shc(all_projs, subset, runs_count, options, mpi_comm=MPI_COMM_WORLD, log=log, ref_vol=ref_vol) mpi_finalize()