def comp_rep(refrings, data, itout, modout, vol, group, nima, nx, myid, main_node, outdir): import os from fundamentals import rot_shift2D from utilities import get_params_proj, params_3D_2D from mpi import mpi_reduce, MPI_COMM_WORLD, MPI_FLOAT, MPI_SUM avg = [EMData() for i in xrange(len(refrings))] avg_csum = [0.0 for i in xrange(len(refrings))] for i in xrange(len(refrings)): avg[i] = EMData() avg[i].set_size(nx, nx) phi = refrings[i].get_attr("phi") theta = refrings[i].get_attr("theta") t = Transform({ "type": "spider", "phi": phi, "theta": theta, "psi": 0.0 }) avg[i].set_attr("xform.projection", t) for im in xrange(nima): iref = data[im].get_attr("assign") gim = data[im].get_attr("group") if gim == group: [phi, theta, psi, s2x, s2y] = get_params_proj(data[im]) [alpha, sx, sy, mirror] = params_3D_2D(phi, theta, psi, s2x, s2y) temp = rot_shift2D(data[im], alpha, sx, sy, mirror, 1.0) avg[iref] = avg[iref] + temp avg_csum[iref] = avg_csum[iref] + 1 from utilities import reduce_EMData_to_root for i in xrange(len(refrings)): reduce_EMData_to_root(avg[i], myid, main_node) avg_sum = mpi_reduce(avg_csum[i], 1, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD) outfile_repro = os.path.join(outdir, "repro_%s%s.hdf" % (itout, modout)) if myid == 0: outfile = os.path.join(outdir, "compare_repro_%s%s.hdf" % (itout, modout)) avg[i].write_image(outfile, -1) t = avg[i].get_attr("xform.projection") proj = vol.project("pawel", t) proj.set_attr("xform.projection", t) proj.set_attr("Raw_im_count", float(avg_sum)) proj.write_image(outfile, -1) proj.write_image(outfile_repro, -1) return outfile_repro
def rot_shift2D(img, psi, tx=None, ty=None, m=None, out=None): ''' Rotate and shift an image in 2D * has alternative :Parameters: img : array Image data psi : float Inplane rotation tx : int Translation in the x-direction ty : int Translation in the y-direction m : bool True if image should be mirrored around x-axis out : array Output array :Returns: img : array Transformed image ''' if fundamentals is None: raise ImportError, "EMAN2/Sparx library not available, rot_shift2D requires EMAN2/Sparx" if tx is None: #m = psi[1] > 179.9 tx = psi[6] ty = psi[7] psi = psi[5] if out is None: out = img.copy() emdata = numpy2em(img) emdata = fundamentals.rot_shift2D(emdata, psi, tx, ty, m, interpolation_method="gridding") out[:, :] = em2numpy(emdata) return out
def comp_rep(refrings, data, itout, modout, vol, group, nima, nx, myid, main_node, outdir): import os from fundamentals import rot_shift2D from utilities import get_params_proj, params_3D_2D from mpi import mpi_reduce, MPI_COMM_WORLD, MPI_FLOAT, MPI_SUM avg = [EMData() for i in xrange(len(refrings))] avg_csum = [0.0 for i in xrange(len(refrings))] for i in xrange(len(refrings)): avg[i] = EMData() avg[i].set_size(nx,nx) phi = refrings[i].get_attr("phi") theta = refrings[i].get_attr("theta") t = Transform({"type":"spider","phi":phi,"theta":theta,"psi":0.0}) avg[i].set_attr("xform.projection",t) for im in xrange(nima): iref = data[im].get_attr("assign") gim = data[im].get_attr("group") if gim == group: [phi, theta, psi, s2x, s2y] = get_params_proj(data[im]) [alpha, sx,sy,mirror] = params_3D_2D(phi,theta,psi,s2x,s2y) temp = rot_shift2D(data[im],alpha, sx, sy, mirror, 1.0) avg[iref] = avg[iref] + temp avg_csum[iref] = avg_csum[iref] + 1 from utilities import reduce_EMData_to_root for i in xrange(len(refrings)): reduce_EMData_to_root(avg[i], myid, main_node) avg_sum = mpi_reduce(avg_csum[i],1,MPI_FLOAT,MPI_SUM,0,MPI_COMM_WORLD) outfile_repro = os.path.join(outdir, "repro_%s%s.hdf"%(itout,modout)) if myid ==0: outfile = os.path.join(outdir, "compare_repro_%s%s.hdf"%(itout,modout)) avg[i].write_image(outfile,-1) t = avg[i].get_attr("xform.projection") proj = vol.project("pawel",t) proj.set_attr("xform.projection",t) proj.set_attr("Raw_im_count", float(avg_sum)) proj.write_image(outfile,-1) proj.write_image(outfile_repro,-1) return outfile_repro
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Forms chains of 2D images based on their similarities. Functionality: 4. Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent). Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers. The ways to use the program: 4.1 Use option initial to specify which image will be used as an initial seed to form the chain. sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25 4.2 If options initial is omitted, the program will determine which image best serves as initial seed to form the chain sxprocess.py input_stack.hdf output_stack.hdf --radius=25 4.3 Use option circular to form a circular chain. sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25 4.4 New circular code based on pairwise alignments sxprocess.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt 4.5 Circular ordering based on pairwise alignments sxprocess.py vols.hdf chain.hdf mask.hdf --dd --radius=25 """ parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False) parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False) parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False) parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)") parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") # import params for 2D alignment parser.add_option("--ou", type="int", default=-1, help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)") parser.add_option("--xr", type="int", default=0, help="range for translation search in x direction, search is +/xr (0)") parser.add_option("--yr", type="int", default=0, help="range for translation search in y direction, search is +/yr (0)") #parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)") parser.add_option("--pairwiseccc", type="string", default= None, help="Input/output pairwise ccc file") (options, args) = parser.parse_args() global_def.BATCH = True if options.dd: nargs = len(args) if nargs != 3: print "must provide name of input and two output files!" return stack = args[0] new_stack = args[1] from utilities import model_circle from statistics import ccc from statistics import mono lend = EMUtil.get_image_count(stack) lccc = [None]*(lend*(lend-1)/2) for i in xrange(lend-1): v1 = get_im( stack, i ) if( i == 0 and nargs == 2): nx = v1.get_xsize() ny = v1.get_ysize() nz = v1.get_ysize() if options.ou < 1 : radius = nx//2-2 else: radius = options.ou mask = model_circle(radius, nx, ny, nz) else: mask = get_im(args[2]) for j in xrange(i+1, lend): lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0] order = tsp(lccc) if(len(order) != lend): print " problem with data length" from sys import exit exit() print "Total sum of cccs :",TotalDistance(order, lccc) print "ordering :",order for i in xrange(lend): get_im(stack, order[i]).write_image( new_stack, i ) elif options.align: nargs = len(args) if nargs != 3: print "must provide name of input and two output files!" return from utilities import get_params2D, model_circle from fundamentals import rot_shift2D from statistics import ccc from time import time from alignment import align2d, align2d_scf from multi_shc import mult_transform stack = args[0] new_stack = args[1] d = EMData.read_images(stack) """ # will align anyway try: ttt = d[0].get_attr('xform.params2d') for i in xrange(len(d)): alpha, sx, sy, mirror, scale = get_params2D(d[i]) d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror) except: pass """ nx = d[0].get_xsize() ny = d[0].get_ysize() if options.ou < 1 : radius = nx//2-2 else: radius = options.ou mask = model_circle(radius, nx, ny) if(options.xr < 0): xrng = 0 else: xrng = options.xr if(options.yr < 0): yrng = xrng else: yrng = options.yr initial = max(options.initial, 0) from statistics import mono lend = len(d) lccc = [None]*(lend*(lend-1)/2) from utilities import read_text_row if options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) : st = time() for i in xrange(lend-1): for j in xrange(i+1, lend): # j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger) #alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F") alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius) lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir] #print " %4d %10.1f"%(i,time()-st) if(not os.path.exists(options.pairwiseccc)): from utilities import write_text_row write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc) elif(os.path.exists(options.pairwiseccc)): lccc = read_text_row(options.pairwiseccc) initial = int(lccc[0][0] + 0.1) del lccc[0] for i in xrange(len(lccc)): T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)}) lccc[i] = [lccc[i][0],T] tdummy = Transform({"type":"2D"}) maxsum = -1.023 for m in xrange(0,lend):#initial, initial+1): indc = range( lend ) lsnake = [[m, tdummy, 0.0]] del indc[m] lsum = 0.0 while len(indc) > 1: maxcit = -111. for i in xrange(len(indc)): cuc = lccc[mono(indc[i], lsnake[-1][0])][0] if cuc > maxcit: maxcit = cuc qi = indc[i] # Here we need transformation from the current to the previous, # meaning indc[i] -> lsnake[-1][0] T = lccc[mono(indc[i], lsnake[-1][0])][1] # If direction is from larger to smaller index, the transformation has to be inverted if( indc[i] > lsnake[-1][0] ): T = T.inverse() lsnake.append([qi,T, maxcit]) lsum += maxcit del indc[indc.index(qi)] T = lccc[mono(indc[-1], lsnake[-1][0])][1] if( indc[-1] > lsnake[-1][0]): T = T.inverse() lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]]) print " initial image and lsum ",m,lsum #print lsnake if(lsum > maxsum): maxsum = lsum init = m snake = [lsnake[i] for i in xrange(lend)] print " Initial image selected : ",init,maxsum," ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc) #for q in snake: print q from copy import deepcopy trans=deepcopy([snake[i][1] for i in xrange(len(snake))]) print [snake[i][0] for i in xrange(len(snake))]
print [snake[i][0] for i in xrange(len(snake))] """ for m in xrange(lend): prms = trans[m].get_params("2D") print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2]) """ for k in xrange(lend-2,0,-1): T = snake[k][1] for i in xrange(k+1, lend): trans[i] = T*trans[i] # To add - apply all transformations and do the overall centering. for m in xrange(lend): prms = trans[m].get_params("2D") #print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2]) #rot_shift2D(d[snake[m][0]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image(new_stack, m) rot_shift2D(d[snake[m][0]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(new_stack, m) order = tsp(lccc) if(len(order) != lend): print " problem with data length" from sys import exit exit() print TotalDistance(order, lccc) print order ibeg = order.index(init) order = [order[(i+ibeg)%lend] for i in xrange(lend)] print TotalDistance(order, lccc) print order snake = [tdummy]
def compare(compare_ref_free, outfile_repro,ref_free_output,yrng, xrng, rstep,nx,apix,ref_free_cutoff, nproc, myid, main_node): from alignment import Numrinit, ringwe, Applyws from random import seed, randint from utilities import get_params2D, set_params2D, model_circle, inverse_transform2, combine_params2 from fundamentals import rot_shift2D from mpi import MPI_COMM_WORLD, mpi_barrier, mpi_bcast, MPI_INT from statistics import fsc_mask from filter import fit_tanh from numpy import array fout = "%s.hdf" % ref_free_output frc_out = "%s_frc" % ref_free_output res_out = "%s_res" % ref_free_output nima = EMUtil.get_image_count(compare_ref_free) image_start, image_end = MPI_start_end(nima, nproc, myid) ima = EMData() ima.read_image(compare_ref_free, image_start) last_ring = nx/2-2 first_ring = 1 mask = model_circle(last_ring, nx, nx) refi = [] numref = EMUtil.get_image_count(outfile_repro) cnx = nx/2 +1 cny = cnx mode = "F" numr = Numrinit(first_ring, last_ring, rstep, mode) wr = ringwe(numr, mode) ima.to_zero() for j in xrange(numref): temp = EMData() temp.read_image(outfile_repro, j) # even, odd, numer of even, number of images. After frc, totav refi.append(temp) # for each node read its share of data data = EMData.read_images(compare_ref_free, range(image_start, image_end)) for im in xrange(image_start, image_end): data[im-image_start].set_attr('ID', im) set_params2D(data[im-image_start],[0,0,0,0,1]) ringref = [] for j in xrange(numref): refi[j].process_inplace("normalize.mask", {"mask":mask, "no_sigma":1}) # normalize reference images to N(0,1) cimage = Util.Polar2Dm(refi[j], cnx, cny, numr, mode) Util.Frngs(cimage, numr) Applyws(cimage, numr, wr) ringref.append(cimage) if myid == main_node: seed(1000) data_shift = [] frc = [] res = [] for im in xrange(image_start, image_end): alpha, sx, sy, mirror, scale = get_params2D(data[im-image_start]) alphai, sxi, syi, scalei = inverse_transform2(alpha, sx, sy, 1.0) # normalize data[im-image_start].process_inplace("normalize.mask", {"mask":mask, "no_sigma":1}) # subtract average under the mask # align current image to the reference [angt, sxst, syst, mirrort, xiref, peakt] = Util.multiref_polar_ali_2d(data[im-image_start], ringref, xrng, yrng, 1, mode, numr, cnx+sxi, cny+syi) iref = int(xiref) [alphan, sxn, syn, mn] = combine_params2(0.0, -sxi, -syi, 0, angt, sxst, syst, (int)(mirrort)) set_params2D(data[im-image_start], [alphan, sxn, syn, int(mn), scale]) temp = rot_shift2D(data[im-image_start], alphan, sxn, syn, mn) temp.set_attr('assign',iref) tfrc = fsc_mask(temp,refi[iref],mask = mask) temp.set_attr('frc',tfrc[1]) res = fit_tanh(tfrc) temp.set_attr('res',res) data_shift.append(temp) for node in xrange(nproc): if myid == node: for image in data_shift: image.write_image(fout,-1) refindex = image.get_attr('assign') refi[refindex].write_image(fout,-1) mpi_barrier(MPI_COMM_WORLD) rejects = [] if myid == main_node: a = EMData() index = 0 frc = [] res = [] temp = [] classes = [] for im in xrange(nima): a.read_image(fout, index) frc.append(a.get_attr("frc")) if ref_free_cutoff != -1: classes.append(a.get_attr("class_ptcl_idxs")) tmp = a.get_attr("res") temp.append(tmp[0]) res.append("%12f" %(apix/tmp[0])) res.append("\n") index = index + 2 res_num = array(temp) mean_score = res_num.mean(axis=0) std_score = res_num.std(axis=0) std = std_score / 2 if ref_free_cutoff !=-1: cutoff = mean_score - std * ref_free_cutoff reject = res_num < cutoff index = 0 for i in reject: if i: rejects.extend(classes[index]) index = index + 1 rejects.sort() length = mpi_bcast(len(rejects),1,MPI_INT,main_node, MPI_COMM_WORLD) rejects = mpi_bcast(rejects,length , MPI_INT, main_node, MPI_COMM_WORLD) del a fout_frc = open(frc_out,'w') fout_res = open(res_out,'w') fout_res.write("".join(res)) temp = zip(*frc) datstrings = [] for i in temp: for j in i: datstrings.append(" %12f" % (j)) datstrings.append("\n") fout_frc.write("".join(datstrings)) fout_frc.close() del refi del ringref return rejects
def main(args): progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" + " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" + " --generation=generation --rand_seed=rand_seed>" ) parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--radius", type="int", help="particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)") parser.add_option("--target_radius", type="int", default=29, help="target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)") parser.add_option("--target_nx", type="int", default=76, help="target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1' (default 76)") parser.add_option("--img_per_grp", type="int", default=100, help="number of images per class: in the ideal case (essentially maximum size of class) (default 100)") parser.add_option("--CTF", action="store_true", default=False, help="apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)") parser.add_option("--ir", type="int", default=1, help="inner ring: of the resampling to polar coordinates. units - pixels (default 1)") parser.add_option("--rs", type="int", default=1, help="ring step: of the resampling to polar coordinates. units - pixels (default 1)") parser.add_option("--xr", type="int", default=1, help="x range: of translational search. By default, set by the program. (default 1)") parser.add_option("--yr", type="int", default=-1, help="y range: of translational search. By default, same as xr. (default -1)") parser.add_option("--ts", type="float", default=1.0, help="search step: of translational search: units - pixels (default 1.0)") parser.add_option("--maxit", type="int", default=30, help="number of iterations for reference-free alignment: (default 30)") #parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)") parser.add_option("--center_method", type="int", default=-1, help="method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)") parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment: (default 90.0)") parser.add_option("--FL", type="float", default=0.2, help="lowest stopband: frequency used in the tangent filter (default 0.2)") parser.add_option("--FH", type="float", default=0.3, help="highest stopband: frequency used in the tangent filter (default 0.3)") parser.add_option("--FF", type="float", default=0.2, help="fall-off of the tangent filter: (default 0.2)") parser.add_option("--init_iter", type="int", default=3, help="SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)") parser.add_option("--main_iter", type="int", default=3, help="SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)") parser.add_option("--iter_reali", type="int", default=1, help="SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)") parser.add_option("--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase: (default 1)") parser.add_option("--max_round", type="int", default=20, help="maximum rounds: of generating candidate class averages in the first phase (default 20)") parser.add_option("--match_second", type="int", default=5, help="number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)") parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability: (default 5)") parser.add_option("--thld_err", type="float", default=0.7, help="threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)") parser.add_option("--indep_run", type="int", default=4, help="level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching. Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)") parser.add_option("--thld_grp", type="int", default=10, help="minimum size of reproducible class (default 10)") parser.add_option("--n_generations", type="int", default=100, help="maximum number of generations: program stops when reaching this total number of generations: (default 100)") #parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)") parser.add_option("--rand_seed", type="int", help="random seed set before calculations: useful for testing purposes. By default, total randomness (type int)") parser.add_option("--new", action="store_true", default=False, help="use new code: (default False)") parser.add_option("--debug", action="store_true", default=False, help="debug info printout: (default False)") # must be switched off in production parser.add_option("--use_latest_master_directory",action="store_true", default=False, help="use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)") parser.add_option("--restart_section", type="string", default=' ', help="restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')") parser.add_option("--stop_after_candidates", action="store_true", default=False, help="stop after candidates: stops after the 'candidate_class_averages' section. (default False)") ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP) parser.add_option("--skip_prealignment", action="store_true", default=False, help="skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)") required_option_list = ['radius'] (options, args) = parser.parse_args(args) if options.return_options: return parser if len(args) > 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" sys.exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True from isac import iter_isac from fundamentals import rot_shift2D, resample from utilities import pad, combine_params2 command_line_provided_stack_filename = args[0] main_node = 0 mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) nproc = mpi_comm_size(MPI_COMM_WORLD) mpi_barrier(MPI_COMM_WORLD) if(myid == main_node): print "****************************************************************" Util.version() print "****************************************************************" sys.stdout.flush() mpi_barrier(MPI_COMM_WORLD) # Making sure all required options appeared. for required_option in required_option_list: if not options.__dict__[required_option]: print "\n ==%s== mandatory option is missing.\n"%required_option print "Please run '" + progname + " -h' for detailed options" return 1 radi = options.radius target_radius = options.target_radius target_nx = options.target_nx center_method = options.center_method if(radi < 1): ERROR("Particle radius has to be provided!","sxisac",1,myid) use_latest_master_directory = options.use_latest_master_directory stop_after_candidates = options.stop_after_candidates # program_state_stack.restart_location_title_from_command_line = options.restart_section from utilities import qw program_state_stack.PROGRAM_STATE_VARIABLES = set(qw(""" isac_generation """)) # create or reuse master directory masterdir = "" stack_processed_by_ali2d_base__filename = "" stack_processed_by_ali2d_base__filename__without_master_dir = "" error_status = 0 if len(args) == 2: masterdir = args[1] elif len(args) == 1: if use_latest_master_directory: all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)] import re; r = re.compile("^master.*$") all_dirs = filter(r.match, all_dirs) if len(all_dirs)>0: # all_dirs = max(all_dirs, key=os.path.getctime) masterdir = max(all_dirs, key=os.path.getmtime) #Create folder for all results or check if there is one created already if(myid == main_node): if( masterdir == ""): timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime()) masterdir = "master"+timestring cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) elif not os.path.exists(masterdir): # os.path.exists(masterdir) does not exist masterdir = args[1] cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) if(args[0][:4] == "bdb:"): filename = args[0][4:] else: filename = args[0][:-4] filename = os.path.basename(filename) stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(masterdir, filename ) stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename if_error_then_all_processes_exit_program(error_status) # send masterdir to all processes masterdir = send_string_to_all(masterdir) if myid == 0: if options.restart_section != " ": if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)): stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)) import re if "," in options.restart_section: parsed_restart_section_option = options.restart_section.split(",") stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"]) generation_str_format = parsed_restart_section_option[1] if generation_str_format != "": isac_generation_from_command_line = int(generation_str_format) stored_state[-1]["isac_generation"] = isac_generation_from_command_line else: isac_generation_from_command_line = 1 if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"]) if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack) else: print "Please remove the restart_section option from the command line. The program must be started from the beginning." mpi_finalize() sys.exit() else: isac_generation_from_command_line = -1 program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)) stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename) stack_processed_by_ali2d_base__filename__without_master_dir = \ send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir) # previous code 2016-05-05--20-14-12-153 # # PARAMETERS OF THE PROCEDURE # if( options.xr == -1 ): # # Default values # # target_nx = 76 # # target_radius = 29 # target_xr = 1 # else: # nx//2 # # Check below! # target_xr = options.xr # # target_nx = 76 + target_xr - 1 # subtract one, which is default # target_nx += target_xr - 1 # subtract one, which is default # # target_radius = 29 target_xr = options.xr target_nx += target_xr - 1 # subtract one, which is default if (options.yr == -1): yr = options.xr else: yr = options.yr mpi_barrier(MPI_COMM_WORLD) # Initialization of stacks if(myid == main_node): print "command_line_provided_stack_filename", command_line_provided_stack_filename number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename) else: number_of_images_in_stack = 0 number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node) nxrsteps = 4 init2dir = os.path.join(masterdir,"2dalignment") # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() if not os.path.exists(os.path.join(init2dir, "Finished_initial_2d_alignment.txt")): if(myid == 0): import subprocess from logger import Logger, BaseLogger_Files # Create output directory log2d = Logger(BaseLogger_Files()) log2d.prefix = os.path.join(init2dir) cmd = "mkdir -p "+log2d.prefix outcome = subprocess.call(cmd, shell=True) log2d.prefix += "/" # outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True) else: outcome = 0 log2d = None if(myid == main_node): a = get_im(command_line_provided_stack_filename) nnxo = a.get_xsize() else: nnxo = 0 nnxo = bcast_number_to_all(nnxo, source_node = main_node) image_start, image_end = MPI_start_end(number_of_images_in_stack, nproc, myid) if options.skip_prealignment: params2d = [[0.0,0.0,0.0,0] for i in xrange(image_start, image_end)] else: original_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end)) # We assume the target radius will be 29, and xr = 1. shrink_ratio = float(target_radius)/float(radi) for im in xrange(len(original_images)): if(shrink_ratio != 1.0): original_images[im] = resample(original_images[im], shrink_ratio) nx = original_images[0].get_xsize() # nx = int(nx*shrink_ratio + 0.5) txrm = (nx - 2*(target_radius+1))//2 if(txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d"%(radi), "sxisac",1, myid) if(txrm/nxrsteps>0): tss = "" txr = "" while(txrm/nxrsteps>0): tts=txrm/nxrsteps tss += " %d"%tts txr += " %d"%(tts*nxrsteps) txrm =txrm//2 else: tss = "1" txr = "%d"%txrm # print "nx, txr, txrm, tss", nx, txr, txrm, tss # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() # section ali2d_base params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \ False, 90.0, center_method, 14, options.CTF, 1.0, False, \ "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False) del original_images for i in xrange(len(params2d)): alpha, sx, sy, mirror = combine_params2(0, params2d[i][1],params2d[i][2], 0, -params2d[i][0], 0, 0, 0) sx /= shrink_ratio sy /= shrink_ratio params2d[i][0] = 0.0 params2d[i][1] = sx params2d[i][2] = sy params2d[i][3] = 0 #set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0]) mpi_barrier(MPI_COMM_WORLD) tmp = params2d[:] tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD) if( myid == main_node ): if options.skip_prealignment: print "=========================================" print "Even though there is no alignment step, '%s' params are set to zero for later use."%os.path.join(init2dir, "initial2Dparams.txt") print "=========================================" write_text_row(tmp,os.path.join(init2dir, "initial2Dparams.txt")) del tmp mpi_barrier(MPI_COMM_WORLD) # We assume the target image size will be target_nx, radius will be 29, and xr = 1. # Note images can be also padded, in which case shrink_ratio > 1. shrink_ratio = float(target_radius)/float(radi) aligned_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end)) nx = aligned_images[0].get_xsize() nima = len(aligned_images) newx = int(nx*shrink_ratio + 0.5) while not os.path.exists(os.path.join(init2dir, "initial2Dparams.txt")): import time time.sleep(1) mpi_barrier(MPI_COMM_WORLD) params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt")) params = params[image_start:image_end] msk = model_circle(radi, nx, nx) for im in xrange(nima): st = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= st[0] if options.CTF: aligned_images[im] = filt_ctf(aligned_images[im], aligned_images[im].get_attr("ctf"), binary = True) if(shrink_ratio < 1.0): if newx > target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) #aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx : msk = model_circle(newx//2-2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif(shrink_ratio == 1.0): if newx > target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx : msk = model_circle(newx//2-2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) #aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif(shrink_ratio > 1.0): if newx > target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx : msk = model_circle(newx//2-2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) del msk gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid) number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node) if( myid == main_node ): for i in range(number_of_images_in_stack): aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i) # It has to be explicitly closed from EMAN2db import db_open_dict DB = db_open_dict(stack_processed_by_ali2d_base__filename) DB.close() fp = open(os.path.join(masterdir,"README_shrink_ratio.txt"), "w") output_text = """ Since, for processing purposes, isac changes the image dimensions, adjustment of pixel size needs to be made in subsequent steps, (e.g. running sxviper.py). The shrink ratio for this particular isac run is -------- %.5f %.5f -------- To get the pixel size for the isac output the user needs to divide the original pixel size by the above value. This info is saved in the following file: README_shrink_ratio.txt """%(shrink_ratio, radi) fp.write(output_text); fp.flush() ;fp.close() print output_text fp = open(os.path.join(init2dir, "Finished_initial_2d_alignment.txt"), "w"); fp.flush() ;fp.close() else: if( myid == main_node ): print "Skipping 2d alignment since it was already done!" mpi_barrier(MPI_COMM_WORLD) # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() os.chdir(masterdir) if program_state_stack(locals(), getframeinfo(currentframe())): # if 1: pass if (myid == main_node): cmdexecute("sxheader.py --consecutive --params=originalid %s"%stack_processed_by_ali2d_base__filename__without_master_dir) cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir)) if (myid == main_node): main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d") print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no if isac_generation_from_command_line < 0: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1): if i == isac_generation_from_command_line+1: backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%i + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i) # it includes both command line and json file my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1] if "restart" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line ) elif "candidate_class_averages" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line) # cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line) elif "reproducible_class_averages" in my_restart_section: cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line) cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line) else: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 0 isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0] isac_generation = isac_generation_from_command_line - 1 if (myid == main_node): if isac_generation == 0: cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation) write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation)) write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation)) # Stopping criterion should be inside the program. while True: isac_generation += 1 if isac_generation > options.n_generations: break data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation program_state_stack.restart_location_title = "restart" if program_state_stack(locals(), getframeinfo(currentframe())): if (myid == main_node): cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation) # reference the original stack list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1)) cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\ stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file)) mpi_barrier(MPI_COMM_WORLD) os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation) program_state_stack.restart_location_title = "candidate_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new) # program_state_stack.restart_location_title = "stopped_program1" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack.restart_location_title = "stop_after_candidates" program_state_stack(locals(), getframeinfo(currentframe())) if stop_after_candidates: mpi_finalize() sys.exit() exit_program = 0 if(myid == main_node): if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation): print "This generation (%d) no class average candidates were generated! Finishing."%isac_generation exit_program = 1 exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if exit_program: os.chdir("..") break program_state_stack.restart_location_title = "reproducible_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new) pass os.chdir("..") if(myid == main_node): accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_accounted.txt"%(isac_generation))) number_of_accounted_images = len(accounted_images) un_accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_unaccounted.txt"%(isac_generation))) number_of_un_accounted_images = len(un_accounted_images) else: number_of_accounted_images = 0 number_of_un_accounted_images = 0 number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) number_of_un_accounted_images = int(mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if number_of_accounted_images == 0: if(myid == main_node): print "This generation (%d) there are no accounted images! Finishing."%isac_generation break while (myid == main_node): def files_are_missing(isac_generation): for i in xrange(1, isac_generation + 1): if not os.path.exists("generation_%04d/class_averages_generation_%d.hdf"%(i,i)): print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting."%(i,i) return 1 return 0 if files_are_missing(isac_generation): break cmdexecute("rm -f class_averages.hdf") cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation + 1)], "class_averages.hdf") break if number_of_un_accounted_images == 0: if(myid == main_node): print "This generation (%d) there are no un accounted images! Finishing."%isac_generation break program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall") mpi_barrier(MPI_COMM_WORLD) mpi_finalize()
def shiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1, oneDx=False, search_rng_y=-1): from applications import MPI_start_end from utilities import model_circle, model_blank, get_image, peak_search, get_im from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all from pap_statistics import varf2d_MPI from fundamentals import fft, ccf, rot_shift3D, rot_shift2D from utilities import get_params2D, set_params2D from utilities import print_msg, print_begin_msg, print_end_msg import os import sys from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv from mpi import MPI_SUM, MPI_FLOAT, MPI_INT from EMAN2 import Processor from time import time number_of_proc = mpi_comm_size(MPI_COMM_WORLD) myid = mpi_comm_rank(MPI_COMM_WORLD) main_node = 0 ftp = file_type(stack) if myid == main_node: print_begin_msg("shiftali_MPI") max_iter = int(maxit) if myid == main_node: if ftp == "bdb": from EMAN2db import db_open_dict dummy = db_open_dict(stack, True) nima = EMUtil.get_image_count(stack) else: nima = 0 nima = bcast_number_to_all(nima, source_node=main_node) list_of_particles = list(range(nima)) image_start, image_end = MPI_start_end(nima, number_of_proc, myid) list_of_particles = list_of_particles[image_start:image_end] # read nx and ctf_app (if CTF) and broadcast to all nodes if myid == main_node: ima = EMData() ima.read_image(stack, list_of_particles[0], True) nx = ima.get_xsize() ny = ima.get_ysize() if CTF: ctf_app = ima.get_attr_default('ctf_applied', 2) del ima else: nx = 0 ny = 0 if CTF: ctf_app = 0 nx = bcast_number_to_all(nx, source_node=main_node) ny = bcast_number_to_all(ny, source_node=main_node) if CTF: ctf_app = bcast_number_to_all(ctf_app, source_node=main_node) if ctf_app > 0: ERROR("data cannot be ctf-applied", "shiftali_MPI", 1, myid) if maskfile == None: mrad = min(nx, ny) mask = model_circle(mrad // 2 - 2, nx, ny) else: mask = get_im(maskfile) if CTF: from filter import filt_ctf from morphology import ctf_img ctf_abs_sum = EMData(nx, ny, 1, False) ctf_2_sum = EMData(nx, ny, 1, False) else: ctf_2_sum = None from global_def import CACHE_DISABLE if CACHE_DISABLE: data = EMData.read_images(stack, list_of_particles) else: for i in range(number_of_proc): if myid == i: data = EMData.read_images(stack, list_of_particles) if ftp == "bdb": mpi_barrier(MPI_COMM_WORLD) for im in range(len(data)): data[im].set_attr('ID', list_of_particles[im]) st = Util.infomask(data[im], mask, False) data[im] -= st[0] if CTF: ctf_params = data[im].get_attr("ctf") ctfimg = ctf_img(nx, ctf_params, ny=ny) Util.add_img2(ctf_2_sum, ctfimg) Util.add_img_abs(ctf_abs_sum, ctfimg) if CTF: reduce_EMData_to_root(ctf_2_sum, myid, main_node) reduce_EMData_to_root(ctf_abs_sum, myid, main_node) else: ctf_2_sum = None if CTF: if myid != main_node: del ctf_2_sum del ctf_abs_sum else: temp = EMData(nx, ny, 1, False) for i in range(0, nx, 2): for j in range(ny): temp.set_value_at(i, j, snr) Util.add_img(ctf_2_sum, temp) del temp total_iter = 0 # apply initial xform.align2d parameters stored in header init_params = [] for im in range(len(data)): t = data[im].get_attr('xform.align2d') init_params.append(t) p = t.get_params("2d") data[im] = rot_shift2D(data[im], p['alpha'], sx=p['tx'], sy=p['ty'], mirror=p['mirror'], scale=p['scale']) # fourier transform all images, and apply ctf if CTF for im in range(len(data)): if CTF: ctf_params = data[im].get_attr("ctf") data[im] = filt_ctf(fft(data[im]), ctf_params) else: data[im] = fft(data[im]) sx_sum = 0 sy_sum = 0 sx_sum_total = 0 sy_sum_total = 0 shift_x = [0.0] * len(data) shift_y = [0.0] * len(data) ishift_x = [0.0] * len(data) ishift_y = [0.0] * len(data) for Iter in range(max_iter): if myid == main_node: start_time = time() print_msg("Iteration #%4d\n" % (total_iter)) total_iter += 1 avg = EMData(nx, ny, 1, False) for im in data: Util.add_img(avg, im) reduce_EMData_to_root(avg, myid, main_node) if myid == main_node: if CTF: tavg = Util.divn_filter(avg, ctf_2_sum) else: tavg = Util.mult_scalar(avg, 1.0 / float(nima)) else: tavg = EMData(nx, ny, 1, False) if Fourvar: bcast_EMData_to_all(tavg, myid, main_node) vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF) if myid == main_node: if Fourvar: tavg = fft(Util.divn_img(fft(tavg), vav)) vav_r = Util.pack_complex_to_real(vav) # normalize and mask tavg in real space tavg = fft(tavg) stat = Util.infomask(tavg, mask, False) tavg -= stat[0] Util.mul_img(tavg, mask) # For testing purposes: shift tavg to some random place and see if the centering is still correct #tavg = rot_shift3D(tavg,sx=3,sy=-4) tavg = fft(tavg) if Fourvar: del vav bcast_EMData_to_all(tavg, myid, main_node) sx_sum = 0 sy_sum = 0 if search_rng > 0: nwx = 2 * search_rng + 1 else: nwx = nx if search_rng_y > 0: nwy = 2 * search_rng_y + 1 else: nwy = ny not_zero = 0 for im in range(len(data)): if oneDx: ctx = Util.window(ccf(data[im], tavg), nwx, 1) p1 = peak_search(ctx) p1_x = -int(p1[0][3]) ishift_x[im] = p1_x sx_sum += p1_x else: p1 = peak_search(Util.window(ccf(data[im], tavg), nwx, nwy)) p1_x = -int(p1[0][4]) p1_y = -int(p1[0][5]) ishift_x[im] = p1_x ishift_y[im] = p1_y sx_sum += p1_x sy_sum += p1_y if not_zero == 0: if (not (ishift_x[im] == 0.0)) or (not (ishift_y[im] == 0.0)): not_zero = 1 sx_sum = mpi_reduce(sx_sum, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if not oneDx: sy_sum = mpi_reduce(sy_sum, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: sx_sum_total = int(sx_sum[0]) if not oneDx: sy_sum_total = int(sy_sum[0]) else: sx_sum_total = 0 sy_sum_total = 0 sx_sum_total = bcast_number_to_all(sx_sum_total, source_node=main_node) if not oneDx: sy_sum_total = bcast_number_to_all(sy_sum_total, source_node=main_node) sx_ave = round(float(sx_sum_total) / nima) sy_ave = round(float(sy_sum_total) / nima) for im in range(len(data)): p1_x = ishift_x[im] - sx_ave p1_y = ishift_y[im] - sy_ave params2 = { "filter_type": Processor.fourier_filter_types.SHIFT, "x_shift": p1_x, "y_shift": p1_y, "z_shift": 0.0 } data[im] = Processor.EMFourierFilter(data[im], params2) shift_x[im] += p1_x shift_y[im] += p1_y # stop if all shifts are zero not_zero = mpi_reduce(not_zero, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: not_zero_all = int(not_zero[0]) else: not_zero_all = 0 not_zero_all = bcast_number_to_all(not_zero_all, source_node=main_node) if myid == main_node: print_msg("Time of iteration = %12.2f\n" % (time() - start_time)) start_time = time() if not_zero_all == 0: break #for im in xrange(len(data)): data[im] = fft(data[im]) This should not be required as only header information is used # combine shifts found with the original parameters for im in range(len(data)): t0 = init_params[im] t1 = Transform() t1.set_params({ "type": "2D", "alpha": 0, "scale": t0.get_scale(), "mirror": 0, "tx": shift_x[im], "ty": shift_y[im] }) # combine t0 and t1 tt = t1 * t0 data[im].set_attr("xform.align2d", tt) # write out headers and STOP, under MPI writing has to be done sequentially mpi_barrier(MPI_COMM_WORLD) par_str = ["xform.align2d", "ID"] if myid == main_node: from utilities import file_type if (file_type(stack) == "bdb"): from utilities import recv_attr_dict_bdb recv_attr_dict_bdb(main_node, stack, data, par_str, image_start, image_end, number_of_proc) else: from utilities import recv_attr_dict recv_attr_dict(main_node, stack, data, par_str, image_start, image_end, number_of_proc) else: send_attr_dict(main_node, data, par_str, image_start, image_end) if myid == main_node: print_end_msg("shiftali_MPI")
def shiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1, oneDx=False, search_rng_y=-1): from applications import MPI_start_end from utilities import model_circle, model_blank, get_image, peak_search, get_im from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all from statistics import varf2d_MPI from fundamentals import fft, ccf, rot_shift3D, rot_shift2D from utilities import get_params2D, set_params2D from utilities import print_msg, print_begin_msg, print_end_msg import os import sys from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv from mpi import MPI_SUM, MPI_FLOAT, MPI_INT from EMAN2 import Processor from time import time number_of_proc = mpi_comm_size(MPI_COMM_WORLD) myid = mpi_comm_rank(MPI_COMM_WORLD) main_node = 0 ftp = file_type(stack) if myid == main_node: print_begin_msg("shiftali_MPI") max_iter=int(maxit) if myid == main_node: if ftp == "bdb": from EMAN2db import db_open_dict dummy = db_open_dict(stack, True) nima = EMUtil.get_image_count(stack) else: nima = 0 nima = bcast_number_to_all(nima, source_node = main_node) list_of_particles = range(nima) image_start, image_end = MPI_start_end(nima, number_of_proc, myid) list_of_particles = list_of_particles[image_start: image_end] # read nx and ctf_app (if CTF) and broadcast to all nodes if myid == main_node: ima = EMData() ima.read_image(stack, list_of_particles[0], True) nx = ima.get_xsize() ny = ima.get_ysize() if CTF: ctf_app = ima.get_attr_default('ctf_applied', 2) del ima else: nx = 0 ny = 0 if CTF: ctf_app = 0 nx = bcast_number_to_all(nx, source_node = main_node) ny = bcast_number_to_all(ny, source_node = main_node) if CTF: ctf_app = bcast_number_to_all(ctf_app, source_node = main_node) if ctf_app > 0: ERROR("data cannot be ctf-applied", "shiftali_MPI", 1, myid) if maskfile == None: mrad = min(nx, ny) mask = model_circle(mrad//2-2, nx, ny) else: mask = get_im(maskfile) if CTF: from filter import filt_ctf from morphology import ctf_img ctf_abs_sum = EMData(nx, ny, 1, False) ctf_2_sum = EMData(nx, ny, 1, False) else: ctf_2_sum = None from global_def import CACHE_DISABLE if CACHE_DISABLE: data = EMData.read_images(stack, list_of_particles) else: for i in xrange(number_of_proc): if myid == i: data = EMData.read_images(stack, list_of_particles) if ftp == "bdb": mpi_barrier(MPI_COMM_WORLD) for im in xrange(len(data)): data[im].set_attr('ID', list_of_particles[im]) st = Util.infomask(data[im], mask, False) data[im] -= st[0] if CTF: ctf_params = data[im].get_attr("ctf") ctfimg = ctf_img(nx, ctf_params, ny=ny) Util.add_img2(ctf_2_sum, ctfimg) Util.add_img_abs(ctf_abs_sum, ctfimg) if CTF: reduce_EMData_to_root(ctf_2_sum, myid, main_node) reduce_EMData_to_root(ctf_abs_sum, myid, main_node) else: ctf_2_sum = None if CTF: if myid != main_node: del ctf_2_sum del ctf_abs_sum else: temp = EMData(nx, ny, 1, False) for i in xrange(0,nx,2): for j in xrange(ny): temp.set_value_at(i,j,snr) Util.add_img(ctf_2_sum, temp) del temp total_iter = 0 # apply initial xform.align2d parameters stored in header init_params = [] for im in xrange(len(data)): t = data[im].get_attr('xform.align2d') init_params.append(t) p = t.get_params("2d") data[im] = rot_shift2D(data[im], p['alpha'], sx=p['tx'], sy=p['ty'], mirror=p['mirror'], scale=p['scale']) # fourier transform all images, and apply ctf if CTF for im in xrange(len(data)): if CTF: ctf_params = data[im].get_attr("ctf") data[im] = filt_ctf(fft(data[im]), ctf_params) else: data[im] = fft(data[im]) sx_sum=0 sy_sum=0 sx_sum_total=0 sy_sum_total=0 shift_x = [0.0]*len(data) shift_y = [0.0]*len(data) ishift_x = [0.0]*len(data) ishift_y = [0.0]*len(data) for Iter in xrange(max_iter): if myid == main_node: start_time = time() print_msg("Iteration #%4d\n"%(total_iter)) total_iter += 1 avg = EMData(nx, ny, 1, False) for im in data: Util.add_img(avg, im) reduce_EMData_to_root(avg, myid, main_node) if myid == main_node: if CTF: tavg = Util.divn_filter(avg, ctf_2_sum) else: tavg = Util.mult_scalar(avg, 1.0/float(nima)) else: tavg = EMData(nx, ny, 1, False) if Fourvar: bcast_EMData_to_all(tavg, myid, main_node) vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF) if myid == main_node: if Fourvar: tavg = fft(Util.divn_img(fft(tavg), vav)) vav_r = Util.pack_complex_to_real(vav) # normalize and mask tavg in real space tavg = fft(tavg) stat = Util.infomask( tavg, mask, False ) tavg -= stat[0] Util.mul_img(tavg, mask) # For testing purposes: shift tavg to some random place and see if the centering is still correct #tavg = rot_shift3D(tavg,sx=3,sy=-4) tavg = fft(tavg) if Fourvar: del vav bcast_EMData_to_all(tavg, myid, main_node) sx_sum=0 sy_sum=0 if search_rng > 0: nwx = 2*search_rng+1 else: nwx = nx if search_rng_y > 0: nwy = 2*search_rng_y+1 else: nwy = ny not_zero = 0 for im in xrange(len(data)): if oneDx: ctx = Util.window(ccf(data[im],tavg),nwx,1) p1 = peak_search(ctx) p1_x = -int(p1[0][3]) ishift_x[im] = p1_x sx_sum += p1_x else: p1 = peak_search(Util.window(ccf(data[im],tavg), nwx,nwy)) p1_x = -int(p1[0][4]) p1_y = -int(p1[0][5]) ishift_x[im] = p1_x ishift_y[im] = p1_y sx_sum += p1_x sy_sum += p1_y if not_zero == 0: if (not(ishift_x[im] == 0.0)) or (not(ishift_y[im] == 0.0)): not_zero = 1 sx_sum = mpi_reduce(sx_sum, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if not oneDx: sy_sum = mpi_reduce(sy_sum, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: sx_sum_total = int(sx_sum[0]) if not oneDx: sy_sum_total = int(sy_sum[0]) else: sx_sum_total = 0 sy_sum_total = 0 sx_sum_total = bcast_number_to_all(sx_sum_total, source_node = main_node) if not oneDx: sy_sum_total = bcast_number_to_all(sy_sum_total, source_node = main_node) sx_ave = round(float(sx_sum_total)/nima) sy_ave = round(float(sy_sum_total)/nima) for im in xrange(len(data)): p1_x = ishift_x[im] - sx_ave p1_y = ishift_y[im] - sy_ave params2 = {"filter_type" : Processor.fourier_filter_types.SHIFT, "x_shift" : p1_x, "y_shift" : p1_y, "z_shift" : 0.0} data[im] = Processor.EMFourierFilter(data[im], params2) shift_x[im] += p1_x shift_y[im] += p1_y # stop if all shifts are zero not_zero = mpi_reduce(not_zero, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: not_zero_all = int(not_zero[0]) else: not_zero_all = 0 not_zero_all = bcast_number_to_all(not_zero_all, source_node = main_node) if myid == main_node: print_msg("Time of iteration = %12.2f\n"%(time()-start_time)) start_time = time() if not_zero_all == 0: break #for im in xrange(len(data)): data[im] = fft(data[im]) This should not be required as only header information is used # combine shifts found with the original parameters for im in xrange(len(data)): t0 = init_params[im] t1 = Transform() t1.set_params({"type":"2D","alpha":0,"scale":t0.get_scale(),"mirror":0,"tx":shift_x[im],"ty":shift_y[im]}) # combine t0 and t1 tt = t1*t0 data[im].set_attr("xform.align2d", tt) # write out headers and STOP, under MPI writing has to be done sequentially mpi_barrier(MPI_COMM_WORLD) par_str = ["xform.align2d", "ID"] if myid == main_node: from utilities import file_type if(file_type(stack) == "bdb"): from utilities import recv_attr_dict_bdb recv_attr_dict_bdb(main_node, stack, data, par_str, image_start, image_end, number_of_proc) else: from utilities import recv_attr_dict recv_attr_dict(main_node, stack, data, par_str, image_start, image_end, number_of_proc) else: send_attr_dict(main_node, data, par_str, image_start, image_end) if myid == main_node: print_end_msg("shiftali_MPI")
def helicalshiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1): from applications import MPI_start_end from utilities import model_circle, model_blank, get_image, peak_search, get_im, pad from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all from statistics import varf2d_MPI from fundamentals import fft, ccf, rot_shift3D, rot_shift2D, fshift from utilities import get_params2D, set_params2D, chunks_distribution from utilities import print_msg, print_begin_msg, print_end_msg import os import sys from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv from mpi import MPI_SUM, MPI_FLOAT, MPI_INT from time import time from pixel_error import ordersegments from math import sqrt, atan2, tan, pi nproc = mpi_comm_size(MPI_COMM_WORLD) myid = mpi_comm_rank(MPI_COMM_WORLD) main_node = 0 ftp = file_type(stack) if myid == main_node: print_begin_msg("helical-shiftali_MPI") max_iter=int(maxit) if( myid == main_node): infils = EMUtil.get_all_attributes(stack, "filament") ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord') filaments = ordersegments(infils, ptlcoords) total_nfils = len(filaments) inidl = [0]*total_nfils for i in xrange(total_nfils): inidl[i] = len(filaments[i]) linidl = sum(inidl) nima = linidl tfilaments = [] for i in xrange(total_nfils): tfilaments += filaments[i] del filaments else: total_nfils = 0 linidl = 0 total_nfils = bcast_number_to_all(total_nfils, source_node = main_node) if myid != main_node: inidl = [-1]*total_nfils inidl = bcast_list_to_all(inidl, myid, source_node = main_node) linidl = bcast_number_to_all(linidl, source_node = main_node) if myid != main_node: tfilaments = [-1]*linidl tfilaments = bcast_list_to_all(tfilaments, myid, source_node = main_node) filaments = [] iendi = 0 for i in xrange(total_nfils): isti = iendi iendi = isti+inidl[i] filaments.append(tfilaments[isti:iendi]) del tfilaments,inidl if myid == main_node: print_msg( "total number of filaments: %d"%total_nfils) if total_nfils< nproc: ERROR('number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'%(nproc, total_nfils), "ehelix_MPI", 1,myid) # balanced load temp = chunks_distribution([[len(filaments[i]), i] for i in xrange(len(filaments))], nproc)[myid:myid+1][0] filaments = [filaments[temp[i][1]] for i in xrange(len(temp))] nfils = len(filaments) #filaments = [[0,1]] #print "filaments",filaments list_of_particles = [] indcs = [] k = 0 for i in xrange(nfils): list_of_particles += filaments[i] k1 = k+len(filaments[i]) indcs.append([k,k1]) k = k1 data = EMData.read_images(stack, list_of_particles) ldata = len(data) print "ldata=", ldata nx = data[0].get_xsize() ny = data[0].get_ysize() if maskfile == None: mrad = min(nx, ny)//2-2 mask = pad( model_blank(2*mrad+1, ny, 1, 1.0), nx, ny, 1, 0.0) else: mask = get_im(maskfile) # apply initial xform.align2d parameters stored in header init_params = [] for im in xrange(ldata): t = data[im].get_attr('xform.align2d') init_params.append(t) p = t.get_params("2d") data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'], p['mirror'], p['scale']) if CTF: from filter import filt_ctf from morphology import ctf_img ctf_abs_sum = EMData(nx, ny, 1, False) ctf_2_sum = EMData(nx, ny, 1, False) else: ctf_2_sum = None ctf_abs_sum = None from utilities import info for im in xrange(ldata): data[im].set_attr('ID', list_of_particles[im]) st = Util.infomask(data[im], mask, False) data[im] -= st[0] if CTF: ctf_params = data[im].get_attr("ctf") qctf = data[im].get_attr("ctf_applied") if qctf == 0: data[im] = filt_ctf(fft(data[im]), ctf_params) data[im].set_attr('ctf_applied', 1) elif qctf != 1: ERROR('Incorrectly set qctf flag', "helicalshiftali_MPI", 1,myid) ctfimg = ctf_img(nx, ctf_params, ny=ny) Util.add_img2(ctf_2_sum, ctfimg) Util.add_img_abs(ctf_abs_sum, ctfimg) else: data[im] = fft(data[im]) del list_of_particles if CTF: reduce_EMData_to_root(ctf_2_sum, myid, main_node) reduce_EMData_to_root(ctf_abs_sum, myid, main_node) if CTF: if myid != main_node: del ctf_2_sum del ctf_abs_sum else: temp = EMData(nx, ny, 1, False) tsnr = 1./snr for i in xrange(0,nx+2,2): for j in xrange(ny): temp.set_value_at(i,j,tsnr) temp.set_value_at(i+1,j,0.0) #info(ctf_2_sum) Util.add_img(ctf_2_sum, temp) #info(ctf_2_sum) del temp total_iter = 0 shift_x = [0.0]*ldata for Iter in xrange(max_iter): if myid == main_node: start_time = time() print_msg("Iteration #%4d\n"%(total_iter)) total_iter += 1 avg = EMData(nx, ny, 1, False) for im in xrange(ldata): Util.add_img(avg, fshift(data[im], shift_x[im])) reduce_EMData_to_root(avg, myid, main_node) if myid == main_node: if CTF: tavg = Util.divn_filter(avg, ctf_2_sum) else: tavg = Util.mult_scalar(avg, 1.0/float(nima)) else: tavg = model_blank(nx,ny) if Fourvar: bcast_EMData_to_all(tavg, myid, main_node) vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF) if myid == main_node: if Fourvar: tavg = fft(Util.divn_img(fft(tavg), vav)) vav_r = Util.pack_complex_to_real(vav) # normalize and mask tavg in real space tavg = fft(tavg) stat = Util.infomask( tavg, mask, False ) tavg -= stat[0] Util.mul_img(tavg, mask) tavg.write_image("tavg.hdf",Iter) # For testing purposes: shift tavg to some random place and see if the centering is still correct #tavg = rot_shift3D(tavg,sx=3,sy=-4) if Fourvar: del vav bcast_EMData_to_all(tavg, myid, main_node) tavg = fft(tavg) sx_sum = 0.0 nxc = nx//2 for ifil in xrange(nfils): """ # Calculate filament average avg = EMData(nx, ny, 1, False) filnima = 0 for im in xrange(indcs[ifil][0], indcs[ifil][1]): Util.add_img(avg, data[im]) filnima += 1 tavg = Util.mult_scalar(avg, 1.0/float(filnima)) """ # Calculate 1D ccf between each segment and filament average nsegms = indcs[ifil][1]-indcs[ifil][0] ctx = [None]*nsegms pcoords = [None]*nsegms for im in xrange(indcs[ifil][0], indcs[ifil][1]): ctx[im-indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx, 1) pcoords[im-indcs[ifil][0]] = data[im].get_attr('ptcl_source_coord') #ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0]) #print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True) # search for best x-shift cents = nsegms//2 dst = sqrt(max((pcoords[cents][0] - pcoords[0][0])**2 + (pcoords[cents][1] - pcoords[0][1])**2, (pcoords[cents][0] - pcoords[-1][0])**2 + (pcoords[cents][1] - pcoords[-1][1])**2)) maxincline = atan2(ny//2-2-float(search_rng),dst) kang = int(dst*tan(maxincline)+0.5) #print " settings ",nsegms,cents,dst,search_rng,maxincline,kang # ## C code for alignment. @ming results = [0.0]*3; results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline, kang, search_rng,nxc) sib = int(results[0]) bang = results[1] qm = results[2] #print qm, sib, bang # qm = -1.e23 # # for six in xrange(-search_rng, search_rng+1,1): # q0 = ctx[cents].get_value_at(six+nxc) # for incline in xrange(kang+1): # qt = q0 # qu = q0 # if(kang>0): tang = tan(maxincline/kang*incline) # else: tang = 0.0 # for kim in xrange(cents+1,nsegms): # dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) # xl = dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # #print " A ", ifil,six,incline,kim,xl,ixl,dxl # qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # xl = -dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # for kim in xrange(cents): # dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) # xl = -dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # xl = dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # if( qt > qm ): # qm = qt # sib = six # bang = tang # if( qu > qm ): # qm = qu # sib = six # bang = -tang #if incline == 0: print "incline = 0 ",six,tang,qt,qu #print qm,six,sib,bang #print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True) for im in xrange(indcs[ifil][0], indcs[ifil][1]): kim = im-indcs[ifil][0] dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) if(kim < cents): xl = -dst*bang+sib else: xl = dst*bang+sib shift_x[im] = xl # Average shift sx_sum += shift_x[indcs[ifil][0]+cents] # #print myid,sx_sum,total_nfils sx_sum = mpi_reduce(sx_sum, 1, MPI_FLOAT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: sx_sum = float(sx_sum[0])/total_nfils print_msg("Average shift %6.2f\n"%(sx_sum)) else: sx_sum = 0.0 sx_sum = 0.0 sx_sum = bcast_number_to_all(sx_sum, source_node = main_node) for im in xrange(ldata): shift_x[im] -= sx_sum #print " %3d %6.3f"%(im,shift_x[im]) #exit() # combine shifts found with the original parameters for im in xrange(ldata): t1 = Transform() ##import random ##shix=random.randint(-10, 10) ##t1.set_params({"type":"2D","tx":shix}) t1.set_params({"type":"2D","tx":shift_x[im]}) # combine t0 and t1 tt = t1*init_params[im] data[im].set_attr("xform.align2d", tt) # write out headers and STOP, under MPI writing has to be done sequentially mpi_barrier(MPI_COMM_WORLD) par_str = ["xform.align2d", "ID"] if myid == main_node: from utilities import file_type if(file_type(stack) == "bdb"): from utilities import recv_attr_dict_bdb recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata, nproc) else: from utilities import recv_attr_dict recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc) else: send_attr_dict(main_node, data, par_str, 0, ldata) if myid == main_node: print_end_msg("helical-shiftali_MPI")
def main(args): progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" + " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" + " --generation=generation --rand_seed=rand_seed>" ) parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--radius", type="int", default=-1, help="<Particle radius>, it has to be provided.") parser.add_option("--img_per_grp", type="int", default=100, help="<number of images per group> in the ideal case (essentially maximum size of class) (100)") parser.add_option("--CTF", action="store_true", default=False, help="<CTF flag>, if set the data will be phase-flipped") parser.add_option("--ir", type="int", default=1, help="<inner ring> of the resampling to polar coordinates (1)") parser.add_option("--rs", type="int", default=1, help="<ring step> of the resampling to polar coordinates (1)") parser.add_option("--xr", type="int", default=-1, help="<x range> of translational search (By default set by the program) (advanced)") parser.add_option("--yr", type="int", default=-1, help="<y range> of translational search (same as xr) (advanced)") parser.add_option("--ts", type="float", default=1.0, help="<search step> of translational search (1.0)") parser.add_option("--maxit", type="int", default=30, help="number of iterations for reference-free alignment (30)") #parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)") parser.add_option("--center_method", type="int", default=7, help="<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)") parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment ") parser.add_option("--FL", type="float", default=0.2, help="<lowest stopband> frequency used in the tangent filter (0.2)") parser.add_option("--FH", type="float", default=0.3, help="<highest stopband> frequency used in the tangent filter (0.3)") parser.add_option("--FF", type="float", default=0.2, help="<fall-off of the tangent> filter (0.2)") parser.add_option("--init_iter", type="int", default=3, help="<init_iter> number of iterations of ISAC program in initialization (3)") parser.add_option("--main_iter", type="int", default=3, help="<main_iter> number of iterations of ISAC program in main part (3)") parser.add_option("--iter_reali", type="int", default=1, help="<iter_reali> number of iterations in ISAC before checking stability (1)") parser.add_option("--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase (1)") parser.add_option("--max_round", type="int", default=20, help="maximum rounds of generating candidate averages in the first phase (20)") parser.add_option("--match_second", type="int", default=5, help="number of iterations to run 2-way (or 3-way) matching in the second phase (5)") parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability (5)") parser.add_option("--thld_err", type="float", default=0.7, help="the threshold of pixel error when checking stability (0.7)") parser.add_option("--indep_run", type="int", default=4, help="number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)") parser.add_option("--thld_grp", type="int", default=10, help="minimum size of class (10)") parser.add_option("--n_generations", type="int", default=100, help="<n_generations> program stops when reaching this total number of generations (advanced)") #parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)") parser.add_option("--rand_seed", type="int", default=None, help="random seed set before calculations, useful for testing purposes (default None - total randomness)") parser.add_option("--new", action="store_true", default=False, help="use new code (default = False)") parser.add_option("--debug", action="store_true", default=False, help="debug info printout (default = False)") # must be switched off in production parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False) parser.add_option("--restart_section", type="string", default="", help="<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)") parser.add_option("--stop_after_candidates", action="store_true", default=False, help="<stop_after_candidates> stops after the 'candidate_class_averages' section") parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP) (options, args) = parser.parse_args(args) if options.return_options: return parser if len(args) > 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" sys.exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() from isac import iter_isac global_def.BATCH = True global_def.BATCH = True command_line_provided_stack_filename = args[0] global_def.BATCH = True main_node = 0 mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) nproc = mpi_comm_size(MPI_COMM_WORLD) radi = options.radius center_method = options.center_method if(radi < 1): ERROR("Particle radius has to be provided!","sxisac",1,myid) use_latest_master_directory = options.use_latest_master_directory stop_after_candidates = options.stop_after_candidates program_state_stack.restart_location_title_from_command_line = options.restart_section from utilities import qw program_state_stack.PROGRAM_STATE_VARIABLES = set(qw(""" isac_generation """)) # create or reuse master directory masterdir = "" stack_processed_by_ali2d_base__filename = "" stack_processed_by_ali2d_base__filename__without_master_dir = "" error_status = 0 if len(args) == 2: masterdir = args[1] elif len(args) == 1: if use_latest_master_directory: all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)] import re; r = re.compile("^master.*$") all_dirs = filter(r.match, all_dirs) if len(all_dirs)>0: # all_dirs = max(all_dirs, key=os.path.getctime) masterdir = max(all_dirs, key=os.path.getmtime) #Create folder for all results or check if there is one created already if(myid == main_node): if( masterdir == ""): timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime()) masterdir = "master"+timestring cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) elif not os.path.exists(masterdir): # os.path.exists(masterdir) does not exist masterdir = args[1] cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) if(args[0][:4] == "bdb:"): filename = args[0][4:] else: filename = args[0][:-4] filename = os.path.basename(filename) stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(masterdir, filename ) stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename if_error_all_processes_quit_program(error_status) # send masterdir to all processes masterdir = send_string_to_all(masterdir) if myid == 0: if options.restart_section != "": if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)): stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)) import re if "," in options.restart_section: parsed_restart_section_option = options.restart_section.split(",") stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"]) generation_str_format = parsed_restart_section_option[1] if generation_str_format != "": isac_generation_from_command_line = int(generation_str_format) stored_state[-1]["isac_generation"] = isac_generation_from_command_line else: isac_generation_from_command_line = 1 if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"]) if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack) else: print "Please remove the restart_section option from the command line. The program must be started from the beginning." mpi_finalize() sys.exit() else: isac_generation_from_command_line = -1 program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)) stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename) stack_processed_by_ali2d_base__filename__without_master_dir = \ send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir) # PARAMETERS OF THE PROCEDURE if( options.xr == -1 ): # Default values target_nx = 76 target_radius = 29 target_xr = 1 else: # nx//2 # Check below! target_xr = options.xr target_nx = 76 + target_xr - 1 # subtract one, which is default target_radius = 29 mpi_barrier(MPI_COMM_WORLD) # Initialization of stacks if(myid == main_node): number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename) else: number_of_images_in_stack = 0 number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node) nxrsteps = 4 init2dir = os.path.join(masterdir,"2dalignment") if(myid == 0): import subprocess from logger import Logger, BaseLogger_Files # Create output directory log2d = Logger(BaseLogger_Files()) log2d.prefix = os.path.join(init2dir) cmd = "mkdir -p "+log2d.prefix outcome = subprocess.call(cmd, shell=True) log2d.prefix += "/" # outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True) else: outcome = 0 log2d = None if(myid == main_node): a = get_im(command_line_provided_stack_filename) nnxo = a.get_xsize() else: nnxo = 0 nnxo = bcast_number_to_all(nnxo, source_node = main_node) txrm = (nnxo - 2*(radi+1))//2 if(txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d"%(radi), "sxisac",1, myid) if(txrm/nxrsteps>0): tss = "" txr = "" while(txrm/nxrsteps>0): tts=txrm/nxrsteps tss += " %d"%tts txr += " %d"%(tts*nxrsteps) txrm =txrm//2 else: tss = "1" txr = "%d"%txrm # section ali2d_base # centering method is set to #7 params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \ False, 90.0, center_method, 14, options.CTF, 1.0, False, \ "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False) if( myid == main_node ): write_text_row(params2d,os.path.join(init2dir, "initial2Dparams.txt")) del params2d mpi_barrier(MPI_COMM_WORLD) # We assume the target image size will be target_nx, radius will be 29, and xr = 1. # Note images can be also padded, in which case shrink_ratio > 1. shrink_ratio = float(target_radius)/float(radi) nx = aligned_images[0].get_xsize() nima = len(aligned_images) newx = int(nx*shrink_ratio + 0.5) from fundamentals import rot_shift2D, resample from utilities import pad, combine_params2 if(shrink_ratio < 1.0): if newx > target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx : msk = model_circle(newx//2-2, newx, newx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif(shrink_ratio == 1.0): if newx > target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx : msk = model_circle(nx//2-2, newx, newx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) #aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif(shrink_ratio > 1.0): target_radius = radi msk = model_circle(target_radius, nx, nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) del msk gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid) number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node) if( myid == main_node ): for i in range(number_of_images_in_stack): aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i) # It has to be explicitly closed from EMAN2db import db_open_dict DB = db_open_dict(stack_processed_by_ali2d_base__filename) DB.close() fp = open("README_shrink_ratio.txt", "w") output_text = """ Since, for processing purposes, isac changes the image dimensions, adjustment of pixel size needs to be made in subsequent steps, (e.g. running sxviper.py). The shrink ratio for this particular isac run is -------- %.5f -------- To get the pixel size for the isac output the user needs to divide the original pixel size by the above value. This info is saved in the following file: README_shrink_ratio.txt """%shrink_ratio fp.write(output_text); fp.flush() ;fp.close() print output_text mpi_barrier(MPI_COMM_WORLD) global_def.BATCH = True os.chdir(masterdir) if program_state_stack(locals(), getframeinfo(currentframe())): # if 1: pass if (myid == main_node): cmdexecute("sxheader.py --consecutive --params=originalid %s"%stack_processed_by_ali2d_base__filename__without_master_dir) cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir)) if (myid == main_node): main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d") print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no if isac_generation_from_command_line < 0: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1): if i == isac_generation_from_command_line+1: backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%i + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i) # it includes both command line and json file my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1] if "restart" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line ) elif "candidate_class_averages" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line) # cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line) elif "reproducible_class_averages" in my_restart_section: cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line) cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line) else: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 0 isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0] isac_generation = isac_generation_from_command_line - 1 if (myid == main_node): if isac_generation == 0: cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation) write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation)) write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation)) # Stopping criterion should be inside the program. while True: isac_generation += 1 if isac_generation > options.n_generations: break data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation if(myid == main_node): accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_accounted.txt"%(isac_generation - 1))) number_of_accounted_images = len(accounted_images) # unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1))) # number_of_unaccounted_images = len(unaccounted_images) else: number_of_accounted_images = 0 number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if number_of_accounted_images == 0: os.chdir("..") break program_state_stack.restart_location_title = "restart" if program_state_stack(locals(), getframeinfo(currentframe())): if (myid == main_node): cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation) # reference the original stack list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1)) cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\ stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file)) mpi_barrier(MPI_COMM_WORLD) os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation) program_state_stack.restart_location_title = "candidate_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new) # program_state_stack.restart_location_title = "stopped_program1" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack.restart_location_title = "stop_after_candidates" program_state_stack(locals(), getframeinfo(currentframe())) if stop_after_candidates: mpi_finalize() sys.exit() exit_program = 0 if(myid == main_node): if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation): print "This generation (%d) no class averages were generated!"%isac_generation exit_program = 1 exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if exit_program: os.chdir("..") break program_state_stack.restart_location_title = "reproducible_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new) pass os.chdir("..") if (myid == main_node): cmdexecute("rm -f class_averages.hdf") cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation)], "class_averages.hdf") # program_state_stack.restart_location_title = "stopped_program2" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall") mpi_finalize()
def main(args): progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" + " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" + " --generation=generation --rand_seed=rand_seed>") parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--radius", type="int", help= "particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)" ) parser.add_option( "--target_radius", type="int", default=29, help= "target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)" ) parser.add_option( "--target_nx", type="int", default=76, help= "target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1' (default 76)" ) parser.add_option( "--img_per_grp", type="int", default=100, help= "number of images per class: in the ideal case (essentially maximum size of class) (default 100)" ) parser.add_option( "--CTF", action="store_true", default=False, help= "apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)" ) parser.add_option( "--ir", type="int", default=1, help= "inner ring: of the resampling to polar coordinates. units - pixels (default 1)" ) parser.add_option( "--rs", type="int", default=1, help= "ring step: of the resampling to polar coordinates. units - pixels (default 1)" ) parser.add_option( "--xr", type="int", default=1, help= "x range: of translational search. By default, set by the program. (default 1)" ) parser.add_option( "--yr", type="int", default=-1, help= "y range: of translational search. By default, same as xr. (default -1)" ) parser.add_option( "--ts", type="float", default=1.0, help= "search step: of translational search: units - pixels (default 1.0)") parser.add_option( "--maxit", type="int", default=30, help="number of iterations for reference-free alignment: (default 30)") #parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)") parser.add_option( "--center_method", type="int", default=-1, help= "method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)" ) parser.add_option( "--dst", type="float", default=90.0, help="discrete angle used in within group alignment: (default 90.0)") parser.add_option( "--FL", type="float", default=0.2, help= "lowest stopband: frequency used in the tangent filter (default 0.2)") parser.add_option( "--FH", type="float", default=0.3, help= "highest stopband: frequency used in the tangent filter (default 0.3)") parser.add_option("--FF", type="float", default=0.2, help="fall-off of the tangent filter: (default 0.2)") parser.add_option( "--init_iter", type="int", default=3, help= "SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)" ) parser.add_option( "--main_iter", type="int", default=3, help= "SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)" ) parser.add_option( "--iter_reali", type="int", default=1, help= "SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)" ) parser.add_option( "--match_first", type="int", default=1, help= "number of iterations to run 2-way matching in the first phase: (default 1)" ) parser.add_option( "--max_round", type="int", default=20, help= "maximum rounds: of generating candidate class averages in the first phase (default 20)" ) parser.add_option( "--match_second", type="int", default=5, help= "number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)" ) parser.add_option( "--stab_ali", type="int", default=5, help="number of alignments when checking stability: (default 5)") parser.add_option( "--thld_err", type="float", default=0.7, help= "threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)" ) parser.add_option( "--indep_run", type="int", default=4, help= "level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching. Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)" ) parser.add_option("--thld_grp", type="int", default=10, help="minimum size of reproducible class (default 10)") parser.add_option( "--n_generations", type="int", default=10, help= "maximum number of generations: program stops when reaching this total number of generations: (default 10)" ) #parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)") parser.add_option( "--rand_seed", type="int", help= "random seed set before calculations: useful for testing purposes. By default, total randomness (type int)" ) parser.add_option("--new", action="store_true", default=False, help="use new code: (default False)") parser.add_option("--debug", action="store_true", default=False, help="debug info printout: (default False)") # must be switched off in production parser.add_option( "--use_latest_master_directory", action="store_true", default=False, help= "use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)" ) parser.add_option( "--restart_section", type="string", default=' ', help= "restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')" ) parser.add_option( "--stop_after_candidates", action="store_true", default=False, help= "stop after candidates: stops after the 'candidate_class_averages' section. (default False)" ) ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help=SUPPRESS_HELP) parser.add_option( "--skip_prealignment", action="store_true", default=False, help= "skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)" ) required_option_list = ['radius'] (options, args) = parser.parse_args(args) if options.return_options: return parser if len(args) > 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" sys.exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True from isac import iter_isac from fundamentals import rot_shift2D, resample from utilities import pad, combine_params2 command_line_provided_stack_filename = args[0] main_node = 0 mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) nproc = mpi_comm_size(MPI_COMM_WORLD) mpi_barrier(MPI_COMM_WORLD) if (myid == main_node): print "****************************************************************" Util.version() print "****************************************************************" sys.stdout.flush() mpi_barrier(MPI_COMM_WORLD) # Making sure all required options appeared. for required_option in required_option_list: if not options.__dict__[required_option]: print "\n ==%s== mandatory option is missing.\n" % required_option print "Please run '" + progname + " -h' for detailed options" return 1 radi = options.radius target_radius = options.target_radius target_nx = options.target_nx center_method = options.center_method if (radi < 1): ERROR("Particle radius has to be provided!", "sxisac", 1, myid) use_latest_master_directory = options.use_latest_master_directory stop_after_candidates = options.stop_after_candidates # program_state_stack.restart_location_title_from_command_line = options.restart_section from utilities import qw program_state_stack.PROGRAM_STATE_VARIABLES = set( qw(""" isac_generation """)) # create or reuse master directory masterdir = "" stack_processed_by_ali2d_base__filename = "" stack_processed_by_ali2d_base__filename__without_master_dir = "" error_status = 0 if len(args) == 2: masterdir = args[1] elif len(args) == 1: if use_latest_master_directory: all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)] import re r = re.compile("^master.*$") all_dirs = filter(r.match, all_dirs) if len(all_dirs) > 0: # all_dirs = max(all_dirs, key=os.path.getctime) masterdir = max(all_dirs, key=os.path.getmtime) #Create folder for all results or check if there is one created already if (myid == main_node): if (masterdir == ""): timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime()) masterdir = "master" + timestring cmd = "{} {}".format("mkdir", masterdir) junk = cmdexecute(cmd) elif not os.path.exists(masterdir): # os.path.exists(masterdir) does not exist masterdir = args[1] cmd = "{} {}".format("mkdir", masterdir) junk = cmdexecute(cmd) if (args[0][:4] == "bdb:"): filename = args[0][4:] else: filename = args[0][:-4] filename = os.path.basename(filename) stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join( masterdir, filename) stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename if_error_then_all_processes_exit_program(error_status) # send masterdir to all processes masterdir = send_string_to_all(masterdir) if myid == 0: if options.restart_section != " ": if os.path.exists(os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)): stored_stack, stored_state = restore_program_stack_and_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) import re if "," in options.restart_section: parsed_restart_section_option = options.restart_section.split( ",") stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % parsed_restart_section_option[0], stored_state[-1]["location_in_program"]) generation_str_format = parsed_restart_section_option[1] if generation_str_format != "": isac_generation_from_command_line = int( generation_str_format) stored_state[-1][ "isac_generation"] = isac_generation_from_command_line else: isac_generation_from_command_line = 1 if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % options.restart_section, stored_state[-1]["location_in_program"]) if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] store_program_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE), stored_state, stored_stack) else: print "Please remove the restart_section option from the command line. The program must be started from the beginning." mpi_finalize() sys.exit() else: isac_generation_from_command_line = -1 program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) stack_processed_by_ali2d_base__filename = send_string_to_all( stack_processed_by_ali2d_base__filename) stack_processed_by_ali2d_base__filename__without_master_dir = \ send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir) # previous code 2016-05-05--20-14-12-153 # # PARAMETERS OF THE PROCEDURE # if( options.xr == -1 ): # # Default values # # target_nx = 76 # # target_radius = 29 # target_xr = 1 # else: # nx//2 # # Check below! # target_xr = options.xr # # target_nx = 76 + target_xr - 1 # subtract one, which is default # target_nx += target_xr - 1 # subtract one, which is default # # target_radius = 29 target_xr = options.xr target_nx += target_xr - 1 # subtract one, which is default if (options.yr == -1): yr = options.xr else: yr = options.yr mpi_barrier(MPI_COMM_WORLD) # Initialization of stacks if (myid == main_node): print "command_line_provided_stack_filename", command_line_provided_stack_filename number_of_images_in_stack = EMUtil.get_image_count( command_line_provided_stack_filename) else: number_of_images_in_stack = 0 number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node=main_node) nxrsteps = 4 init2dir = os.path.join(masterdir, "2dalignment") # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() if not os.path.exists( os.path.join(init2dir, "Finished_initial_2d_alignment.txt")): if (myid == 0): import subprocess from logger import Logger, BaseLogger_Files # Create output directory log2d = Logger(BaseLogger_Files()) log2d.prefix = os.path.join(init2dir) cmd = "mkdir -p " + log2d.prefix outcome = subprocess.call(cmd, shell=True) log2d.prefix += "/" # outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True) else: outcome = 0 log2d = None if (myid == main_node): a = get_im(command_line_provided_stack_filename) nnxo = a.get_xsize() else: nnxo = 0 nnxo = bcast_number_to_all(nnxo, source_node=main_node) image_start, image_end = MPI_start_end(number_of_images_in_stack, nproc, myid) if options.skip_prealignment: params2d = [[0.0, 0.0, 0.0, 0] for i in xrange(image_start, image_end)] else: original_images = EMData.read_images( command_line_provided_stack_filename, range(image_start, image_end)) # We assume the target radius will be 29, and xr = 1. shrink_ratio = float(target_radius) / float(radi) for im in xrange(len(original_images)): if (shrink_ratio != 1.0): original_images[im] = resample(original_images[im], shrink_ratio) nx = original_images[0].get_xsize() # nx = int(nx*shrink_ratio + 0.5) txrm = (nx - 2 * (target_radius + 1)) // 2 if (txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d" % (radi), "sxisac", 1, myid) if (txrm / nxrsteps > 0): tss = "" txr = "" while (txrm / nxrsteps > 0): tts = txrm / nxrsteps tss += " %d" % tts txr += " %d" % (tts * nxrsteps) txrm = txrm // 2 else: tss = "1" txr = "%d" % txrm # print "nx, txr, txrm, tss", nx, txr, txrm, tss # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() # section ali2d_base params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \ False, 90.0, center_method, 14, options.CTF, 1.0, False, \ "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False) del original_images for i in xrange(len(params2d)): alpha, sx, sy, mirror = combine_params2( 0, params2d[i][1], params2d[i][2], 0, -params2d[i][0], 0, 0, 0) sx /= shrink_ratio sy /= shrink_ratio params2d[i][0] = 0.0 params2d[i][1] = sx params2d[i][2] = sy params2d[i][3] = 0 #set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0]) mpi_barrier(MPI_COMM_WORLD) tmp = params2d[:] tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD) if (myid == main_node): if options.skip_prealignment: print "=========================================" print "Even though there is no alignment step, '%s' params are set to zero for later use." % os.path.join( init2dir, "initial2Dparams.txt") print "=========================================" write_text_row(tmp, os.path.join(init2dir, "initial2Dparams.txt")) del tmp mpi_barrier(MPI_COMM_WORLD) # We assume the target image size will be target_nx, radius will be 29, and xr = 1. # Note images can be also padded, in which case shrink_ratio > 1. shrink_ratio = float(target_radius) / float(radi) aligned_images = EMData.read_images( command_line_provided_stack_filename, range(image_start, image_end)) nx = aligned_images[0].get_xsize() nima = len(aligned_images) newx = int(nx * shrink_ratio + 0.5) while not os.path.exists(os.path.join(init2dir, "initial2Dparams.txt")): import time time.sleep(1) mpi_barrier(MPI_COMM_WORLD) params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt")) params = params[image_start:image_end] msk = model_circle(radi, nx, nx) for im in xrange(nima): st = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= st[0] if options.CTF: aligned_images[im] = filt_ctf( aligned_images[im], aligned_images[im].get_attr("ctf"), binary=True) if (shrink_ratio < 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) #aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(newx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio == 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(newx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) #aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio > 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(newx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) del msk gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid) number_of_images_in_stack = bcast_number_to_all( number_of_images_in_stack, source_node=main_node) if (myid == main_node): for i in range(number_of_images_in_stack): aligned_images[i].write_image( stack_processed_by_ali2d_base__filename, i) # It has to be explicitly closed from EMAN2db import db_open_dict DB = db_open_dict(stack_processed_by_ali2d_base__filename) DB.close() fp = open(os.path.join(masterdir, "README_shrink_ratio.txt"), "w") output_text = """ Since, for processing purposes, isac changes the image dimensions, adjustment of pixel size needs to be made in subsequent steps, (e.g. running sxviper.py). The shrink ratio for this particular isac run is -------- %.5f %.5f -------- To get the pixel size for the isac output the user needs to divide the original pixel size by the above value. This info is saved in the following file: README_shrink_ratio.txt """ % (shrink_ratio, radi) fp.write(output_text) fp.flush() fp.close() print output_text fp = open( os.path.join(init2dir, "Finished_initial_2d_alignment.txt"), "w") fp.flush() fp.close() else: if (myid == main_node): print "Skipping 2d alignment since it was already done!" mpi_barrier(MPI_COMM_WORLD) # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() os.chdir(masterdir) if program_state_stack(locals(), getframeinfo(currentframe())): # if 1: pass if (myid == main_node): junk = cmdexecute( "sxheader.py --consecutive --params=originalid %s" % stack_processed_by_ali2d_base__filename__without_master_dir) junk = cmdexecute( "e2bdb.py %s --makevstack=%s_000" % (stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir)) if (myid == main_node): main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d") print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no if isac_generation_from_command_line < 0: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = -1 if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: for i in xrange(isac_generation_from_command_line + 1, main_dir_no + 1): if i == isac_generation_from_command_line + 1: backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) junk = cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i + " 000_backup" + "%05d" % backup_dir_no) junk = cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % i) # it includes both command line and json file my_restart_section = stored_state[-1]["location_in_program"].split( "___")[-1] if "restart" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) junk = cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) junk = cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % isac_generation_from_command_line) elif "candidate_class_averages" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) junk = cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line) # junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line) elif "reproducible_class_averages" in my_restart_section: junk = cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*" % isac_generation_from_command_line) junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*" % isac_generation_from_command_line) else: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 0 isac_generation_from_command_line = mpi_bcast( isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0] isac_generation = isac_generation_from_command_line - 1 if (myid == main_node): if isac_generation == 0: junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) write_text_file( [1], os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_accounted.txt" % isac_generation)) write_text_file( range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_unaccounted.txt" % isac_generation)) # Stopping criterion should be inside the program. while True: isac_generation += 1 if isac_generation > options.n_generations: break data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d" % isac_generation program_state_stack.restart_location_title = "restart" if program_state_stack(locals(), getframeinfo(currentframe())): if (myid == main_node): junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) # reference the original stack list_file = os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1), "generation_%d_unaccounted.txt" % (isac_generation - 1)) junk = cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\ stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file)) mpi_barrier(MPI_COMM_WORLD) os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation) program_state_stack.restart_location_title = "candidate_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new) # program_state_stack.restart_location_title = "stopped_program1" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack.restart_location_title = "stop_after_candidates" program_state_stack(locals(), getframeinfo(currentframe())) if stop_after_candidates: mpi_finalize() sys.exit() exit_program = 0 if (myid == main_node): if not os.path.exists( "class_averages_candidate_generation_%d.hdf" % isac_generation): print "This generation (%d) no class average candidates were generated! Finishing." % isac_generation exit_program = 1 exit_program = int( mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if exit_program: os.chdir("..") break program_state_stack.restart_location_title = "reproducible_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new) pass os.chdir("..") if (myid == main_node): accounted_images = read_text_file( os.path.join(NAME_OF_MAIN_DIR + "%04d" % (isac_generation), "generation_%d_accounted.txt" % (isac_generation))) number_of_accounted_images = len(accounted_images) un_accounted_images = read_text_file( os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation), "generation_%d_unaccounted.txt" % (isac_generation))) number_of_un_accounted_images = len(un_accounted_images) else: number_of_accounted_images = 0 number_of_un_accounted_images = 0 number_of_accounted_images = int( mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) number_of_un_accounted_images = int( mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if number_of_accounted_images == 0: if (myid == main_node): print "This generation (%d) there are no accounted images! Finishing." % isac_generation break while (myid == main_node): def files_are_missing(isac_generation): for i in xrange(1, isac_generation + 1): if not os.path.exists( "generation_%04d/class_averages_generation_%d.hdf" % (i, i)): print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting." % ( i, i) return 1 return 0 if files_are_missing(isac_generation): break junk = cmdexecute("rm -f class_averages.hdf") cpy([ "generation_%04d/class_averages_generation_%d.hdf" % (i, i) for i in xrange(1, isac_generation + 1) ], "class_averages.hdf") break if number_of_un_accounted_images == 0: if (myid == main_node): print "This generation (%d) there are no un accounted images! Finishing." % isac_generation break program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall") mpi_barrier(MPI_COMM_WORLD) mpi_finalize()
def main(args): progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" + " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" + " --generation=generation --rand_seed=rand_seed>") parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--radius", type="int", default=-1, help="<Particle radius>, it has to be provided.") parser.add_option( "--img_per_grp", type="int", default=100, help= "<number of images per group> in the ideal case (essentially maximum size of class) (100)" ) parser.add_option("--CTF", action="store_true", default=False, help="<CTF flag>, if set the data will be phase-flipped") parser.add_option( "--ir", type="int", default=1, help="<inner ring> of the resampling to polar coordinates (1)") parser.add_option( "--rs", type="int", default=1, help="<ring step> of the resampling to polar coordinates (1)") parser.add_option( "--xr", type="int", default=-1, help= "<x range> of translational search (By default set by the program) (advanced)" ) parser.add_option( "--yr", type="int", default=-1, help="<y range> of translational search (same as xr) (advanced)") parser.add_option("--ts", type="float", default=1.0, help="<search step> of translational search (1.0)") parser.add_option( "--maxit", type="int", default=30, help="number of iterations for reference-free alignment (30)") #parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)") parser.add_option( "--center_method", type="int", default=7, help= "<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)" ) parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment ") parser.add_option( "--FL", type="float", default=0.2, help="<lowest stopband> frequency used in the tangent filter (0.2)") parser.add_option( "--FH", type="float", default=0.3, help="<highest stopband> frequency used in the tangent filter (0.3)") parser.add_option("--FF", type="float", default=0.2, help="<fall-off of the tangent> filter (0.2)") parser.add_option( "--init_iter", type="int", default=3, help= "<init_iter> number of iterations of ISAC program in initialization (3)" ) parser.add_option( "--main_iter", type="int", default=3, help="<main_iter> number of iterations of ISAC program in main part (3)" ) parser.add_option( "--iter_reali", type="int", default=1, help= "<iter_reali> number of iterations in ISAC before checking stability (1)" ) parser.add_option( "--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase (1)" ) parser.add_option( "--max_round", type="int", default=20, help= "maximum rounds of generating candidate averages in the first phase (20)" ) parser.add_option( "--match_second", type="int", default=5, help= "number of iterations to run 2-way (or 3-way) matching in the second phase (5)" ) parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability (5)") parser.add_option( "--thld_err", type="float", default=0.7, help="the threshold of pixel error when checking stability (0.7)") parser.add_option( "--indep_run", type="int", default=4, help= "number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)" ) parser.add_option("--thld_grp", type="int", default=10, help="minimum size of class (10)") parser.add_option( "--n_generations", type="int", default=100, help= "<n_generations> program stops when reaching this total number of generations (advanced)" ) #parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)") parser.add_option( "--rand_seed", type="int", default=None, help= "random seed set before calculations, useful for testing purposes (default None - total randomness)" ) parser.add_option("--new", action="store_true", default=False, help="use new code (default = False)") parser.add_option("--debug", action="store_true", default=False, help="debug info printout (default = False)") # must be switched off in production parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False) parser.add_option( "--restart_section", type="string", default="", help= "<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)" ) parser.add_option( "--stop_after_candidates", action="store_true", default=False, help= "<stop_after_candidates> stops after the 'candidate_class_averages' section" ) parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help=SUPPRESS_HELP) (options, args) = parser.parse_args(args) if options.return_options: return parser if len(args) > 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" sys.exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() from isac import iter_isac global_def.BATCH = True global_def.BATCH = True command_line_provided_stack_filename = args[0] global_def.BATCH = True main_node = 0 mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) nproc = mpi_comm_size(MPI_COMM_WORLD) radi = options.radius center_method = options.center_method if (radi < 1): ERROR("Particle radius has to be provided!", "sxisac", 1, myid) use_latest_master_directory = options.use_latest_master_directory stop_after_candidates = options.stop_after_candidates program_state_stack.restart_location_title_from_command_line = options.restart_section from utilities import qw program_state_stack.PROGRAM_STATE_VARIABLES = set( qw(""" isac_generation """)) # create or reuse master directory masterdir = "" stack_processed_by_ali2d_base__filename = "" stack_processed_by_ali2d_base__filename__without_master_dir = "" error_status = 0 if len(args) == 2: masterdir = args[1] elif len(args) == 1: if use_latest_master_directory: all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)] import re r = re.compile("^master.*$") all_dirs = filter(r.match, all_dirs) if len(all_dirs) > 0: # all_dirs = max(all_dirs, key=os.path.getctime) masterdir = max(all_dirs, key=os.path.getmtime) #Create folder for all results or check if there is one created already if (myid == main_node): if (masterdir == ""): timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime()) masterdir = "master" + timestring cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) elif not os.path.exists(masterdir): # os.path.exists(masterdir) does not exist masterdir = args[1] cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) if (args[0][:4] == "bdb:"): filename = args[0][4:] else: filename = args[0][:-4] filename = os.path.basename(filename) stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join( masterdir, filename) stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename if_error_all_processes_quit_program(error_status) # send masterdir to all processes masterdir = send_string_to_all(masterdir) if myid == 0: if options.restart_section != "": if os.path.exists(os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)): stored_stack, stored_state = restore_program_stack_and_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) import re if "," in options.restart_section: parsed_restart_section_option = options.restart_section.split( ",") stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % parsed_restart_section_option[0], stored_state[-1]["location_in_program"]) generation_str_format = parsed_restart_section_option[1] if generation_str_format != "": isac_generation_from_command_line = int( generation_str_format) stored_state[-1][ "isac_generation"] = isac_generation_from_command_line else: isac_generation_from_command_line = 1 if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % options.restart_section, stored_state[-1]["location_in_program"]) if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] store_program_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE), stored_state, stored_stack) else: print "Please remove the restart_section option from the command line. The program must be started from the beginning." mpi_finalize() sys.exit() else: isac_generation_from_command_line = -1 program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) stack_processed_by_ali2d_base__filename = send_string_to_all( stack_processed_by_ali2d_base__filename) stack_processed_by_ali2d_base__filename__without_master_dir = \ send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir) # PARAMETERS OF THE PROCEDURE if (options.xr == -1): # Default values target_nx = 76 target_radius = 29 target_xr = 1 else: # nx//2 # Check below! target_xr = options.xr target_nx = 76 + target_xr - 1 # subtract one, which is default target_radius = 29 mpi_barrier(MPI_COMM_WORLD) # Initialization of stacks if (myid == main_node): number_of_images_in_stack = EMUtil.get_image_count( command_line_provided_stack_filename) else: number_of_images_in_stack = 0 number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node=main_node) nxrsteps = 4 init2dir = os.path.join(masterdir, "2dalignment") if (myid == 0): import subprocess from logger import Logger, BaseLogger_Files # Create output directory log2d = Logger(BaseLogger_Files()) log2d.prefix = os.path.join(init2dir) cmd = "mkdir -p " + log2d.prefix outcome = subprocess.call(cmd, shell=True) log2d.prefix += "/" # outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True) else: outcome = 0 log2d = None if (myid == main_node): a = get_im(command_line_provided_stack_filename) nnxo = a.get_xsize() else: nnxo = 0 nnxo = bcast_number_to_all(nnxo, source_node=main_node) txrm = (nnxo - 2 * (radi + 1)) // 2 if (txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d" % (radi), "sxisac", 1, myid) if (txrm / nxrsteps > 0): tss = "" txr = "" while (txrm / nxrsteps > 0): tts = txrm / nxrsteps tss += " %d" % tts txr += " %d" % (tts * nxrsteps) txrm = txrm // 2 else: tss = "1" txr = "%d" % txrm # section ali2d_base # centering method is set to #7 params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \ False, 90.0, center_method, 14, options.CTF, 1.0, False, \ "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False) if (myid == main_node): write_text_row(params2d, os.path.join(init2dir, "initial2Dparams.txt")) del params2d mpi_barrier(MPI_COMM_WORLD) # We assume the target image size will be target_nx, radius will be 29, and xr = 1. # Note images can be also padded, in which case shrink_ratio > 1. shrink_ratio = float(target_radius) / float(radi) nx = aligned_images[0].get_xsize() nima = len(aligned_images) newx = int(nx * shrink_ratio + 0.5) from fundamentals import rot_shift2D, resample from utilities import pad, combine_params2 if (shrink_ratio < 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(nx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio == 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(nx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio > 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(target_radius, nx, nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) del msk gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid) number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node=main_node) if (myid == main_node): for i in range(number_of_images_in_stack): aligned_images[i].write_image( stack_processed_by_ali2d_base__filename, i) # It has to be explicitly closed from EMAN2db import db_open_dict DB = db_open_dict(stack_processed_by_ali2d_base__filename) DB.close() mpi_barrier(MPI_COMM_WORLD) global_def.BATCH = True os.chdir(masterdir) if program_state_stack(locals(), getframeinfo(currentframe())): # if 1: pass if (myid == main_node): cmdexecute( "sxheader.py --consecutive --params=originalid %s" % stack_processed_by_ali2d_base__filename__without_master_dir) cmdexecute( "e2bdb.py %s --makevstack=%s_000" % (stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir)) if (myid == main_node): main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d") print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no if isac_generation_from_command_line < 0: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = -1 if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: for i in xrange(isac_generation_from_command_line + 1, main_dir_no + 1): if i == isac_generation_from_command_line + 1: backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i + " 000_backup" + "%05d" % backup_dir_no) cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % i) # it includes both command line and json file my_restart_section = stored_state[-1]["location_in_program"].split( "___")[-1] if "restart" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % isac_generation_from_command_line) elif "candidate_class_averages" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line) # cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line) elif "reproducible_class_averages" in my_restart_section: cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*" % isac_generation_from_command_line) cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*" % isac_generation_from_command_line) else: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 0 isac_generation_from_command_line = mpi_bcast( isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0] isac_generation = isac_generation_from_command_line - 1 if (myid == main_node): if isac_generation == 0: cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) write_text_file( [1], os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_accounted.txt" % isac_generation)) write_text_file( range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_unaccounted.txt" % isac_generation)) # Stopping criterion should be inside the program. while True: isac_generation += 1 if isac_generation > options.n_generations: break data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d" % isac_generation if (myid == main_node): accounted_images = read_text_file( os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1), "generation_%d_accounted.txt" % (isac_generation - 1))) number_of_accounted_images = len(accounted_images) # unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1))) # number_of_unaccounted_images = len(unaccounted_images) else: number_of_accounted_images = 0 number_of_accounted_images = int( mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if number_of_accounted_images == 0: os.chdir("..") break program_state_stack.restart_location_title = "restart" if program_state_stack(locals(), getframeinfo(currentframe())): if (myid == main_node): cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) # reference the original stack list_file = os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1), "generation_%d_unaccounted.txt" % (isac_generation - 1)) cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\ stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file)) mpi_barrier(MPI_COMM_WORLD) os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation) program_state_stack.restart_location_title = "candidate_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new) # program_state_stack.restart_location_title = "stopped_program1" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack.restart_location_title = "stop_after_candidates" program_state_stack(locals(), getframeinfo(currentframe())) if stop_after_candidates: mpi_finalize() sys.exit() exit_program = 0 if (myid == main_node): if not os.path.exists( "class_averages_candidate_generation_%d.hdf" % isac_generation): print "This generation (%d) no class averages were generated!" % isac_generation exit_program = 1 exit_program = int( mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if exit_program: os.chdir("..") break program_state_stack.restart_location_title = "reproducible_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new) pass os.chdir("..") if (myid == main_node): cmdexecute("rm -f class_averages.hdf") cpy([ "generation_%04d/class_averages_generation_%d.hdf" % (i, i) for i in xrange(1, isac_generation) ], "class_averages.hdf") # program_state_stack.restart_location_title = "stopped_program2" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall") mpi_finalize()
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Forms chains of 2D images based on their similarities. Functionality: 4. Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent). Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers. The ways to use the program: 4.1 Use option initial to specify which image will be used as an initial seed to form the chain. sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25 4.2 If options initial is omitted, the program will determine which image best serves as initial seed to form the chain sxprocess.py input_stack.hdf output_stack.hdf --radius=25 4.3 Use option circular to form a circular chain. sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25 4.4 New circular code based on pairwise alignments sxprocess.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt 4.5 Circular ordering based on pairwise alignments sxprocess.py vols.hdf chain.hdf mask.hdf --dd --radius=25 """ parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False) parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False) parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False) parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)") parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") # import params for 2D alignment parser.add_option("--ou", type="int", default=-1, help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)") parser.add_option("--xr", type="int", default=0, help="range for translation search in x direction, search is +/xr (0)") parser.add_option("--yr", type="int", default=0, help="range for translation search in y direction, search is +/yr (0)") #parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)") parser.add_option("--pairwiseccc", type="string", default= None, help="Input/output pairwise ccc file") (options, args) = parser.parse_args() global_def.BATCH = True if options.dd: nargs = len(args) if nargs != 3: print "must provide name of input and two output files!" return stack = args[0] new_stack = args[1] from utilities import model_circle from statistics import ccc from statistics import mono lend = EMUtil.get_image_count(stack) lccc = [None]*(lend*(lend-1)/2) for i in xrange(lend-1): v1 = get_im( stack, i ) if( i == 0 and nargs == 2): nx = v1.get_xsize() ny = v1.get_ysize() nz = v1.get_ysize() if options.ou < 1 : radius = nx//2-2 else: radius = options.ou mask = model_circle(radius, nx, ny, nz) else: mask = get_im(args[2]) for j in xrange(i+1, lend): lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0] order = tsp(lccc) if(len(order) != lend): print " problem with data length" from sys import exit exit() print "Total sum of cccs :",TotalDistance(order, lccc) print "ordering :",order for i in xrange(lend): get_im(stack, order[i]).write_image( new_stack, i ) elif options.align: nargs = len(args) if nargs != 3: print "must provide name of input and two output files!" return from utilities import get_params2D, model_circle from fundamentals import rot_shift2D from statistics import ccc from time import time from alignment import align2d, align2d_scf from multi_shc import mult_transform stack = args[0] new_stack = args[1] d = EMData.read_images(stack) """ # will align anyway try: ttt = d[0].get_attr('xform.params2d') for i in xrange(len(d)): alpha, sx, sy, mirror, scale = get_params2D(d[i]) d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror) except: pass """ nx = d[0].get_xsize() ny = d[0].get_ysize() if options.ou < 1 : radius = nx//2-2 else: radius = options.ou mask = model_circle(radius, nx, ny) if(options.xr < 0): xrng = 0 else: xrng = options.xr if(options.yr < 0): yrng = xrng else: yrng = options.yr initial = max(options.initial, 0) from statistics import mono lend = len(d) lccc = [None]*(lend*(lend-1)/2) from utilities import read_text_row if options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) : st = time() for i in xrange(lend-1): for j in xrange(i+1, lend): # j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger) #alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F") alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius) lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir] #print " %4d %10.1f"%(i,time()-st) if(not os.path.exists(options.pairwiseccc)): from utilities import write_text_row write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc) elif(os.path.exists(options.pairwiseccc)): lccc = read_text_row(options.pairwiseccc) initial = int(lccc[0][0] + 0.1) del lccc[0] for i in xrange(len(lccc)): T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)}) lccc[i] = [lccc[i][0],T] tdummy = Transform({"type":"2D"}) maxsum = -1.023 for m in xrange(0,lend):#initial, initial+1): indc = range( lend ) lsnake = [[m, tdummy, 0.0]] del indc[m] lsum = 0.0 while len(indc) > 1: maxcit = -111. for i in xrange(len(indc)): cuc = lccc[mono(indc[i], lsnake[-1][0])][0] if cuc > maxcit: maxcit = cuc qi = indc[i] # Here we need transformation from the current to the previous, # meaning indc[i] -> lsnake[-1][0] T = lccc[mono(indc[i], lsnake[-1][0])][1] # If direction is from larger to smaller index, the transformation has to be inverted if( indc[i] > lsnake[-1][0] ): T = T.inverse() lsnake.append([qi,T, maxcit]) lsum += maxcit del indc[indc.index(qi)] T = lccc[mono(indc[-1], lsnake[-1][0])][1] if( indc[-1] > lsnake[-1][0]): T = T.inverse() lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]]) print " initial image and lsum ",m,lsum #print lsnake if(lsum > maxsum): maxsum = lsum init = m snake = [lsnake[i] for i in xrange(lend)] print " Initial image selected : ",init,maxsum," ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc) #for q in snake: print q from copy import deepcopy trans=deepcopy([snake[i][1] for i in xrange(len(snake))]) print [snake[i][0] for i in xrange(len(snake))] """ for m in xrange(lend): prms = trans[m].get_params("2D") print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2]) """ for k in xrange(lend-2,0,-1): T = snake[k][1] for i in xrange(k+1, lend): trans[i] = T*trans[i] # To add - apply all transformations and do the overall centering. for m in xrange(lend): prms = trans[m].get_params("2D") #print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2]) #rot_shift2D(d[snake[m][0]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image(new_stack, m) rot_shift2D(d[snake[m][0]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(new_stack, m) order = tsp(lccc) if(len(order) != lend): print " problem with data length" from sys import exit exit() print TotalDistance(order, lccc) print order ibeg = order.index(init) order = [order[(i+ibeg)%lend] for i in xrange(lend)] print TotalDistance(order, lccc) print order snake = [tdummy] for i in xrange(1,lend): # Here we need transformation from the current to the previous, # meaning order[i] -> order[i-1]] T = lccc[mono(order[i], order[i-1])][1] # If direction is from larger to smaller index, the transformation has to be inverted if( order[i] > order[i-1] ): T = T.inverse() snake.append(T) assert(len(snake) == lend) from copy import deepcopy trans = deepcopy(snake) for k in xrange(lend-2,0,-1): T = snake[k] for i in xrange(k+1, lend): trans[i] = T*trans[i] # Try to smooth the angles - complicated, I am afraid one would have to use angles forward and backwards # and find their average?? # In addition, one would have to recenter them """ trms = [] for m in xrange(lend): prms = trans[m].get_params("2D") trms.append([prms["alpha"], prms["mirror"]]) for i in xrange(3): for m in xrange(lend): mb = (m-1)%lend me = (m+1)%lend # angles order mb,m,me # calculate predicted angles mb->m """ for m in xrange(lend): prms = trans[m].get_params("2D") #rot_shift2D(d[order[m]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image("metro.hdf", m) rot_shift2D(d[order[m]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(args[2], m) """ # This was an effort to get number of loops, inconclusive, to say the least from numpy import outer, zeros, float32, sqrt lend = len(d) cor = zeros(lend,float32) cor = outer(cor, cor) for i in xrange(lend): cor[i][i] = 1.0 for i in xrange(lend-1): for j in xrange(i+1, lend): cor[i,j] = lccc[mono(i,j)][0] cor[j,i] = cor[i,j] lmbd, eigvec = pca(cor) from utilities import write_text_file nvec=20 print [lmbd[j] for j in xrange(nvec)] print " G" mm = [-1]*lend for i in xrange(lend): # row mi = -1.0e23 for j in xrange(nvec): qt = eigvec[j][i] if(abs(qt)>mi): mi = abs(qt) mm[i] = j for j in xrange(nvec): qt = eigvec[j][i] print round(qt,3), # eigenvector print mm[i] print for j in xrange(nvec): qt = [] for i in xrange(lend): if(mm[i] == j): qt.append(i) if(len(qt)>0): write_text_file(qt,"loop%02d.txt"%j) """ """ print [lmbd[j] for j in xrange(nvec)] print " B" mm = [-1]*lend for i in xrange(lend): # row mi = -1.0e23 for j in xrange(nvec): qt = eigvec[j][i]/sqrt(lmbd[j]) if(abs(qt)>mi): mi = abs(qt) mm[i] = j for j in xrange(nvec): qt = eigvec[j][i]/sqrt(lmbd[j]) print round(qt,3), # eigenvector print mm[i] print """ """ lend=3 cor = zeros(lend,float32) cor = outer(cor, cor) cor[0][0] =136.77 cor[0][1] = 79.15 cor[0][2] = 37.13 cor[1][0] = 79.15 cor[2][0] = 37.13 cor[1][1] = 50.04 cor[1][2] = 21.65 cor[2][1] = 21.65 cor[2][2] = 13.26 lmbd, eigvec = pca(cor) print lmbd print eigvec for i in xrange(lend): # row for j in xrange(lend): print eigvec[j][i], # eigenvector print print " B" for i in xrange(lend): # row for j in xrange(lend): print eigvec[j][i]/sqrt(lmbd[j]), # eigenvector print print " G" for i in xrange(lend): # row for j in xrange(lend): print eigvec[j][i]*sqrt(lmbd[j]), # eigenvector print """ else: nargs = len(args) if nargs != 2: print "must provide name of input and output file!" return from utilities import get_params2D, model_circle from fundamentals import rot_shift2D from statistics import ccc from time import time from alignment import align2d from multi_shc import mult_transform stack = args[0] new_stack = args[1] d = EMData.read_images(stack) try: ttt = d[0].get_attr('xform.params2d') for i in xrange(len(d)): alpha, sx, sy, mirror, scale = get_params2D(d[i]) d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror) except: pass nx = d[0].get_xsize() ny = d[0].get_ysize() if options.radius < 1 : radius = nx//2-2 else: radius = options.radius mask = model_circle(radius, nx, ny) init = options.initial if init > -1 : print " initial image: %d" % init temp = d[init].copy() temp.write_image(new_stack, 0) del d[init] k = 1 lsum = 0.0 while len(d) > 1: maxcit = -111. for i in xrange(len(d)): cuc = ccc(d[i], temp, mask) if cuc > maxcit: maxcit = cuc qi = i # print k, maxcit lsum += maxcit temp = d[qi].copy() del d[qi] temp.write_image(new_stack, k) k += 1 print lsum d[0].write_image(new_stack, k) else: if options.circular : print "Using options.circular" # figure the "best circular" starting image maxsum = -1.023 for m in xrange(len(d)): indc = range(len(d) ) lsnake = [-1]*(len(d)+1) lsnake[0] = m lsnake[-1] = m del indc[m] temp = d[m].copy() lsum = 0.0 direction = +1 k = 1 while len(indc) > 1: maxcit = -111. for i in xrange(len(indc)): cuc = ccc(d[indc[i]], temp, mask) if cuc > maxcit: maxcit = cuc qi = indc[i] lsnake[k] = qi lsum += maxcit del indc[indc.index(qi)] direction = -direction for i in xrange( 1,len(d) ): if( direction > 0 ): if(lsnake[i] == -1): temp = d[lsnake[i-1]].copy() #print " forw ",lsnake[i-1] k = i break else: if(lsnake[len(d) - i] == -1): temp = d[lsnake[len(d) - i +1]].copy() #print " back ",lsnake[len(d) - i +1] k = len(d) - i break lsnake[lsnake.index(-1)] = indc[-1] #print " initial image and lsum ",m,lsum #print lsnake if(lsum > maxsum): maxsum = lsum init = m snake = [lsnake[i] for i in xrange(len(d))] print " Initial image selected : ",init,maxsum print lsnake for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m) else: # figure the "best" starting image maxsum = -1.023 for m in xrange(len(d)): indc = range(len(d) ) lsnake = [m] del indc[m] temp = d[m].copy() lsum = 0.0 while len(indc) > 1: maxcit = -111. for i in xrange(len(indc)): cuc = ccc(d[indc[i]], temp, mask) if cuc > maxcit: maxcit = cuc qi = indc[i] lsnake.append(qi) lsum += maxcit temp = d[qi].copy() del indc[indc.index(qi)] lsnake.append(indc[-1]) #print " initial image and lsum ",m,lsum #print lsnake if(lsum > maxsum): maxsum = lsum init = m snake = [lsnake[i] for i in xrange(len(d))] print " Initial image selected : ",init,maxsum print lsnake for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m)
def main(): from utilities import get_input_from_string progname = os.path.basename(sys.argv[0]) usage = progname + " stack output_average --radius=particle_radius --xr=xr --yr=yr --ts=ts --thld_err=thld_err --num_ali=num_ali --fl=fl --aa=aa --CTF --verbose --stables" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--radius", type="int", default=-1, help=" particle radius for alignment") parser.add_option("--xr", type="string" , default="2 1", help="range for translation search in x direction, search is +/xr (default 2,1)") parser.add_option("--yr", type="string" , default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)") parser.add_option("--ts", type="string" , default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default: 1,0.5)") parser.add_option("--thld_err", type="float", default=0.75, help="threshld of pixel error (default = 0.75)") parser.add_option("--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)") parser.add_option("--maxit", type="int", default=30, help="number of iterations for each xr (default = 30)") parser.add_option("--fl", type="float" , default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter (default = 0.3)") parser.add_option("--aa", type="float" , default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter (default = 0.2)") parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction during the alignment ") parser.add_option("--verbose", action="store_true", default=False, help="print individual pixel error (default = False)") parser.add_option("--stables", action="store_true", default=False, help="output the stable particles number in file (default = False)") parser.add_option("--method", type="string" , default=" ", help="SHC (standard method is default when flag is ommitted)") (options, args) = parser.parse_args() if len(args) != 1 and len(args) != 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" else: if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() from applications import within_group_refinement, ali2d_ras from pixel_error import multi_align_stability from utilities import write_text_file, write_text_row global_def.BATCH = True xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else : yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) class_data = EMData.read_images(args[0]) nx = class_data[0].get_xsize() ou = options.radius num_ali = options.num_ali if ou == -1: ou = nx/2-2 from utilities import model_circle, get_params2D, set_params2D mask = model_circle(ou, nx, nx) if options.CTF : from filter import filt_ctf for im in xrange(len(class_data)): # Flip phases class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: im.set_attr("previousmax", -1.0e10) try: t = im.get_attr("xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) all_ali_params = [] for ii in xrange(num_ali): ali_params = [] if options.verbose: ALPHA = [] SX = [] SY = [] MIRROR = [] if( xrng[0] == 0.0 and yrng[0] == 0.0 ): avet = ali2d_ras(class_data, randomize = True, ir = 1, ou = ou, rs = 1, step = 1.0, dst = 90.0, \ maxit = options.maxit, check_mirror = True, FH=options.fl, FF=options.aa) else: avet = within_group_refinement(class_data, mask, True, 1, ou, 1, xrng, yrng, step, 90.0, \ maxit = options.maxit, FH=options.fl, FF=options.aa, method = options.method) from utilities import info #print " avet ",info(avet) for im in class_data: alpha, sx, sy, mirror, scale = get_params2D(im) ali_params.extend([alpha, sx, sy, mirror]) if options.verbose: ALPHA.append(alpha) SX.append(sx) SY.append(sy) MIRROR.append(mirror) all_ali_params.append(ali_params) if options.verbose: write_text_file([ALPHA, SX, SY, MIRROR], "ali_params_run_%d"%ii) """ avet = class_data[0] from utilities import read_text_file all_ali_params = [] for ii in xrange(5): temp = read_text_file( "ali_params_run_%d"%ii,-1) uuu = [] for k in xrange(len(temp[0])): uuu.extend([temp[0][k],temp[1][k],temp[2][k],temp[3][k]]) all_ali_params.append(uuu) """ stable_set, mir_stab_rate, pix_err = multi_align_stability(all_ali_params, 0.0, 10000.0, options.thld_err, options.verbose, 2*ou+1) print "%4s %20s %20s %20s %30s %6.2f"%("", "Size of set", "Size of stable set", "Mirror stab rate", "Pixel error prior to pruning the set above threshold of",options.thld_err) print "Average stat: %10d %20d %20.2f %15.2f"%( len(class_data), len(stable_set), mir_stab_rate, pix_err) if( len(stable_set) > 0): if options.stables: stab_mem = [[0,0.0,0] for j in xrange(len(stable_set))] for j in xrange(len(stable_set)): stab_mem[j] = [int(stable_set[j][1]), stable_set[j][0], j] write_text_row(stab_mem, "stable_particles.txt") stable_set_id = [] particle_pixerr = [] for s in stable_set: stable_set_id.append(s[1]) particle_pixerr.append(s[0]) from fundamentals import rot_shift2D avet.to_zero() l = -1 print "average parameters: angle, x-shift, y-shift, mirror" for j in stable_set_id: l += 1 print " %4d %4d %12.2f %12.2f %12.2f %1d"%(l,j, stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], int(stable_set[l][2][3])) avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] ) avet /= (l+1) avet.set_attr('members', stable_set_id) avet.set_attr('pix_err', pix_err) avet.set_attr('pixerr', particle_pixerr) avet.write_image(args[1]) global_def.BATCH = False
def compare(compare_ref_free, outfile_repro, ref_free_output, yrng, xrng, rstep, nx, apix, ref_free_cutoff, nproc, myid, main_node): from alignment import Numrinit, ringwe, Applyws from random import seed, randint from utilities import get_params2D, set_params2D, model_circle, inverse_transform2, combine_params2 from fundamentals import rot_shift2D from mpi import MPI_COMM_WORLD, mpi_barrier, mpi_bcast, MPI_INT from statistics import fsc_mask from filter import fit_tanh from numpy import array fout = "%s.hdf" % ref_free_output frc_out = "%s_frc" % ref_free_output res_out = "%s_res" % ref_free_output nima = EMUtil.get_image_count(compare_ref_free) image_start, image_end = MPI_start_end(nima, nproc, myid) ima = EMData() ima.read_image(compare_ref_free, image_start) last_ring = nx / 2 - 2 first_ring = 1 mask = model_circle(last_ring, nx, nx) refi = [] numref = EMUtil.get_image_count(outfile_repro) cnx = nx / 2 + 1 cny = cnx mode = "F" numr = Numrinit(first_ring, last_ring, rstep, mode) wr = ringwe(numr, mode) ima.to_zero() for j in xrange(numref): temp = EMData() temp.read_image(outfile_repro, j) # even, odd, numer of even, number of images. After frc, totav refi.append(temp) # for each node read its share of data data = EMData.read_images(compare_ref_free, range(image_start, image_end)) for im in xrange(image_start, image_end): data[im - image_start].set_attr('ID', im) set_params2D(data[im - image_start], [0, 0, 0, 0, 1]) ringref = [] for j in xrange(numref): refi[j].process_inplace("normalize.mask", { "mask": mask, "no_sigma": 1 }) # normalize reference images to N(0,1) cimage = Util.Polar2Dm(refi[j], cnx, cny, numr, mode) Util.Frngs(cimage, numr) Applyws(cimage, numr, wr) ringref.append(cimage) if myid == main_node: seed(1000) data_shift = [] frc = [] res = [] for im in xrange(image_start, image_end): alpha, sx, sy, mirror, scale = get_params2D(data[im - image_start]) alphai, sxi, syi, scalei = inverse_transform2(alpha, sx, sy, 1.0) # normalize data[im - image_start].process_inplace("normalize.mask", { "mask": mask, "no_sigma": 1 }) # subtract average under the mask # align current image to the reference [angt, sxst, syst, mirrort, xiref, peakt] = Util.multiref_polar_ali_2d(data[im - image_start], ringref, xrng, yrng, 1, mode, numr, cnx + sxi, cny + syi) iref = int(xiref) [alphan, sxn, syn, mn] = combine_params2(0.0, -sxi, -syi, 0, angt, sxst, syst, (int)(mirrort)) set_params2D(data[im - image_start], [alphan, sxn, syn, int(mn), scale]) temp = rot_shift2D(data[im - image_start], alphan, sxn, syn, mn) temp.set_attr('assign', iref) tfrc = fsc_mask(temp, refi[iref], mask=mask) temp.set_attr('frc', tfrc[1]) res = fit_tanh(tfrc) temp.set_attr('res', res) data_shift.append(temp) for node in xrange(nproc): if myid == node: for image in data_shift: image.write_image(fout, -1) refindex = image.get_attr('assign') refi[refindex].write_image(fout, -1) mpi_barrier(MPI_COMM_WORLD) rejects = [] if myid == main_node: a = EMData() index = 0 frc = [] res = [] temp = [] classes = [] for im in xrange(nima): a.read_image(fout, index) frc.append(a.get_attr("frc")) if ref_free_cutoff != -1: classes.append(a.get_attr("class_ptcl_idxs")) tmp = a.get_attr("res") temp.append(tmp[0]) res.append("%12f" % (apix / tmp[0])) res.append("\n") index = index + 2 res_num = array(temp) mean_score = res_num.mean(axis=0) std_score = res_num.std(axis=0) std = std_score / 2 if ref_free_cutoff != -1: cutoff = mean_score - std * ref_free_cutoff reject = res_num < cutoff index = 0 for i in reject: if i: rejects.extend(classes[index]) index = index + 1 rejects.sort() length = mpi_bcast(len(rejects), 1, MPI_INT, main_node, MPI_COMM_WORLD) rejects = mpi_bcast(rejects, length, MPI_INT, main_node, MPI_COMM_WORLD) del a fout_frc = open(frc_out, 'w') fout_res = open(res_out, 'w') fout_res.write("".join(res)) temp = zip(*frc) datstrings = [] for i in temp: for j in i: datstrings.append(" %12f" % (j)) datstrings.append("\n") fout_frc.write("".join(datstrings)) fout_frc.close() del refi del ringref return rejects
def helicalshiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1): from applications import MPI_start_end from utilities import model_circle, model_blank, get_image, peak_search, get_im, pad from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all from pap_statistics import varf2d_MPI from fundamentals import fft, ccf, rot_shift3D, rot_shift2D, fshift from utilities import get_params2D, set_params2D, chunks_distribution from utilities import print_msg, print_begin_msg, print_end_msg import os import sys from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv from mpi import MPI_SUM, MPI_FLOAT, MPI_INT from time import time from pixel_error import ordersegments from math import sqrt, atan2, tan, pi nproc = mpi_comm_size(MPI_COMM_WORLD) myid = mpi_comm_rank(MPI_COMM_WORLD) main_node = 0 ftp = file_type(stack) if myid == main_node: print_begin_msg("helical-shiftali_MPI") max_iter = int(maxit) if (myid == main_node): infils = EMUtil.get_all_attributes(stack, "filament") ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord') filaments = ordersegments(infils, ptlcoords) total_nfils = len(filaments) inidl = [0] * total_nfils for i in range(total_nfils): inidl[i] = len(filaments[i]) linidl = sum(inidl) nima = linidl tfilaments = [] for i in range(total_nfils): tfilaments += filaments[i] del filaments else: total_nfils = 0 linidl = 0 total_nfils = bcast_number_to_all(total_nfils, source_node=main_node) if myid != main_node: inidl = [-1] * total_nfils inidl = bcast_list_to_all(inidl, myid, source_node=main_node) linidl = bcast_number_to_all(linidl, source_node=main_node) if myid != main_node: tfilaments = [-1] * linidl tfilaments = bcast_list_to_all(tfilaments, myid, source_node=main_node) filaments = [] iendi = 0 for i in range(total_nfils): isti = iendi iendi = isti + inidl[i] filaments.append(tfilaments[isti:iendi]) del tfilaments, inidl if myid == main_node: print_msg("total number of filaments: %d" % total_nfils) if total_nfils < nproc: ERROR( 'number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used' % (nproc, total_nfils), "ehelix_MPI", 1, myid) # balanced load temp = chunks_distribution([[len(filaments[i]), i] for i in range(len(filaments))], nproc)[myid:myid + 1][0] filaments = [filaments[temp[i][1]] for i in range(len(temp))] nfils = len(filaments) #filaments = [[0,1]] #print "filaments",filaments list_of_particles = [] indcs = [] k = 0 for i in range(nfils): list_of_particles += filaments[i] k1 = k + len(filaments[i]) indcs.append([k, k1]) k = k1 data = EMData.read_images(stack, list_of_particles) ldata = len(data) print("ldata=", ldata) nx = data[0].get_xsize() ny = data[0].get_ysize() if maskfile == None: mrad = min(nx, ny) // 2 - 2 mask = pad(model_blank(2 * mrad + 1, ny, 1, 1.0), nx, ny, 1, 0.0) else: mask = get_im(maskfile) # apply initial xform.align2d parameters stored in header init_params = [] for im in range(ldata): t = data[im].get_attr('xform.align2d') init_params.append(t) p = t.get_params("2d") data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'], p['mirror'], p['scale']) if CTF: from filter import filt_ctf from morphology import ctf_img ctf_abs_sum = EMData(nx, ny, 1, False) ctf_2_sum = EMData(nx, ny, 1, False) else: ctf_2_sum = None ctf_abs_sum = None from utilities import info for im in range(ldata): data[im].set_attr('ID', list_of_particles[im]) st = Util.infomask(data[im], mask, False) data[im] -= st[0] if CTF: ctf_params = data[im].get_attr("ctf") qctf = data[im].get_attr("ctf_applied") if qctf == 0: data[im] = filt_ctf(fft(data[im]), ctf_params) data[im].set_attr('ctf_applied', 1) elif qctf != 1: ERROR('Incorrectly set qctf flag', "helicalshiftali_MPI", 1, myid) ctfimg = ctf_img(nx, ctf_params, ny=ny) Util.add_img2(ctf_2_sum, ctfimg) Util.add_img_abs(ctf_abs_sum, ctfimg) else: data[im] = fft(data[im]) del list_of_particles if CTF: reduce_EMData_to_root(ctf_2_sum, myid, main_node) reduce_EMData_to_root(ctf_abs_sum, myid, main_node) if CTF: if myid != main_node: del ctf_2_sum del ctf_abs_sum else: temp = EMData(nx, ny, 1, False) tsnr = 1. / snr for i in range(0, nx + 2, 2): for j in range(ny): temp.set_value_at(i, j, tsnr) temp.set_value_at(i + 1, j, 0.0) #info(ctf_2_sum) Util.add_img(ctf_2_sum, temp) #info(ctf_2_sum) del temp total_iter = 0 shift_x = [0.0] * ldata for Iter in range(max_iter): if myid == main_node: start_time = time() print_msg("Iteration #%4d\n" % (total_iter)) total_iter += 1 avg = EMData(nx, ny, 1, False) for im in range(ldata): Util.add_img(avg, fshift(data[im], shift_x[im])) reduce_EMData_to_root(avg, myid, main_node) if myid == main_node: if CTF: tavg = Util.divn_filter(avg, ctf_2_sum) else: tavg = Util.mult_scalar(avg, 1.0 / float(nima)) else: tavg = model_blank(nx, ny) if Fourvar: bcast_EMData_to_all(tavg, myid, main_node) vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF) if myid == main_node: if Fourvar: tavg = fft(Util.divn_img(fft(tavg), vav)) vav_r = Util.pack_complex_to_real(vav) # normalize and mask tavg in real space tavg = fft(tavg) stat = Util.infomask(tavg, mask, False) tavg -= stat[0] Util.mul_img(tavg, mask) tavg.write_image("tavg.hdf", Iter) # For testing purposes: shift tavg to some random place and see if the centering is still correct #tavg = rot_shift3D(tavg,sx=3,sy=-4) if Fourvar: del vav bcast_EMData_to_all(tavg, myid, main_node) tavg = fft(tavg) sx_sum = 0.0 nxc = nx // 2 for ifil in range(nfils): """ # Calculate filament average avg = EMData(nx, ny, 1, False) filnima = 0 for im in xrange(indcs[ifil][0], indcs[ifil][1]): Util.add_img(avg, data[im]) filnima += 1 tavg = Util.mult_scalar(avg, 1.0/float(filnima)) """ # Calculate 1D ccf between each segment and filament average nsegms = indcs[ifil][1] - indcs[ifil][0] ctx = [None] * nsegms pcoords = [None] * nsegms for im in range(indcs[ifil][0], indcs[ifil][1]): ctx[im - indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx, 1) pcoords[im - indcs[ifil][0]] = data[im].get_attr( 'ptcl_source_coord') #ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0]) #print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True) # search for best x-shift cents = nsegms // 2 dst = sqrt( max((pcoords[cents][0] - pcoords[0][0])**2 + (pcoords[cents][1] - pcoords[0][1])**2, (pcoords[cents][0] - pcoords[-1][0])**2 + (pcoords[cents][1] - pcoords[-1][1])**2)) maxincline = atan2(ny // 2 - 2 - float(search_rng), dst) kang = int(dst * tan(maxincline) + 0.5) #print " settings ",nsegms,cents,dst,search_rng,maxincline,kang # ## C code for alignment. @ming results = [0.0] * 3 results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline, kang, search_rng, nxc) sib = int(results[0]) bang = results[1] qm = results[2] #print qm, sib, bang # qm = -1.e23 # # for six in xrange(-search_rng, search_rng+1,1): # q0 = ctx[cents].get_value_at(six+nxc) # for incline in xrange(kang+1): # qt = q0 # qu = q0 # if(kang>0): tang = tan(maxincline/kang*incline) # else: tang = 0.0 # for kim in xrange(cents+1,nsegms): # dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) # xl = dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # #print " A ", ifil,six,incline,kim,xl,ixl,dxl # qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # xl = -dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # for kim in xrange(cents): # dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) # xl = -dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # xl = dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # if( qt > qm ): # qm = qt # sib = six # bang = tang # if( qu > qm ): # qm = qu # sib = six # bang = -tang #if incline == 0: print "incline = 0 ",six,tang,qt,qu #print qm,six,sib,bang #print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True) for im in range(indcs[ifil][0], indcs[ifil][1]): kim = im - indcs[ifil][0] dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) if (kim < cents): xl = -dst * bang + sib else: xl = dst * bang + sib shift_x[im] = xl # Average shift sx_sum += shift_x[indcs[ifil][0] + cents] # #print myid,sx_sum,total_nfils sx_sum = mpi_reduce(sx_sum, 1, MPI_FLOAT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: sx_sum = float(sx_sum[0]) / total_nfils print_msg("Average shift %6.2f\n" % (sx_sum)) else: sx_sum = 0.0 sx_sum = 0.0 sx_sum = bcast_number_to_all(sx_sum, source_node=main_node) for im in range(ldata): shift_x[im] -= sx_sum #print " %3d %6.3f"%(im,shift_x[im]) #exit() # combine shifts found with the original parameters for im in range(ldata): t1 = Transform() ##import random ##shix=random.randint(-10, 10) ##t1.set_params({"type":"2D","tx":shix}) t1.set_params({"type": "2D", "tx": shift_x[im]}) # combine t0 and t1 tt = t1 * init_params[im] data[im].set_attr("xform.align2d", tt) # write out headers and STOP, under MPI writing has to be done sequentially mpi_barrier(MPI_COMM_WORLD) par_str = ["xform.align2d", "ID"] if myid == main_node: from utilities import file_type if (file_type(stack) == "bdb"): from utilities import recv_attr_dict_bdb recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata, nproc) else: from utilities import recv_attr_dict recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc) else: send_attr_dict(main_node, data, par_str, 0, ldata) if myid == main_node: print_end_msg("helical-shiftali_MPI")
def main(): import global_def from optparse import OptionParser from EMAN2 import EMUtil import os import sys from time import time progname = os.path.basename(sys.argv[0]) usage = progname + " proj_stack output_averages --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--img_per_group",type="int" , default=100 , help="number of images per group" ) parser.add_option("--radius", type="int" , default=-1 , help="radius for alignment" ) parser.add_option("--xr", type="string" , default="2 1", help="range for translation search in x direction, search is +/xr") parser.add_option("--yr", type="string" , default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)") parser.add_option("--ts", type="string" , default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional") parser.add_option("--iter", type="int" , default=30, help="number of iterations within alignment (default = 30)" ) parser.add_option("--num_ali", type="int" , default=5, help="number of alignments performed for stability (default = 5)" ) parser.add_option("--thld_err", type="float" , default=1.0, help="threshold of pixel error (default = 1.732)" ) parser.add_option("--grouping" , type="string" , default="GRP", help="do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size") parser.add_option("--delta", type="float" , default=-1.0, help="angular step for reference projections (required for GEV method)") parser.add_option("--fl", type="float" , default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter") parser.add_option("--aa", type="float" , default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter") parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ") parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version") (options,args) = parser.parse_args() from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD, MPI_TAG_UB from mpi import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT from applications import MPI_start_end, within_group_refinement, ali2d_ras from pixel_error import multi_align_stability from utilities import send_EMData, recv_EMData from utilities import get_image, bcast_number_to_all, set_params2D, get_params2D from utilities import group_proj_by_phitheta, model_circle, get_input_from_string sys.argv = mpi_init(len(sys.argv), sys.argv) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 2: stack = args[0] outdir = args[1] else: ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid) exit() if not options.MPI: ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid) exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True #if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid) #mpi_barrier(MPI_COMM_WORLD) img_per_grp = options.img_per_group radius = options.radius ite = options.iter num_ali = options.num_ali thld_err = options.thld_err xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else : yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) if radius == -1: radius = nx/2-2 mask = model_circle(radius, nx, nx) st = time() if options.grouping == "GRP": if myid == main_node: print " A ",myid," ",time()-st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") proj_params = [] for i in xrange(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) # Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta, # So I will briefly explain it here # proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers # except for the last one. Depending on the number of particles left, they will either form a # group or append themselves to the last group # angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi, # theta) is the projection angle of the center of the group, delta is the range of this group # mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates # whether it should take mirror position. # In this program angle_list and mirror list are not of interest. proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(proj_params, img_per_grp=img_per_grp) del proj_params print " B number of groups ",myid," ",len(proj_list_all),time()-st mpi_barrier(MPI_COMM_WORLD) # Number of groups, actually there could be one or two more groups, since the size of the remaining group varies # we will simply assign them to main node. n_grp = nima/img_per_grp-1 # Divide proj_list_all equally to all nodes, and becomes proj_list proj_list = [] for i in xrange(n_grp): proc_to_stay = i%number_of_proc if proc_to_stay == main_node: if myid == main_node: proj_list.append(proj_list_all[i]) elif myid == main_node: mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD) mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD) elif myid == proc_to_stay: img_per_grp = mpi_recv(1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) img_per_grp = int(img_per_grp[0]) temp = mpi_recv(img_per_grp, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) proj_list.append(map(int, temp)) del temp mpi_barrier(MPI_COMM_WORLD) print " C ",myid," ",time()-st if myid == main_node: # Assign the remaining groups to main_node for i in xrange(n_grp, len(proj_list_all)): proj_list.append(proj_list_all[i]) del proj_list_all, angle_list, mirror_list # Compute stability per projection projection direction, equal number assigned, thus overlaps elif options.grouping == "GEV": if options.delta == -1.0: ERROR("Angular step for reference projections is required for GEV method","sxproj_stability",1) from utilities import even_angles, nearestk_to_refdir, getvec refproj = even_angles(options.delta) img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid) # Now each processor keeps its own share of reference projections refprojdir = refproj[img_begin: img_end] del refproj ref_ang = [0.0]*(len(refprojdir)*2) for i in xrange(len(refprojdir)): ref_ang[i*2] = refprojdir[0][0] ref_ang[i*2+1] = refprojdir[0][1]+i*0.1 print " A ",myid," ",time()-st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") # the solution below is very slow, do not use it unless there is a problem with the i/O """ for i in xrange(number_of_proc): if myid == i: proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") mpi_barrier(MPI_COMM_WORLD) """ print " B ",myid," ",time()-st proj_ang = [0.0]*(nima*2) for i in xrange(nima): dp = proj_attr[i].get_params("spider") proj_ang[i*2] = dp["phi"] proj_ang[i*2+1] = dp["theta"] print " C ",myid," ",time()-st asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp) del proj_ang, ref_ang proj_list = [] for i in xrange(len(refprojdir)): proj_list.append(asi[i*img_per_grp:(i+1)*img_per_grp]) del asi print " D ",myid," ",time()-st #from sys import exit #exit() # Compute stability per projection elif options.grouping == "PPR": print " A ",myid," ",time()-st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") print " B ",myid," ",time()-st proj_params = [] for i in xrange(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) print " C ",myid," ",time()-st from utilities import nearest_proj proj_list, mirror_list = nearest_proj(proj_params, img_per_grp, range(img_begin, img_begin+1))#range(img_begin, img_end)) refprojdir = proj_params[img_begin: img_end] del proj_params, mirror_list print " D ",myid," ",time()-st else: ERROR("Incorrect projection grouping option","sxproj_stability",1) """ from utilities import write_text_file for i in xrange(len(proj_list)): write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid)) """ ########################################################################################################### # Begin stability test from utilities import get_params_proj, read_text_file #if myid == 0: # from utilities import read_text_file # proj_list[0] = map(int, read_text_file("lggrpp0.txt")) from utilities import model_blank aveList = [model_blank(nx,ny)]*len(proj_list) if options.grouping == "GRP": refprojdir = [[0.0,0.0,-1.0]]*len(proj_list) for i in xrange(len(proj_list)): print " E ",myid," ",time()-st class_data = EMData.read_images(stack, proj_list[i]) #print " R ",myid," ",time()-st if options.CTF : from filter import filt_ctf for im in xrange(len(class_data)): # MEM LEAK!! atemp = class_data[im].copy() btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1) class_data[im] = btemp #class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: try: t = im.get_attr("xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0,-d["tx"],-d["ty"],0,1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) #print " F ",myid," ",time()-st # Here, we perform realignment num_ali times all_ali_params = [] for j in xrange(num_ali): if( xrng[0] == 0.0 and yrng[0] == 0.0 ): avet = ali2d_ras(class_data, randomize = True, ir = 1, ou = radius, rs = 1, step = 1.0, dst = 90.0, maxit = ite, check_mirror = True, FH=options.fl, FF=options.aa) else: avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa) ali_params = [] for im in xrange(len(class_data)): alpha, sx, sy, mirror, scale = get_params2D(class_data[im]) ali_params.extend( [alpha, sx, sy, mirror] ) all_ali_params.append(ali_params) #aveList[i] = avet #print " G ",myid," ",time()-st del ali_params # We determine the stability of this group here. # stable_set contains all particles deemed stable, it is a list of list # each list has two elements, the first is the pixel error, the second is the image number # stable_set is sorted based on pixel error #from utilities import write_text_file #write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid) stable_set, mir_stab_rate, average_pix_err = multi_align_stability(all_ali_params, 0.0, 10000.0, thld_err, False, 2*radius+1) #print " H ",myid," ",time()-st if(len(stable_set) > 5): stable_set_id = [] members = [] pix_err = [] # First put the stable members into attr 'members' and 'pix_err' for s in stable_set: # s[1] - number in this subset stable_set_id.append(s[1]) # the original image number members.append(proj_list[i][s[1]]) pix_err.append(s[0]) # Then put the unstable members into attr 'members' and 'pix_err' from fundamentals import rot_shift2D avet.to_zero() if options.grouping == "GRP": aphi = 0.0 atht = 0.0 vphi = 0.0 vtht = 0.0 l = -1 for j in xrange(len(proj_list[i])): # Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses if j in stable_set_id: l += 1 avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] ) if options.grouping == "GRP": phi, theta, psi, sxs, sys = get_params_proj(class_data[j]) if( theta > 90.0): phi = (phi+540.0)%360.0 theta = 180.0 - theta aphi += phi atht += theta vphi += phi*phi vtht += theta*theta else: members.append(proj_list[i][j]) pix_err.append(99999.99) aveList[i] = avet.copy() if l>1 : l += 1 aveList[i] /= l if options.grouping == "GRP": aphi /= l atht /= l vphi = (vphi - l*aphi*aphi)/l vtht = (vtht - l*atht*atht)/l from math import sqrt refprojdir[i] = [aphi, atht, (sqrt(max(vphi,0.0))+sqrt(max(vtht,0.0)))/2.0] # Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION aveList[i].set_attr('members', members) aveList[i].set_attr('refprojdir',refprojdir[i]) aveList[i].set_attr('pixerr', pix_err) else: print " empty group ",i, refprojdir[i] aveList[i].set_attr('members',[-1]) aveList[i].set_attr('refprojdir',refprojdir[i]) aveList[i].set_attr('pixerr', [99999.]) del class_data if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node : for im in xrange(len(aveList)): aveList[im].write_image(args[1], km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im+i+70000) nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('members', map(int, members)) members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('pixerr', map(float, members)) members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('refprojdir', map(float, members)) ave.write_image(args[1], km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) for im in xrange(len(aveList)): send_EMData(aveList[im], main_node,im+myid+70000) members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) members = aveList[im].get_attr('pixerr') mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) except: mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) global_def.BATCH = False mpi_barrier(MPI_COMM_WORLD) from mpi import mpi_finalize mpi_finalize()
def main(): import global_def from optparse import OptionParser from EMAN2 import EMUtil import os import sys from time import time progname = os.path.basename(sys.argv[0]) usage = progname + " proj_stack output_averages --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--img_per_group", type="int", default=100, help="number of images per group") parser.add_option("--radius", type="int", default=-1, help="radius for alignment") parser.add_option( "--xr", type="string", default="2 1", help="range for translation search in x direction, search is +/xr") parser.add_option( "--yr", type="string", default="-1", help= "range for translation search in y direction, search is +/yr (default = same as xr)" ) parser.add_option( "--ts", type="string", default="1 0.5", help= "step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional" ) parser.add_option( "--iter", type="int", default=30, help="number of iterations within alignment (default = 30)") parser.add_option( "--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)") parser.add_option("--thld_err", type="float", default=1.0, help="threshold of pixel error (default = 1.732)") parser.add_option( "--grouping", type="string", default="GRP", help= "do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size" ) parser.add_option( "--delta", type="float", default=-1.0, help="angular step for reference projections (required for GEV method)" ) parser.add_option( "--fl", type="float", default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter") parser.add_option( "--aa", type="float", default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter") parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version") (options, args) = parser.parse_args() from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD from mpi import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT from applications import MPI_start_end, within_group_refinement, ali2d_ras from pixel_error import multi_align_stability from utilities import send_EMData, recv_EMData from utilities import get_image, bcast_number_to_all, set_params2D, get_params2D from utilities import group_proj_by_phitheta, model_circle, get_input_from_string sys.argv = mpi_init(len(sys.argv), sys.argv) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 2: stack = args[0] outdir = args[1] else: ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid) exit() if not options.MPI: ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid) exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True #if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid) #mpi_barrier(MPI_COMM_WORLD) img_per_grp = options.img_per_group radius = options.radius ite = options.iter num_ali = options.num_ali thld_err = options.thld_err xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else: yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) if radius == -1: radius = nx / 2 - 2 mask = model_circle(radius, nx, nx) st = time() if options.grouping == "GRP": if myid == main_node: print " A ", myid, " ", time() - st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") proj_params = [] for i in xrange(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) # Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta, # So I will briefly explain it here # proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers # except for the last one. Depending on the number of particles left, they will either form a # group or append themselves to the last group # angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi, # theta) is the projection angle of the center of the group, delta is the range of this group # mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates # whether it should take mirror position. # In this program angle_list and mirror list are not of interest. proj_list_all, angle_list, mirror_list = group_proj_by_phitheta( proj_params, img_per_grp=img_per_grp) del proj_params print " B number of groups ", myid, " ", len( proj_list_all), time() - st mpi_barrier(MPI_COMM_WORLD) # Number of groups, actually there could be one or two more groups, since the size of the remaining group varies # we will simply assign them to main node. n_grp = nima / img_per_grp - 1 # Divide proj_list_all equally to all nodes, and becomes proj_list proj_list = [] for i in xrange(n_grp): proc_to_stay = i % number_of_proc if proc_to_stay == main_node: if myid == main_node: proj_list.append(proj_list_all[i]) elif myid == main_node: mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) elif myid == proc_to_stay: img_per_grp = mpi_recv(1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) img_per_grp = int(img_per_grp[0]) temp = mpi_recv(img_per_grp, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) proj_list.append(map(int, temp)) del temp mpi_barrier(MPI_COMM_WORLD) print " C ", myid, " ", time() - st if myid == main_node: # Assign the remaining groups to main_node for i in xrange(n_grp, len(proj_list_all)): proj_list.append(proj_list_all[i]) del proj_list_all, angle_list, mirror_list # Compute stability per projection projection direction, equal number assigned, thus overlaps elif options.grouping == "GEV": if options.delta == -1.0: ERROR( "Angular step for reference projections is required for GEV method", "sxproj_stability", 1) from utilities import even_angles, nearestk_to_refdir, getvec refproj = even_angles(options.delta) img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid) # Now each processor keeps its own share of reference projections refprojdir = refproj[img_begin:img_end] del refproj ref_ang = [0.0] * (len(refprojdir) * 2) for i in xrange(len(refprojdir)): ref_ang[i * 2] = refprojdir[0][0] ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1 print " A ", myid, " ", time() - st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") # the solution below is very slow, do not use it unless there is a problem with the i/O """ for i in xrange(number_of_proc): if myid == i: proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") mpi_barrier(MPI_COMM_WORLD) """ print " B ", myid, " ", time() - st proj_ang = [0.0] * (nima * 2) for i in xrange(nima): dp = proj_attr[i].get_params("spider") proj_ang[i * 2] = dp["phi"] proj_ang[i * 2 + 1] = dp["theta"] print " C ", myid, " ", time() - st asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp) del proj_ang, ref_ang proj_list = [] for i in xrange(len(refprojdir)): proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp]) del asi print " D ", myid, " ", time() - st #from sys import exit #exit() # Compute stability per projection elif options.grouping == "PPR": print " A ", myid, " ", time() - st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") print " B ", myid, " ", time() - st proj_params = [] for i in xrange(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) print " C ", myid, " ", time() - st from utilities import nearest_proj proj_list, mirror_list = nearest_proj( proj_params, img_per_grp, range(img_begin, img_begin + 1)) #range(img_begin, img_end)) refprojdir = proj_params[img_begin:img_end] del proj_params, mirror_list print " D ", myid, " ", time() - st else: ERROR("Incorrect projection grouping option", "sxproj_stability", 1) """ from utilities import write_text_file for i in xrange(len(proj_list)): write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid)) """ ########################################################################################################### # Begin stability test from utilities import get_params_proj, read_text_file #if myid == 0: # from utilities import read_text_file # proj_list[0] = map(int, read_text_file("lggrpp0.txt")) from utilities import model_blank aveList = [model_blank(nx, ny)] * len(proj_list) if options.grouping == "GRP": refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list) for i in xrange(len(proj_list)): print " E ", myid, " ", time() - st class_data = EMData.read_images(stack, proj_list[i]) #print " R ",myid," ",time()-st if options.CTF: from filter import filt_ctf for im in xrange(len(class_data)): # MEM LEAK!! atemp = class_data[im].copy() btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1) class_data[im] = btemp #class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: try: t = im.get_attr( "xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) #print " F ",myid," ",time()-st # Here, we perform realignment num_ali times all_ali_params = [] for j in xrange(num_ali): if (xrng[0] == 0.0 and yrng[0] == 0.0): avet = ali2d_ras(class_data, randomize=True, ir=1, ou=radius, rs=1, step=1.0, dst=90.0, maxit=ite, check_mirror=True, FH=options.fl, FF=options.aa) else: avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa) ali_params = [] for im in xrange(len(class_data)): alpha, sx, sy, mirror, scale = get_params2D(class_data[im]) ali_params.extend([alpha, sx, sy, mirror]) all_ali_params.append(ali_params) #aveList[i] = avet #print " G ",myid," ",time()-st del ali_params # We determine the stability of this group here. # stable_set contains all particles deemed stable, it is a list of list # each list has two elements, the first is the pixel error, the second is the image number # stable_set is sorted based on pixel error #from utilities import write_text_file #write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid) stable_set, mir_stab_rate, average_pix_err = multi_align_stability( all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1) #print " H ",myid," ",time()-st if (len(stable_set) > 5): stable_set_id = [] members = [] pix_err = [] # First put the stable members into attr 'members' and 'pix_err' for s in stable_set: # s[1] - number in this subset stable_set_id.append(s[1]) # the original image number members.append(proj_list[i][s[1]]) pix_err.append(s[0]) # Then put the unstable members into attr 'members' and 'pix_err' from fundamentals import rot_shift2D avet.to_zero() if options.grouping == "GRP": aphi = 0.0 atht = 0.0 vphi = 0.0 vtht = 0.0 l = -1 for j in xrange(len(proj_list[i])): # Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses if j in stable_set_id: l += 1 avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3]) if options.grouping == "GRP": phi, theta, psi, sxs, sys = get_params_proj( class_data[j]) if (theta > 90.0): phi = (phi + 540.0) % 360.0 theta = 180.0 - theta aphi += phi atht += theta vphi += phi * phi vtht += theta * theta else: members.append(proj_list[i][j]) pix_err.append(99999.99) aveList[i] = avet.copy() if l > 1: l += 1 aveList[i] /= l if options.grouping == "GRP": aphi /= l atht /= l vphi = (vphi - l * aphi * aphi) / l vtht = (vtht - l * atht * atht) / l from math import sqrt refprojdir[i] = [ aphi, atht, (sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0 ] # Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION aveList[i].set_attr('members', members) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', pix_err) else: print " empty group ", i, refprojdir[i] aveList[i].set_attr('members', [-1]) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', [99999.]) del class_data if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node: for im in xrange(len(aveList)): aveList[im].write_image(args[1], km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im + i + 70000) nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('members', map(int, members)) members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('pixerr', map(float, members)) members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('refprojdir', map(float, members)) ave.write_image(args[1], km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in xrange(len(aveList)): send_EMData(aveList[im], main_node, im + myid + 70000) members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) members = aveList[im].get_attr('pixerr') mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) except: mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) global_def.BATCH = False mpi_barrier(MPI_COMM_WORLD) from mpi import mpi_finalize mpi_finalize()
def main(): from utilities import get_input_from_string progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack output_average --radius=particle_radius --xr=xr --yr=yr --ts=ts --thld_err=thld_err --num_ali=num_ali --fl=fl --aa=aa --CTF --verbose --stables" ) parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--radius", type="int", default=-1, help=" particle radius for alignment") parser.add_option( "--xr", type="string", default="2 1", help="range for translation search in x direction, search is +/xr (default 2,1)", ) parser.add_option( "--yr", type="string", default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)", ) parser.add_option( "--ts", type="string", default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default: 1,0.5)", ) parser.add_option("--thld_err", type="float", default=0.75, help="threshld of pixel error (default = 0.75)") parser.add_option( "--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)" ) parser.add_option("--maxit", type="int", default=30, help="number of iterations for each xr (default = 30)") parser.add_option( "--fl", type="float", default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter (default = 0.3)", ) parser.add_option( "--aa", type="float", default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter (default = 0.2)" ) parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction during the alignment ") parser.add_option( "--verbose", action="store_true", default=False, help="print individual pixel error (default = False)" ) parser.add_option( "--stables", action="store_true", default=False, help="output the stable particles number in file (default = False)", ) parser.add_option( "--method", type="string", default=" ", help="SHC (standard method is default when flag is ommitted)" ) (options, args) = parser.parse_args() if len(args) != 1 and len(args) != 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" else: if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() from applications import within_group_refinement, ali2d_ras from pixel_error import multi_align_stability from utilities import write_text_file, write_text_row global_def.BATCH = True xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else: yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) class_data = EMData.read_images(args[0]) nx = class_data[0].get_xsize() ou = options.radius num_ali = options.num_ali if ou == -1: ou = nx / 2 - 2 from utilities import model_circle, get_params2D, set_params2D mask = model_circle(ou, nx, nx) if options.CTF: from filter import filt_ctf for im in xrange(len(class_data)): # Flip phases class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: im.set_attr("previousmax", -1.0e10) try: t = im.get_attr("xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) all_ali_params = [] for ii in xrange(num_ali): ali_params = [] if options.verbose: ALPHA = [] SX = [] SY = [] MIRROR = [] if xrng[0] == 0.0 and yrng[0] == 0.0: avet = ali2d_ras( class_data, randomize=True, ir=1, ou=ou, rs=1, step=1.0, dst=90.0, maxit=options.maxit, check_mirror=True, FH=options.fl, FF=options.aa, ) else: avet = within_group_refinement( class_data, mask, True, 1, ou, 1, xrng, yrng, step, 90.0, maxit=options.maxit, FH=options.fl, FF=options.aa, method=options.method, ) from utilities import info # print " avet ",info(avet) for im in class_data: alpha, sx, sy, mirror, scale = get_params2D(im) ali_params.extend([alpha, sx, sy, mirror]) if options.verbose: ALPHA.append(alpha) SX.append(sx) SY.append(sy) MIRROR.append(mirror) all_ali_params.append(ali_params) if options.verbose: write_text_file([ALPHA, SX, SY, MIRROR], "ali_params_run_%d" % ii) """ avet = class_data[0] from utilities import read_text_file all_ali_params = [] for ii in xrange(5): temp = read_text_file( "ali_params_run_%d"%ii,-1) uuu = [] for k in xrange(len(temp[0])): uuu.extend([temp[0][k],temp[1][k],temp[2][k],temp[3][k]]) all_ali_params.append(uuu) """ stable_set, mir_stab_rate, pix_err = multi_align_stability( all_ali_params, 0.0, 10000.0, options.thld_err, options.verbose, 2 * ou + 1 ) print "%4s %20s %20s %20s %30s %6.2f" % ( "", "Size of set", "Size of stable set", "Mirror stab rate", "Pixel error prior to pruning the set above threshold of", options.thld_err, ) print "Average stat: %10d %20d %20.2f %15.2f" % (len(class_data), len(stable_set), mir_stab_rate, pix_err) if len(stable_set) > 0: if options.stables: stab_mem = [[0, 0.0, 0] for j in xrange(len(stable_set))] for j in xrange(len(stable_set)): stab_mem[j] = [int(stable_set[j][1]), stable_set[j][0], j] write_text_row(stab_mem, "stable_particles.txt") stable_set_id = [] particle_pixerr = [] for s in stable_set: stable_set_id.append(s[1]) particle_pixerr.append(s[0]) from fundamentals import rot_shift2D avet.to_zero() l = -1 print "average parameters: angle, x-shift, y-shift, mirror" for j in stable_set_id: l += 1 print " %4d %4d %12.2f %12.2f %12.2f %1d" % ( l, j, stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], int(stable_set[l][2][3]), ) avet += rot_shift2D( class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] ) avet /= l + 1 avet.set_attr("members", stable_set_id) avet.set_attr("pix_err", pix_err) avet.set_attr("pixerr", particle_pixerr) avet.write_image(args[1]) global_def.BATCH = False