def cml_init_global_var(dpsi, delta, nprj, debug): from sp_utilities import even_angles global g_anglst, g_d_psi, g_n_psi, g_i_prj, g_n_lines, g_n_prj, g_n_anglst, g_debug, g_seq # TO FIX v = 180.0 / float(dpsi) if v != int(v): v = int(v + 0.5) dpsi = 180 // v g_anglst = even_angles(delta, 0.0, 179.9, 0.0, 359.9, 'P') g_n_anglst = len(g_anglst) g_d_psi = dpsi g_n_psi = int(360 / dpsi) g_i_prj = -1 g_n_lines = (nprj - 1) * nprj / 2 g_n_prj = nprj g_debug = debug g_seq = [0] * 2 * g_n_lines c = 0 # prepare pairwise indexes ij for i in range(g_n_prj): for j in range(i + 1, g_n_prj): g_seq[c] = i g_seq[c + 1] = j c += 2
def prgq(volft, kb, nx, delta, ref_a, sym, MPI=False): """ Generate set of projections based on even angles The command returns list of ffts of projections """ from sp_projection import prep_vol, prgs from sp_applications import MPI_start_end from sp_utilities import even_angles, model_blank from sp_fundamentals import fft # generate list of Eulerian angles for reference projections # phi, theta, psi mode = "F" ref_angles = even_angles(delta, symmetry=sym, method=ref_a, phiEqpsi="Minus") cnx = nx // 2 + 1 cny = nx // 2 + 1 num_ref = len(ref_angles) if MPI: from mpi import mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD myid = mpi_comm_rank(MPI_COMM_WORLD) ncpu = mpi_comm_size(MPI_COMM_WORLD) else: ncpu = 1 myid = 0 from sp_applications import MPI_start_end ref_start, ref_end = MPI_start_end(num_ref, ncpu, myid) prjref = [ ] # list of (image objects) reference projections in Fourier representation for i in range(num_ref): prjref.append(model_blank( nx, nx)) # I am not sure why is that necessary, why not put None's?? for i in range(ref_start, ref_end): prjref[i] = prgs( volft, kb, [ref_angles[i][0], ref_angles[i][1], ref_angles[i][2], 0.0, 0.0]) if MPI: from sp_utilities import bcast_EMData_to_all for i in range(num_ref): for j in range(ncpu): ref_start, ref_end = MPI_start_end(num_ref, ncpu, j) if i >= ref_start and i < ref_end: rootid = j bcast_EMData_to_all(prjref[i], myid, rootid) for i in range(len(ref_angles)): prjref[i].set_attr_dict({ "phi": ref_angles[i][0], "theta": ref_angles[i][1], "psi": ref_angles[i][2] }) return prjref
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " proj_stack output_averages --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--img_per_group", type="int", default=100, help="number of images per group") parser.add_option("--radius", type="int", default=-1, help="radius for alignment") parser.add_option( "--xr", type="string", default="2 1", help="range for translation search in x direction, search is +/xr") parser.add_option( "--yr", type="string", default="-1", help= "range for translation search in y direction, search is +/yr (default = same as xr)" ) parser.add_option( "--ts", type="string", default="1 0.5", help= "step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional" ) parser.add_option( "--iter", type="int", default=30, help="number of iterations within alignment (default = 30)") parser.add_option( "--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)") parser.add_option("--thld_err", type="float", default=1.0, help="threshold of pixel error (default = 1.732)") parser.add_option( "--grouping", type="string", default="GRP", help= "do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size" ) parser.add_option( "--delta", type="float", default=-1.0, help="angular step for reference projections (required for GEV method)" ) parser.add_option( "--fl", type="float", default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter") parser.add_option( "--aa", type="float", default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter") parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version") (options, args) = parser.parse_args() myid = mpi.mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi.mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 2: stack = args[0] outdir = args[1] else: sp_global_def.ERROR("Incomplete list of arguments", "sxproj_stability.main", 1, myid=myid) return if not options.MPI: sp_global_def.ERROR("Non-MPI not supported!", "sxproj_stability.main", 1, myid=myid) return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() sp_global_def.BATCH = True img_per_grp = options.img_per_group radius = options.radius ite = options.iter num_ali = options.num_ali thld_err = options.thld_err xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else: yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) if radius == -1: radius = nx / 2 - 2 mask = model_circle(radius, nx, nx) st = time() if options.grouping == "GRP": if myid == main_node: sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") proj_params = [] for i in range(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) # Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta, # So I will briefly explain it here # proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers # except for the last one. Depending on the number of particles left, they will either form a # group or append themselves to the last group # angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi, # theta) is the projection angle of the center of the group, delta is the range of this group # mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates # whether it should take mirror position. # In this program angle_list and mirror list are not of interest. proj_list_all, angle_list, mirror_list = group_proj_by_phitheta( proj_params, img_per_grp=img_per_grp) del proj_params sxprint(" B number of groups ", myid, " ", len(proj_list_all), time() - st) mpi_barrier(MPI_COMM_WORLD) # Number of groups, actually there could be one or two more groups, since the size of the remaining group varies # we will simply assign them to main node. n_grp = nima / img_per_grp - 1 # Divide proj_list_all equally to all nodes, and becomes proj_list proj_list = [] for i in range(n_grp): proc_to_stay = i % number_of_proc if proc_to_stay == main_node: if myid == main_node: proj_list.append(proj_list_all[i]) elif myid == main_node: mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) elif myid == proc_to_stay: img_per_grp = mpi_recv(1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) img_per_grp = int(img_per_grp[0]) temp = mpi_recv(img_per_grp, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) proj_list.append(list(map(int, temp))) del temp mpi_barrier(MPI_COMM_WORLD) sxprint(" C ", myid, " ", time() - st) if myid == main_node: # Assign the remaining groups to main_node for i in range(n_grp, len(proj_list_all)): proj_list.append(proj_list_all[i]) del proj_list_all, angle_list, mirror_list # Compute stability per projection projection direction, equal number assigned, thus overlaps elif options.grouping == "GEV": if options.delta == -1.0: ERROR( "Angular step for reference projections is required for GEV method" ) return from sp_utilities import even_angles, nearestk_to_refdir, getvec refproj = even_angles(options.delta) img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid) # Now each processor keeps its own share of reference projections refprojdir = refproj[img_begin:img_end] del refproj ref_ang = [0.0] * (len(refprojdir) * 2) for i in range(len(refprojdir)): ref_ang[i * 2] = refprojdir[0][0] ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1 sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") # the solution below is very slow, do not use it unless there is a problem with the i/O """ for i in xrange(number_of_proc): if myid == i: proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") mpi_barrier(MPI_COMM_WORLD) """ sxprint(" B ", myid, " ", time() - st) proj_ang = [0.0] * (nima * 2) for i in range(nima): dp = proj_attr[i].get_params("spider") proj_ang[i * 2] = dp["phi"] proj_ang[i * 2 + 1] = dp["theta"] sxprint(" C ", myid, " ", time() - st) asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp) del proj_ang, ref_ang proj_list = [] for i in range(len(refprojdir)): proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp]) del asi sxprint(" D ", myid, " ", time() - st) #from sys import exit #exit() # Compute stability per projection elif options.grouping == "PPR": sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") sxprint(" B ", myid, " ", time() - st) proj_params = [] for i in range(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) sxprint(" C ", myid, " ", time() - st) from sp_utilities import nearest_proj proj_list, mirror_list = nearest_proj( proj_params, img_per_grp, list(range(img_begin, img_begin + 1))) #range(img_begin, img_end)) refprojdir = proj_params[img_begin:img_end] del proj_params, mirror_list sxprint(" D ", myid, " ", time() - st) else: ERROR("Incorrect projection grouping option") return ########################################################################################################### # Begin stability test from sp_utilities import get_params_proj, read_text_file #if myid == 0: # from utilities import read_text_file # proj_list[0] = map(int, read_text_file("lggrpp0.txt")) from sp_utilities import model_blank aveList = [model_blank(nx, ny)] * len(proj_list) if options.grouping == "GRP": refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list) for i in range(len(proj_list)): sxprint(" E ", myid, " ", time() - st) class_data = EMData.read_images(stack, proj_list[i]) #print " R ",myid," ",time()-st if options.CTF: from sp_filter import filt_ctf for im in range(len(class_data)): # MEM LEAK!! atemp = class_data[im].copy() btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1) class_data[im] = btemp #class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: try: t = im.get_attr( "xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) #print " F ",myid," ",time()-st # Here, we perform realignment num_ali times all_ali_params = [] for j in range(num_ali): if (xrng[0] == 0.0 and yrng[0] == 0.0): avet = ali2d_ras(class_data, randomize=True, ir=1, ou=radius, rs=1, step=1.0, dst=90.0, maxit=ite, check_mirror=True, FH=options.fl, FF=options.aa) else: avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa) ali_params = [] for im in range(len(class_data)): alpha, sx, sy, mirror, scale = get_params2D(class_data[im]) ali_params.extend([alpha, sx, sy, mirror]) all_ali_params.append(ali_params) #aveList[i] = avet #print " G ",myid," ",time()-st del ali_params # We determine the stability of this group here. # stable_set contains all particles deemed stable, it is a list of list # each list has two elements, the first is the pixel error, the second is the image number # stable_set is sorted based on pixel error #from utilities import write_text_file #write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid) stable_set, mir_stab_rate, average_pix_err = multi_align_stability( all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1) #print " H ",myid," ",time()-st if (len(stable_set) > 5): stable_set_id = [] members = [] pix_err = [] # First put the stable members into attr 'members' and 'pix_err' for s in stable_set: # s[1] - number in this subset stable_set_id.append(s[1]) # the original image number members.append(proj_list[i][s[1]]) pix_err.append(s[0]) # Then put the unstable members into attr 'members' and 'pix_err' from sp_fundamentals import rot_shift2D avet.to_zero() if options.grouping == "GRP": aphi = 0.0 atht = 0.0 vphi = 0.0 vtht = 0.0 l = -1 for j in range(len(proj_list[i])): # Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses if j in stable_set_id: l += 1 avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3]) if options.grouping == "GRP": phi, theta, psi, sxs, sy_s = get_params_proj( class_data[j]) if (theta > 90.0): phi = (phi + 540.0) % 360.0 theta = 180.0 - theta aphi += phi atht += theta vphi += phi * phi vtht += theta * theta else: members.append(proj_list[i][j]) pix_err.append(99999.99) aveList[i] = avet.copy() if l > 1: l += 1 aveList[i] /= l if options.grouping == "GRP": aphi /= l atht /= l vphi = (vphi - l * aphi * aphi) / l vtht = (vtht - l * atht * atht) / l from math import sqrt refprojdir[i] = [ aphi, atht, (sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0 ] # Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION aveList[i].set_attr('members', members) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', pix_err) else: sxprint(" empty group ", i, refprojdir[i]) aveList[i].set_attr('members', [-1]) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', [99999.]) del class_data if myid == main_node: km = 0 for i in range(number_of_proc): if i == main_node: for im in range(len(aveList)): aveList[im].write_image(args[1], km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in range(nl): ave = recv_EMData(i, im + i + 70000) nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('members', list(map(int, members))) members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('pixerr', list(map(float, members))) members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('refprojdir', list(map(float, members))) ave.write_image(args[1], km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in range(len(aveList)): send_EMData(aveList[im], main_node, im + myid + 70000) members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) members = aveList[im].get_attr('pixerr') mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) except: mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) sp_global_def.BATCH = False mpi_barrier(MPI_COMM_WORLD)
def resample( prjfile, outdir, bufprefix, nbufvol, nvol, seedbase,\ delta, d, snr, CTF, npad,\ MPI, myid, ncpu, verbose = 0 ): from sp_utilities import even_angles from random import seed, jumpahead, shuffle import os from sys import exit nprj = EMUtil.get_image_count( prjfile ) if MPI: if myid == 0: if os.path.exists(outdir): nx = 1 else: nx = 0 else: nx = 0 ny = bcast_number_to_all(nx, source_node = 0) if ny == 1: ERROR('Output directory exists, please change the name and restart the program', "resample", 1,myid) mpi.mpi_barrier( mpi.MPI_COMM_WORLD ) if myid == 0: os.makedirs(outdir) sp_global_def.write_command(outdir) mpi.mpi_barrier( mpi.MPI_COMM_WORLD ) else: if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "resample", 1,0) os.makedirs(outdir) sp_global_def.write_command(outdir) if(verbose == 1): finfo=open( os.path.join(outdir, "progress%04d.txt" % myid), "w" ) else: finfo = None #print " before evenangles",myid from sp_utilities import getvec from numpy import array, reshape refa = even_angles(delta) nrefa = len(refa) refnormal = zeros((nrefa,3),'float32') tetref = [0.0]*nrefa for i in range(nrefa): tr = getvec( refa[i][0], refa[i][1] ) for j in range(3): refnormal[i][j] = tr[j] tetref[i] = refa[i][1] del refa vct = array([0.0]*(3*nprj),'float32') if myid == 0: sxprint(" will read ",myid) tr = EMUtil.get_all_attributes(prjfile,'xform.projection') tetprj = [0.0]*nprj for i in range(nprj): temp = tr[i].get_params("spider") tetprj[i] = temp["theta"] if(tetprj[i] > 90.0): tetprj[i] = 180.0 - tetprj[i] vct[3*i+0] = tr[i].at(2,0) vct[3*i+1] = tr[i].at(2,1) vct[3*i+2] = tr[i].at(2,2) del tr else: tetprj = [0.0]*nprj #print " READ ",myid if MPI: #print " will bcast",myid vct = mpi.mpi_bcast( vct, len(vct), mpi.MPI_FLOAT, 0, mpi.MPI_COMM_WORLD ) from sp_utilities import bcast_list_to_all tetprj = bcast_list_to_all(tetprj, myid, 0) #print " reshape ",myid vct = reshape(vct,(nprj,3)) assignments = [[] for i in range(nrefa)] dspn = 1.25*delta for k in range(nprj): best_s = -1.0 best_i = -1 for i in range( nrefa ): if(abs(tetprj[k] - tetref[i]) <= dspn): s = abs(refnormal[i][0]*vct[k][0] + refnormal[i][1]*vct[k][1] + refnormal[i][2]*vct[k][2]) if s > best_s: best_s = s best_i = i assignments[best_i].append(k) am = len(assignments[0]) mufur = 1.0/am for i in range(1,len(assignments)): ti = len(assignments[i]) am = min(am, ti) if(ti>0): mufur += 1.0/ti del tetprj,tetref dp = 1.0 - d # keep that many in each direction keep = int(am*dp +0.5) mufur = keep*nrefa/(1.0 - mufur*keep/float(nrefa)) if myid == 0: sxprint(" Number of projections ",nprj,". Number of reference directions ",nrefa,", multiplicative factor for the variance ",mufur) sxprint(" Minimum number of assignments ",am," Number of projections used per stratum ", keep," Number of projections in resampled structure ",int(am*dp +0.5)*nrefa) if am <2 or am == keep: sxprint("incorrect settings") exit() # FIX if(seedbase < 1): seed() jumpahead(17*myid+123) else: seed(seedbase) jumpahead(17*myid+123) volfile = os.path.join(outdir, "bsvol%04d.hdf" % myid) from random import randint niter = nvol/ncpu/nbufvol for kiter in range(niter): if(verbose == 1): finfo.write( "Iteration %d: \n" % kiter ) finfo.flush() iter_start = time() # the following has to be converted to resample mults=1 means take given projection., mults=0 means omit mults = [ [0]*nprj for i in range(nbufvol) ] for i in range(nbufvol): for l in range(nrefa): mass = assignments[l][:] shuffle(mass) mass = mass[:keep] mass.sort() #print l, " * ",mass for k in range(keep): mults[i][mass[k]] = 1 ''' lout = [] for l in xrange(len(mults[i])): if mults[i][l] == 1: lout.append(l) write_text_file(lout, os.path.join(outdir, "list%04d_%03d.txt" %(i, myid))) del lout ''' del mass rectors, fftvols, wgtvols = resample_prepare( prjfile, nbufvol, snr, CTF, npad ) resample_insert( bufprefix, fftvols, wgtvols, mults, CTF, npad, finfo ) del mults resample_finish( rectors, fftvols, wgtvols, volfile, kiter, nprj, finfo ) rectors = None fftvols = None wgtvols = None if(verbose == 1): finfo.write( "time for iteration: %10.3f\n" % (time() - iter_start) ) finfo.flush()
def main(): from sp_logger import Logger, BaseLogger_Files arglist = [] i = 0 while (i < len(sys.argv)): if sys.argv[i] == '-p4pg': i = i + 2 elif sys.argv[i] == '-p4wd': i = i + 2 else: arglist.append(sys.argv[i]) i = i + 1 progname = os.path.basename(arglist[0]) usage = progname + " stack outdir <mask> --focus=3Dmask --radius=outer_radius --delta=angular_step" +\ "--an=angular_neighborhood --maxit=max_iter --CTF --sym=c1 --function=user_function --independent=indenpendent_runs --number_of_images_per_group=number_of_images_per_group --low_pass_filter=.25 --seed=random_seed" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--focus", type="string", default='', help="bineary 3D mask for focused clustering ") parser.add_option( "--ir", type="int", default=1, help="inner radius for rotational correlation > 0 (set to 1)") parser.add_option( "--radius", type="int", default=-1, help= "particle radius in pixel for rotational correlation <nx-1 (set to the radius of the particle)" ) parser.add_option("--maxit", type="int", default=25, help="maximum number of iteration") parser.add_option( "--rs", type="int", default=1, help="step between rings in rotational correlation >0 (set to 1)") parser.add_option( "--xr", type="string", default='1', help="range for translation search in x direction, search is +/-xr ") parser.add_option( "--yr", type="string", default='-1', help= "range for translation search in y direction, search is +/-yr (default = same as xr)" ) parser.add_option( "--ts", type="string", default='0.25', help= "step size of the translation search in both directions direction, search is -xr, -xr+ts, 0, xr-ts, xr " ) parser.add_option("--delta", type="string", default='2', help="angular step of reference projections") parser.add_option("--an", type="string", default='-1', help="angular neighborhood for local searches") parser.add_option( "--center", type="int", default=0, help= "0 - if you do not want the volume to be centered, 1 - center the volume using cog (default=0)" ) parser.add_option( "--nassign", type="int", default=1, help= "number of reassignment iterations performed for each angular step (set to 3) " ) parser.add_option( "--nrefine", type="int", default=0, help= "number of alignment iterations performed for each angular step (set to 0)" ) parser.add_option("--CTF", action="store_true", default=False, help="do CTF correction during clustring") parser.add_option( "--stoprnct", type="float", default=3.0, help="Minimum percentage of assignment change to stop the program") parser.add_option("--sym", type="string", default='c1', help="symmetry of the structure ") parser.add_option("--function", type="string", default='do_volume_mrk05', help="name of the reference preparation function") parser.add_option("--independent", type="int", default=3, help="number of independent run") parser.add_option("--number_of_images_per_group", type="int", default=1000, help="number of groups") parser.add_option( "--low_pass_filter", type="float", default=-1.0, help= "absolute frequency of low-pass filter for 3d sorting on the original image size" ) parser.add_option("--nxinit", type="int", default=64, help="initial image size for sorting") parser.add_option("--unaccounted", action="store_true", default=False, help="reconstruct the unaccounted images") parser.add_option( "--seed", type="int", default=-1, help="random seed for create initial random assignment for EQ Kmeans") parser.add_option("--smallest_group", type="int", default=500, help="minimum members for identified group") parser.add_option("--sausage", action="store_true", default=False, help="way of filter volume") parser.add_option("--chunk0", type="string", default='', help="chunk0 for computing margin of error") parser.add_option("--chunk1", type="string", default='', help="chunk1 for computing margin of error") parser.add_option( "--PWadjustment", type="string", default='', help= "1-D power spectrum of PDB file used for EM volume power spectrum correction" ) parser.add_option( "--protein_shape", type="string", default='g', help= "protein shape. It defines protein preferred orientation angles. Currently it has g and f two types " ) parser.add_option( "--upscale", type="float", default=0.5, help=" scaling parameter to adjust the power spectrum of EM volumes") parser.add_option("--wn", type="int", default=0, help="optimal window size for data processing") parser.add_option( "--interpolation", type="string", default="4nn", help="3-d reconstruction interpolation method, two options trl and 4nn" ) (options, args) = parser.parse_args(arglist[1:]) if len(args) < 1 or len(args) > 4: sxprint("Usage: " + usage) sxprint("Please run \'" + progname + " -h\' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: if len(args) > 2: mask_file = args[2] else: mask_file = None orgstack = args[0] masterdir = args[1] sp_global_def.BATCH = True #---initialize MPI related variables nproc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD) myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD) mpi_comm = mpi.MPI_COMM_WORLD main_node = 0 # import some utilities from sp_utilities import get_im, bcast_number_to_all, cmdexecute, write_text_file, read_text_file, wrap_mpi_bcast, get_params_proj, write_text_row from sp_applications import recons3d_n_MPI, mref_ali3d_MPI, Kmref_ali3d_MPI from sp_statistics import k_means_match_clusters_asg_new, k_means_stab_bbenum from sp_applications import mref_ali3d_EQ_Kmeans, ali3d_mref_Kmeans_MPI # Create the main log file from sp_logger import Logger, BaseLogger_Files if myid == main_node: log_main = Logger(BaseLogger_Files()) log_main.prefix = masterdir + "/" else: log_main = None #--- fill input parameters into dictionary named after Constants Constants = {} Constants["stack"] = args[0] Constants["masterdir"] = masterdir Constants["mask3D"] = mask_file Constants["focus3Dmask"] = options.focus Constants["indep_runs"] = options.independent Constants["stoprnct"] = options.stoprnct Constants[ "number_of_images_per_group"] = options.number_of_images_per_group Constants["CTF"] = options.CTF Constants["maxit"] = options.maxit Constants["ir"] = options.ir Constants["radius"] = options.radius Constants["nassign"] = options.nassign Constants["rs"] = options.rs Constants["xr"] = options.xr Constants["yr"] = options.yr Constants["ts"] = options.ts Constants["delta"] = options.delta Constants["an"] = options.an Constants["sym"] = options.sym Constants["center"] = options.center Constants["nrefine"] = options.nrefine #Constants["fourvar"] = options.fourvar Constants["user_func"] = options.function Constants[ "low_pass_filter"] = options.low_pass_filter # enforced low_pass_filter #Constants["debug"] = options.debug Constants["main_log_prefix"] = args[1] #Constants["importali3d"] = options.importali3d Constants["myid"] = myid Constants["main_node"] = main_node Constants["nproc"] = nproc Constants["log_main"] = log_main Constants["nxinit"] = options.nxinit Constants["unaccounted"] = options.unaccounted Constants["seed"] = options.seed Constants["smallest_group"] = options.smallest_group Constants["sausage"] = options.sausage Constants["chunk0"] = options.chunk0 Constants["chunk1"] = options.chunk1 Constants["PWadjustment"] = options.PWadjustment Constants["upscale"] = options.upscale Constants["wn"] = options.wn Constants["3d-interpolation"] = options.interpolation Constants["protein_shape"] = options.protein_shape # ----------------------------------------------------- # # Create and initialize Tracker dictionary with input options Tracker = {} Tracker["constants"] = Constants Tracker["maxit"] = Tracker["constants"]["maxit"] Tracker["radius"] = Tracker["constants"]["radius"] #Tracker["xr"] = "" #Tracker["yr"] = "-1" # Do not change! #Tracker["ts"] = 1 #Tracker["an"] = "-1" #Tracker["delta"] = "2.0" #Tracker["zoom"] = True #Tracker["nsoft"] = 0 #Tracker["local"] = False #Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"] Tracker["upscale"] = Tracker["constants"]["upscale"] #Tracker["upscale"] = 0.5 Tracker[ "applyctf"] = False # Should the data be premultiplied by the CTF. Set to False for local continuous. #Tracker["refvol"] = None Tracker["nxinit"] = Tracker["constants"]["nxinit"] #Tracker["nxstep"] = 32 Tracker["icurrentres"] = -1 #Tracker["ireachedres"] = -1 #Tracker["lowpass"] = 0.4 #Tracker["falloff"] = 0.2 #Tracker["inires"] = options.inires # Now in A, convert to absolute before using Tracker["fuse_freq"] = 50 # Now in A, convert to absolute before using #Tracker["delpreviousmax"] = False #Tracker["anger"] = -1.0 #Tracker["shifter"] = -1.0 #Tracker["saturatecrit"] = 0.95 #Tracker["pixercutoff"] = 2.0 #Tracker["directory"] = "" #Tracker["previousoutputdir"] = "" #Tracker["eliminated-outliers"] = False #Tracker["mainiteration"] = 0 #Tracker["movedback"] = False #Tracker["state"] = Tracker["constants"]["states"][0] #Tracker["global_resolution"] =0.0 Tracker["orgstack"] = orgstack #-------------------------------------------------------------------- # import from utilities from sp_utilities import sample_down_1D_curve, get_initial_ID, remove_small_groups, print_upper_triangular_matrix, print_a_line_with_timestamp from sp_utilities import print_dict, get_resolution_mrk01, partition_to_groups, partition_independent_runs, get_outliers from sp_utilities import merge_groups, save_alist, margin_of_error, get_margin_of_error, do_two_way_comparison, select_two_runs, get_ali3d_params from sp_utilities import counting_projections, unload_dict, load_dict, get_stat_proj, create_random_list, get_number_of_groups, recons_mref from sp_utilities import apply_low_pass_filter, get_groups_from_partition, get_number_of_groups, get_complementary_elements_total, update_full_dict from sp_utilities import count_chunk_members, set_filter_parameters_from_adjusted_fsc, get_two_chunks_from_stack ####------------------------------------------------------------------ # # Get the pixel size; if none, set to 1.0, and the original image size from sp_utilities import get_shrink_data_huang if (myid == main_node): line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" sxprint((line + "Initialization of 3-D sorting")) a = get_im(orgstack) nnxo = a.get_xsize() if (Tracker["nxinit"] > nnxo): sp_global_def.ERROR( "Image size less than minimum permitted $d" % Tracker["nxinit"]) nnxo = -1 else: if Tracker["constants"]["CTF"]: i = a.get_attr('ctf') pixel_size = i.apix fq = pixel_size / Tracker["fuse_freq"] else: pixel_size = 1.0 # No pixel size, fusing computed as 5 Fourier pixels fq = 5.0 / nnxo del a else: nnxo = 0 fq = 0.0 pixel_size = 1.0 nnxo = bcast_number_to_all(nnxo, source_node=main_node) if (nnxo < 0): return pixel_size = bcast_number_to_all(pixel_size, source_node=main_node) fq = bcast_number_to_all(fq, source_node=main_node) if Tracker["constants"]["wn"] == 0: Tracker["constants"]["nnxo"] = nnxo else: Tracker["constants"]["nnxo"] = Tracker["constants"]["wn"] nnxo = Tracker["constants"]["nnxo"] Tracker["constants"]["pixel_size"] = pixel_size Tracker["fuse_freq"] = fq del fq, nnxo, pixel_size if (Tracker["constants"]["radius"] < 1): Tracker["constants"][ "radius"] = Tracker["constants"]["nnxo"] // 2 - 2 elif ((2 * Tracker["constants"]["radius"] + 2) > Tracker["constants"]["nnxo"]): sp_global_def.ERROR("Particle radius set too large!", myid=myid) ####----------------------------------------------------------------------------------------- # Master directory if myid == main_node: if masterdir == "": timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime()) masterdir = "master_sort3d" + timestring li = len(masterdir) cmd = "{} {}".format("mkdir -p", masterdir) os.system(cmd) else: li = 0 li = mpi.mpi_bcast(li, 1, mpi.MPI_INT, main_node, mpi.MPI_COMM_WORLD)[0] if li > 0: masterdir = mpi.mpi_bcast(masterdir, li, mpi.MPI_CHAR, main_node, mpi.MPI_COMM_WORLD) import string masterdir = string.join(masterdir, "") if myid == main_node: print_dict(Tracker["constants"], "Permanent settings of 3-D sorting program") ######### create a vstack from input stack to the local stack in masterdir # stack name set to default Tracker["constants"]["stack"] = "bdb:" + masterdir + "/rdata" Tracker["constants"]["ali3d"] = os.path.join(masterdir, "ali3d_init.txt") Tracker["constants"]["ctf_params"] = os.path.join( masterdir, "ctf_params.txt") Tracker["constants"]["partstack"] = Tracker["constants"][ "ali3d"] # also serves for refinement if myid == main_node: total_stack = EMUtil.get_image_count(Tracker["orgstack"]) else: total_stack = 0 total_stack = bcast_number_to_all(total_stack, source_node=main_node) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) from time import sleep while not os.path.exists(masterdir): sxprint("Node ", myid, " waiting...") sleep(5) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) if myid == main_node: log_main.add("Sphire sort3d ") log_main.add("the sort3d master directory is " + masterdir) ##### ###---------------------------------------------------------------------------------- # Initial data analysis and handle two chunk files from random import shuffle # Compute the resolution #### make chunkdir dictionary for computing margin of error import sp_user_functions user_func = sp_user_functions.factory[Tracker["constants"] ["user_func"]] chunk_dict = {} chunk_list = [] if myid == main_node: chunk_one = read_text_file(Tracker["constants"]["chunk0"]) chunk_two = read_text_file(Tracker["constants"]["chunk1"]) else: chunk_one = 0 chunk_two = 0 chunk_one = wrap_mpi_bcast(chunk_one, main_node) chunk_two = wrap_mpi_bcast(chunk_two, main_node) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) ######################## Read/write bdb: data on main node ############################ if myid == main_node: if (orgstack[:4] == "bdb:"): cmd = "{} {} {}".format( "e2bdb.py", orgstack, "--makevstack=" + Tracker["constants"]["stack"]) else: cmd = "{} {} {}".format("sp_cpy.py", orgstack, Tracker["constants"]["stack"]) junk = cmdexecute(cmd) cmd = "{} {} {}".format( "sp_header.py --params=xform.projection", "--export=" + Tracker["constants"]["ali3d"], orgstack) junk = cmdexecute(cmd) cmd = "{} {} {}".format( "sp_header.py --params=ctf", "--export=" + Tracker["constants"]["ctf_params"], orgstack) junk = cmdexecute(cmd) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) ########----------------------------------------------------------------------------- Tracker["total_stack"] = total_stack Tracker["constants"]["total_stack"] = total_stack Tracker["shrinkage"] = float( Tracker["nxinit"]) / Tracker["constants"]["nnxo"] Tracker[ "radius"] = Tracker["constants"]["radius"] * Tracker["shrinkage"] if Tracker["constants"]["mask3D"]: Tracker["mask3D"] = os.path.join(masterdir, "smask.hdf") else: Tracker["mask3D"] = None if Tracker["constants"]["focus3Dmask"]: Tracker["focus3D"] = os.path.join(masterdir, "sfocus.hdf") else: Tracker["focus3D"] = None if myid == main_node: if Tracker["constants"]["mask3D"]: mask_3D = get_shrink_3dmask(Tracker["nxinit"], Tracker["constants"]["mask3D"]) mask_3D.write_image(Tracker["mask3D"]) if Tracker["constants"]["focus3Dmask"]: mask_3D = get_shrink_3dmask( Tracker["nxinit"], Tracker["constants"]["focus3Dmask"]) st = Util.infomask(mask_3D, None, True) if (st[0] == 0.0): ERROR( "Incorrect focused mask, after binarize all values zero" ) mask_3D.write_image(Tracker["focus3D"]) del mask_3D if Tracker["constants"]["PWadjustment"] != '': PW_dict = {} nxinit_pwsp = sample_down_1D_curve( Tracker["constants"]["nxinit"], Tracker["constants"]["nnxo"], Tracker["constants"]["PWadjustment"]) Tracker["nxinit_PW"] = os.path.join(masterdir, "spwp.txt") if myid == main_node: write_text_file(nxinit_pwsp, Tracker["nxinit_PW"]) PW_dict[Tracker["constants"] ["nnxo"]] = Tracker["constants"]["PWadjustment"] PW_dict[Tracker["constants"]["nxinit"]] = Tracker["nxinit_PW"] Tracker["PW_dict"] = PW_dict mpi.mpi_barrier(mpi.MPI_COMM_WORLD) #-----------------------From two chunks to FSC, and low pass filter-----------------------------------------### for element in chunk_one: chunk_dict[element] = 0 for element in chunk_two: chunk_dict[element] = 1 chunk_list = [chunk_one, chunk_two] Tracker["chunk_dict"] = chunk_dict Tracker["P_chunk0"] = len(chunk_one) / float(total_stack) Tracker["P_chunk1"] = len(chunk_two) / float(total_stack) ### create two volumes to estimate resolution if myid == main_node: for index in range(2): write_text_file( chunk_list[index], os.path.join(masterdir, "chunk%01d.txt" % index)) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) vols = [] for index in range(2): data, old_shifts = get_shrink_data_huang( Tracker, Tracker["constants"]["nxinit"], os.path.join(masterdir, "chunk%01d.txt" % index), Tracker["constants"]["partstack"], myid, main_node, nproc, preshift=True) vol = recons3d_4nn_ctf_MPI(myid=myid, prjlist=data, symmetry=Tracker["constants"]["sym"], finfo=None) if myid == main_node: vol.write_image(os.path.join(masterdir, "vol%d.hdf" % index)) vols.append(vol) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) if myid == main_node: low_pass, falloff, currentres = get_resolution_mrk01( vols, Tracker["constants"]["radius"], Tracker["constants"]["nxinit"], masterdir, Tracker["mask3D"]) if low_pass > Tracker["constants"]["low_pass_filter"]: low_pass = Tracker["constants"]["low_pass_filter"] else: low_pass = 0.0 falloff = 0.0 currentres = 0.0 bcast_number_to_all(currentres, source_node=main_node) bcast_number_to_all(low_pass, source_node=main_node) bcast_number_to_all(falloff, source_node=main_node) Tracker["currentres"] = currentres Tracker["falloff"] = falloff if Tracker["constants"]["low_pass_filter"] == -1.0: Tracker["low_pass_filter"] = min( .45, low_pass / Tracker["shrinkage"]) # no better than .45 else: Tracker["low_pass_filter"] = min( .45, Tracker["constants"]["low_pass_filter"] / Tracker["shrinkage"]) Tracker["lowpass"] = Tracker["low_pass_filter"] Tracker["falloff"] = .1 Tracker["global_fsc"] = os.path.join(masterdir, "fsc.txt") ############################################################################################ if myid == main_node: log_main.add("The command-line inputs are as following:") log_main.add( "**********************************************************") for a in sys.argv: if myid == main_node: log_main.add(a) if myid == main_node: log_main.add("number of cpus used in this run is %d" % Tracker["constants"]["nproc"]) log_main.add( "**********************************************************") from sp_filter import filt_tanl ### START 3-D sorting if myid == main_node: log_main.add("----------3-D sorting program------- ") log_main.add( "current resolution %6.3f for images of original size in terms of absolute frequency" % Tracker["currentres"]) log_main.add("equivalent to %f Angstrom resolution" % (Tracker["constants"]["pixel_size"] / Tracker["currentres"] / Tracker["shrinkage"])) log_main.add("the user provided enforced low_pass_filter is %f" % Tracker["constants"]["low_pass_filter"]) #log_main.add("equivalent to %f Angstrom resolution"%(Tracker["constants"]["pixel_size"]/Tracker["constants"]["low_pass_filter"])) for index in range(2): filt_tanl( get_im(os.path.join(masterdir, "vol%01d.hdf" % index)), Tracker["low_pass_filter"], Tracker["falloff"]).write_image( os.path.join(masterdir, "volf%01d.hdf" % index)) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) from sp_utilities import get_input_from_string delta = get_input_from_string(Tracker["constants"]["delta"]) delta = delta[0] from sp_utilities import even_angles n_angles = even_angles(delta, 0, 180) this_ali3d = Tracker["constants"]["ali3d"] sampled = get_stat_proj(Tracker, delta, this_ali3d) if myid == main_node: nc = 0 for a in sampled: if len(sampled[a]) > 0: nc += 1 log_main.add("total sampled direction %10d at angle step %6.3f" % (len(n_angles), delta)) log_main.add( "captured sampled directions %10d percentage covered by data %6.3f" % (nc, float(nc) / len(n_angles) * 100)) number_of_images_per_group = Tracker["constants"][ "number_of_images_per_group"] if myid == main_node: log_main.add("user provided number_of_images_per_group %d" % number_of_images_per_group) Tracker["number_of_images_per_group"] = number_of_images_per_group number_of_groups = get_number_of_groups(total_stack, number_of_images_per_group) Tracker["number_of_groups"] = number_of_groups generation = 0 partition_dict = {} full_dict = {} workdir = os.path.join(masterdir, "generation%03d" % generation) Tracker["this_dir"] = workdir if myid == main_node: log_main.add("---- generation %5d" % generation) log_main.add("number of images per group is set as %d" % number_of_images_per_group) log_main.add("the initial number of groups is %10d " % number_of_groups) cmd = "{} {}".format("mkdir", workdir) os.system(cmd) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) list_to_be_processed = list(range(Tracker["constants"]["total_stack"])) Tracker["this_data_list"] = list_to_be_processed create_random_list(Tracker) ################################# full_dict = {} for iptl in range(Tracker["constants"]["total_stack"]): full_dict[iptl] = iptl Tracker["full_ID_dict"] = full_dict ################################# for indep_run in range(Tracker["constants"]["indep_runs"]): Tracker["this_particle_list"] = Tracker["this_indep_list"][ indep_run] ref_vol = recons_mref(Tracker) if myid == main_node: log_main.add("independent run %10d" % indep_run) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) Tracker["this_data_list"] = list_to_be_processed Tracker["total_stack"] = len(Tracker["this_data_list"]) Tracker["this_particle_text_file"] = os.path.join( workdir, "independent_list_%03d.txt" % indep_run) # for get_shrink_data if myid == main_node: write_text_file(Tracker["this_data_list"], Tracker["this_particle_text_file"]) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) outdir = os.path.join(workdir, "EQ_Kmeans%03d" % indep_run) ref_vol = apply_low_pass_filter(ref_vol, Tracker) mref_ali3d_EQ_Kmeans(ref_vol, outdir, Tracker["this_particle_text_file"], Tracker) partition_dict[indep_run] = Tracker["this_partition"] Tracker["partition_dict"] = partition_dict Tracker["total_stack"] = len(Tracker["this_data_list"]) Tracker["this_total_stack"] = Tracker["total_stack"] ############################### do_two_way_comparison(Tracker) ############################### ref_vol_list = [] from time import sleep number_of_ref_class = [] for igrp in range(len(Tracker["two_way_stable_member"])): Tracker["this_data_list"] = Tracker["two_way_stable_member"][igrp] Tracker["this_data_list_file"] = os.path.join( workdir, "stable_class%d.txt" % igrp) if myid == main_node: write_text_file(Tracker["this_data_list"], Tracker["this_data_list_file"]) data, old_shifts = get_shrink_data_huang( Tracker, Tracker["nxinit"], Tracker["this_data_list_file"], Tracker["constants"]["partstack"], myid, main_node, nproc, preshift=True) volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist=data, symmetry=Tracker["constants"]["sym"], finfo=None) ref_vol_list.append(volref) number_of_ref_class.append(len(Tracker["this_data_list"])) if myid == main_node: log_main.add("group %d members %d " % (igrp, len(Tracker["this_data_list"]))) Tracker["number_of_ref_class"] = number_of_ref_class nx_of_image = ref_vol_list[0].get_xsize() if Tracker["constants"]["PWadjustment"]: Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image] else: Tracker["PWadjustment"] = Tracker["constants"][ "PWadjustment"] # no PW adjustment if myid == main_node: for iref in range(len(ref_vol_list)): refdata = [None] * 4 refdata[0] = ref_vol_list[iref] refdata[1] = Tracker refdata[2] = Tracker["constants"]["myid"] refdata[3] = Tracker["constants"]["nproc"] volref = user_func(refdata) volref.write_image(os.path.join(workdir, "volf_stable.hdf"), iref) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) Tracker["this_data_list"] = Tracker["this_accounted_list"] outdir = os.path.join(workdir, "Kmref") empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI( ref_vol_list, outdir, Tracker["this_accounted_text"], Tracker) Tracker["this_unaccounted_list"] = get_complementary_elements( list_to_be_processed, final_list) if myid == main_node: log_main.add("the number of particles not processed is %d" % len(Tracker["this_unaccounted_list"])) write_text_file(Tracker["this_unaccounted_list"], Tracker["this_unaccounted_text"]) update_full_dict(Tracker["this_unaccounted_list"], Tracker) ####################################### number_of_groups = len(res_groups) vol_list = [] number_of_ref_class = [] for igrp in range(number_of_groups): data, old_shifts = get_shrink_data_huang( Tracker, Tracker["constants"]["nnxo"], os.path.join(outdir, "Class%d.txt" % igrp), Tracker["constants"]["partstack"], myid, main_node, nproc, preshift=True) volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist=data, symmetry=Tracker["constants"]["sym"], finfo=None) vol_list.append(volref) if (myid == main_node): npergroup = len( read_text_file(os.path.join(outdir, "Class%d.txt" % igrp))) else: npergroup = 0 npergroup = bcast_number_to_all(npergroup, main_node) number_of_ref_class.append(npergroup) Tracker["number_of_ref_class"] = number_of_ref_class mpi.mpi_barrier(mpi.MPI_COMM_WORLD) nx_of_image = vol_list[0].get_xsize() if Tracker["constants"]["PWadjustment"]: Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image] else: Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"] if myid == main_node: for ivol in range(len(vol_list)): refdata = [None] * 4 refdata[0] = vol_list[ivol] refdata[1] = Tracker refdata[2] = Tracker["constants"]["myid"] refdata[3] = Tracker["constants"]["nproc"] volref = user_func(refdata) volref.write_image( os.path.join(workdir, "volf_of_Classes.hdf"), ivol) log_main.add("number of unaccounted particles %10d" % len(Tracker["this_unaccounted_list"])) log_main.add("number of accounted particles %10d" % len(Tracker["this_accounted_list"])) Tracker["this_data_list"] = Tracker[ "this_unaccounted_list"] # reset parameters for the next round calculation Tracker["total_stack"] = len(Tracker["this_unaccounted_list"]) Tracker["this_total_stack"] = Tracker["total_stack"] number_of_groups = get_number_of_groups( len(Tracker["this_unaccounted_list"]), number_of_images_per_group) Tracker["number_of_groups"] = number_of_groups while number_of_groups >= 2: generation += 1 partition_dict = {} workdir = os.path.join(masterdir, "generation%03d" % generation) Tracker["this_dir"] = workdir if myid == main_node: log_main.add("*********************************************") log_main.add("----- generation %5d " % generation) log_main.add("number of images per group is set as %10d " % number_of_images_per_group) log_main.add("the number of groups is %10d " % number_of_groups) log_main.add(" number of particles for clustering is %10d" % Tracker["total_stack"]) cmd = "{} {}".format("mkdir", workdir) os.system(cmd) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) create_random_list(Tracker) for indep_run in range(Tracker["constants"]["indep_runs"]): Tracker["this_particle_list"] = Tracker["this_indep_list"][ indep_run] ref_vol = recons_mref(Tracker) if myid == main_node: log_main.add("independent run %10d" % indep_run) outdir = os.path.join(workdir, "EQ_Kmeans%03d" % indep_run) Tracker["this_data_list"] = Tracker["this_unaccounted_list"] #ref_vol=apply_low_pass_filter(ref_vol,Tracker) mref_ali3d_EQ_Kmeans(ref_vol, outdir, Tracker["this_unaccounted_text"], Tracker) partition_dict[indep_run] = Tracker["this_partition"] Tracker["this_data_list"] = Tracker["this_unaccounted_list"] Tracker["total_stack"] = len(Tracker["this_unaccounted_list"]) Tracker["partition_dict"] = partition_dict Tracker["this_total_stack"] = Tracker["total_stack"] total_list_of_this_run = Tracker["this_unaccounted_list"] ############################### do_two_way_comparison(Tracker) ############################### ref_vol_list = [] number_of_ref_class = [] for igrp in range(len(Tracker["two_way_stable_member"])): Tracker["this_data_list"] = Tracker["two_way_stable_member"][ igrp] Tracker["this_data_list_file"] = os.path.join( workdir, "stable_class%d.txt" % igrp) if myid == main_node: write_text_file(Tracker["this_data_list"], Tracker["this_data_list_file"]) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) data, old_shifts = get_shrink_data_huang( Tracker, Tracker["constants"]["nxinit"], Tracker["this_data_list_file"], Tracker["constants"]["partstack"], myid, main_node, nproc, preshift=True) volref = recons3d_4nn_ctf_MPI( myid=myid, prjlist=data, symmetry=Tracker["constants"]["sym"], finfo=None) #volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1) if myid == main_node: volref.write_image(os.path.join(workdir, "vol_stable.hdf"), iref) #volref = resample(volref,Tracker["shrinkage"]) ref_vol_list.append(volref) number_of_ref_class.append(len(Tracker["this_data_list"])) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) Tracker["number_of_ref_class"] = number_of_ref_class Tracker["this_data_list"] = Tracker["this_accounted_list"] outdir = os.path.join(workdir, "Kmref") empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI( ref_vol_list, outdir, Tracker["this_accounted_text"], Tracker) # calculate the 3-D structure of original image size for each group number_of_groups = len(res_groups) Tracker["this_unaccounted_list"] = get_complementary_elements( total_list_of_this_run, final_list) if myid == main_node: log_main.add("the number of particles not processed is %d" % len(Tracker["this_unaccounted_list"])) write_text_file(Tracker["this_unaccounted_list"], Tracker["this_unaccounted_text"]) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) update_full_dict(Tracker["this_unaccounted_list"], Tracker) vol_list = [] for igrp in range(number_of_groups): data, old_shifts = get_shrink_data_huang( Tracker, Tracker["constants"]["nnxo"], os.path.join(outdir, "Class%d.txt" % igrp), Tracker["constants"]["partstack"], myid, main_node, nproc, preshift=True) volref = recons3d_4nn_ctf_MPI( myid=myid, prjlist=data, symmetry=Tracker["constants"]["sym"], finfo=None) vol_list.append(volref) mpi.mpi_barrier(mpi.MPI_COMM_WORLD) nx_of_image = ref_vol_list[0].get_xsize() if Tracker["constants"]["PWadjustment"]: Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image] else: Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"] if myid == main_node: for ivol in range(len(vol_list)): refdata = [None] * 4 refdata[0] = vol_list[ivol] refdata[1] = Tracker refdata[2] = Tracker["constants"]["myid"] refdata[3] = Tracker["constants"]["nproc"] volref = user_func(refdata) volref.write_image( os.path.join(workdir, "volf_of_Classes.hdf"), ivol) log_main.add("number of unaccounted particles %10d" % len(Tracker["this_unaccounted_list"])) log_main.add("number of accounted particles %10d" % len(Tracker["this_accounted_list"])) del vol_list mpi.mpi_barrier(mpi.MPI_COMM_WORLD) number_of_groups = get_number_of_groups( len(Tracker["this_unaccounted_list"]), number_of_images_per_group) Tracker["number_of_groups"] = number_of_groups Tracker["this_data_list"] = Tracker["this_unaccounted_list"] Tracker["total_stack"] = len(Tracker["this_unaccounted_list"]) if Tracker["constants"]["unaccounted"]: data, old_shifts = get_shrink_data_huang( Tracker, Tracker["constants"]["nnxo"], Tracker["this_unaccounted_text"], Tracker["constants"]["partstack"], myid, main_node, nproc, preshift=True) volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist=data, symmetry=Tracker["constants"]["sym"], finfo=None) nx_of_image = volref.get_xsize() if Tracker["constants"]["PWadjustment"]: Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image] else: Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"] if (myid == main_node): refdata = [None] * 4 refdata[0] = volref refdata[1] = Tracker refdata[2] = Tracker["constants"]["myid"] refdata[3] = Tracker["constants"]["nproc"] volref = user_func(refdata) #volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1) volref.write_image( os.path.join(workdir, "volf_unaccounted.hdf")) # Finish program if myid == main_node: log_main.add("sxsort3d finishes") mpi.mpi_barrier(mpi.MPI_COMM_WORLD) return
def prepare_refrings( volft, kb, nz=-1, delta=2.0, ref_a="P", sym="c1", numr=None, MPI=False, phiEqpsi="Zero", kbx=None, kby=None, initial_theta=None, delta_theta=None, initial_phi=None, ): """ Generate quasi-evenly distributed reference projections converted to rings ref_a can be a list of angles, in which case it is used instead of being generated """ # mpi communicator can be sent by the MPI parameter if type(MPI) is bool: if MPI: mpi_comm = mpi.MPI_COMM_WORLD else: mpi_comm = MPI MPI = True mode = "F" if type(ref_a) is list: # if ref_a is list, it has to be a list of projection directions, use it ref_angles = ref_a else: # generate list of Eulerian angles for reference projections # phi, theta, psi if initial_theta and initial_phi: ref_angles = sp_utilities.even_angles( delta, theta1=initial_theta, phi1=initial_phi, symmetry=sym, method=ref_a, phiEqpsi=phiEqpsi, ) else: if initial_theta is None: if sym[:1] == "c" or sym[:1] == "d": ref_angles = sp_utilities.even_angles(delta, symmetry=sym, method=ref_a, phiEqpsi=phiEqpsi) else: psp = sp_fundamentals.symclass(sym) ref_angles = psp.even_angles(delta) del psp else: if delta_theta is None: delta_theta = 1.0 ref_angles = sp_utilities.even_angles( delta, theta1=initial_theta, theta2=delta_theta, symmetry=sym, method=ref_a, phiEqpsi=phiEqpsi, ) wr_four = ringwe(numr, mode) cnx = old_div(nz, 2) + 1 cny = old_div(nz, 2) + 1 num_ref = len(ref_angles) if MPI: myid = mpi.mpi_comm_rank(mpi_comm) ncpu = mpi.mpi_comm_size(mpi_comm) else: ncpu = 1 myid = 0 if nz < 1: sp_global_def.ERROR("Data size has to be given (nz)", "prepare_refrings", 1, myid) ref_start, ref_end = sp_applications.MPI_start_end(num_ref, ncpu, myid) refrings = ( [] ) # list of (image objects) reference projections in Fourier representation sizex = numr[len(numr) - 2] + numr[len(numr) - 1] - 1 for i in range(num_ref): prjref = EMAN2_cppwrap.EMData() prjref.set_size(sizex, 1, 1) refrings.append(prjref) if kbx is None: for i in range(ref_start, ref_end): prjref = sp_projection.prgs( volft, kb, [ ref_angles[i][0], ref_angles[i][1], ref_angles[i][2], 0.0, 0.0 ], ) cimage = EMAN2_cppwrap.Util.Polar2Dm( prjref, cnx, cny, numr, mode) # currently set to quadratic.... EMAN2_cppwrap.Util.Normalize_ring(cimage, numr, 0) EMAN2_cppwrap.Util.Frngs(cimage, numr) EMAN2_cppwrap.Util.Applyws(cimage, numr, wr_four) refrings[i] = cimage else: for i in range(ref_start, ref_end): prjref = sp_projection.prgs( volft, kb, [ ref_angles[i][0], ref_angles[i][1], ref_angles[i][2], 0.0, 0.0 ], kbx, kby, ) cimage = EMAN2_cppwrap.Util.Polar2Dm( prjref, cnx, cny, numr, mode) # currently set to quadratic.... EMAN2_cppwrap.Util.Normalize_ring(cimage, numr, 0) EMAN2_cppwrap.Util.Frngs(cimage, numr) EMAN2_cppwrap.Util.Applyws(cimage, numr, wr_four) refrings[i] = cimage if MPI: sp_utilities.bcast_compacted_EMData_all_to_all(refrings, myid, comm=mpi_comm) for i in range(len(ref_angles)): n1, n2, n3 = sp_utilities.getfvec(ref_angles[i][0], ref_angles[i][1]) refrings[i].set_attr_dict({ "phi": ref_angles[i][0], "theta": ref_angles[i][1], "psi": ref_angles[i][2], "n1": n1, "n2": n2, "n3": n3, }) return refrings