def ave_ali_err(data1, data2=None, r=25, suffix="_ideal"): ''' This function determines the relative angle, shifts and mirrorness between the two lists of data. It also calculates the mirror consistent rate and average pixel error between two sets of parameters. ''' from utilities import get_params2D, combine_params2 from math import sqrt, sin, pi # Determine relative angle, shifts and mirror alphai, sxi, syi, mirror = align_diff(data1, data2, suffix) # Determine the average pixel error err = 0.0 nima = len(data1) mirror_same = 0 for i in xrange(nima): alpha1, sx1, sy1, mirror1, scale1 = get_params2D(data1[i]) if data2 != None: alpha2, sx2, sy2, mirror2, scale2 = get_params2D(data2[i]) else: alpha2, sx2, sy2, mirror2, scale2 = get_params2D(data1[i], "xform.align2d"+suffix) if abs(mirror1-mirror2) == mirror: mirror_same += 1 alpha12, sx12, sy12, mirror12 = combine_params2(alpha1, sx1, sy1, int(mirror1), alphai, sxi, syi, 0) err += max_2D_pixel_error([alpha12, sx12, sy12], [alpha2, sx2, sy2], r) return alphai, sxi, syi, mirror, float(mirror_same)/nima, err/mirror_same
def align_diff(data1, data2=None, suffix="_ideal"): ''' This function determines the relative angle, shifts and mirrorness between two list of data ''' from utilities import get_params2D nima = len(data1) if data2 != None: nima2 = len(data2) if nima2 != nima: print "Error: Number of images don't agree!" return 0.0, 0.0, 0.0, 0 else: del nima2 # Read the alignment parameters and determine the relative mirrorness ali_params1 = [] ali_params2 = [] for i in xrange(nima): alpha1, sx1, sy1, mirror1, scale1 = get_params2D(data1[i]) if data2 != None: alpha2, sx2, sy2, mirror2, scale2 = get_params2D(data2[i]) else: alpha2, sx2, sy2, mirror2, scale2 = get_params2D(data1[i], "xform.align2d"+suffix) ali_params1.extend([alpha1, sx1, sy1, mirror1]) ali_params2.extend([alpha2, sx2, sy2, mirror2]) return align_diff_params(ali_params1, ali_params2)
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + """ Input Output [options] Generate three micrographs, each micrograph contains one projection of a long filament. Input: Reference Volume, output directory Output: Three micrographs stored in output directory sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84 Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200 sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35 Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60 Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams Generate run through example script for helicon sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35 """ parser = OptionParser(usage, version=SPARXVERSION) # helicise the Atom coordinates # generate micrographs of helical filament parser.add_option( "--generate_micrograph", action="store_true", default=False, help= "Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory" ) parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction") parser.add_option("--apix", type="float", default=-1, help="pixel size in Angstroms") parser.add_option( "--rand_seed", type="int", default=14567, help= "the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs." ) parser.add_option("--Cs", type="float", default=2.0, help="Microscope Cs (spherical aberation)") parser.add_option("--voltage", type="float", default=200.0, help="Microscope voltage in KV") parser.add_option("--ac", type="float", default=10.0, help="Amplitude contrast (percentage, default=10)") parser.add_option("--nonoise", action="store_true", default=False, help="Do not add noise to the micrograph.") # generate initial volume parser.add_option("--generate_noisycyl", action="store_true", default=False, help="Generate initial volume of noisy cylinder.") parser.add_option( "--boxsize", type="string", default="100,100,200", help= "String containing x , y, z dimensions (separated by comma) in pixels") parser.add_option("--rad", type="int", default=35, help="Radius of initial volume in pixels") # generate 2D mask parser.add_option("--generate_mask", action="store_true", default=False, help="Generate 2D rectangular mask.") parser.add_option( "--masksize", type="string", default="200,200", help= "String containing x and y dimensions (separated by comma) in pixels") parser.add_option("--maskwidth", type="int", default=60, help="Width of rectangular mask") # Apply 2D alignment parameters to input stack and output new images to output stack parser.add_option( "--applyparams", action="store_true", default=False, help= "Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack" ) # Generate run script parser.add_option("--generate_script", action="store_true", default=False, help="Generate script for helicon run through example") parser.add_option("--filename", type="string", default="runhelicon", help="Name of run script to generate") parser.add_option("--seg_ny", type="int", default=180, help="y-dimension of segment used for refinement") parser.add_option( "--ptcl_dist", type="int", default=15, help= "Distance in pixels between adjacent segments windowed from same filament" ) parser.add_option( "--fract", type="float", default=0.35, help="Fraction of the volume used for applying helical symmetry.") (options, args) = parser.parse_args() if len(args) > 3: print("usage: " + usage) print("Please run '" + progname + " -h' for detailed options") else: if options.generate_script: generate_runscript(options.filename, options.seg_ny, options.ptcl_dist, options.fract) if options.generate_micrograph: if options.apix <= 0: print("Please enter pixel size.") sys.exit() generate_helimic(args[0], args[1], options.apix, options.CTF, options.Cs, options.voltage, options.ac, options.nonoise, options.rand_seed) if options.generate_noisycyl: from utilities import model_cylinder, model_gauss_noise outvol = args[0] boxdims = options.boxsize.split(',') if len(boxdims) < 1 or len(boxdims) > 3: print( "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'" ) sys.exit() nx = int(boxdims[0]) if len(boxdims) == 1: ny = nx nz = nx else: ny = int(boxdims[1]) if len(boxdims) == 3: nz = int(boxdims[2]) (model_cylinder(options.rad, nx, ny, nz) * model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol) if options.generate_mask: from utilities import model_blank, pad outvol = args[0] maskdims = options.masksize.split(',') if len(maskdims) < 1 or len(maskdims) > 2: print( "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'" ) sys.exit() nx = int(maskdims[0]) if len(maskdims) == 1: ny = nx else: ny = int(maskdims[1]) mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1, 0.0) mask.write_image(outvol) if options.applyparams: from utilities import get_im, get_params2D, set_params2D from fundamentals import cyclic_shift stack = args[0] newstack = args[1] mask = get_im(args[2]) nima = EMUtil.get_image_count(stack) for im in range(nima): prj = get_im(stack, im) alpha, sx, sy, mirror, scale = get_params2D(prj) prj = cyclic_shift(prj, int(sx)) set_params2D(prj, [0.0, 0., 0.0, 0, 1]) stat = Util.infomask(prj, mask, False) prj = old_div((prj - stat[0]), stat[1]) ctf_params = prj.get_attr("ctf") prj.set_attr('ctf_applied', 0) prj.write_image(newstack, im)
def main(): import global_def from optparse import OptionParser from EMAN2 import EMUtil import os import sys from time import time progname = os.path.basename(sys.argv[0]) usage = progname + " proj_stack output_averages --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--img_per_group", type="int", default=100, help="number of images per group") parser.add_option("--radius", type="int", default=-1, help="radius for alignment") parser.add_option( "--xr", type="string", default="2 1", help="range for translation search in x direction, search is +/xr") parser.add_option( "--yr", type="string", default="-1", help= "range for translation search in y direction, search is +/yr (default = same as xr)" ) parser.add_option( "--ts", type="string", default="1 0.5", help= "step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional" ) parser.add_option( "--iter", type="int", default=30, help="number of iterations within alignment (default = 30)") parser.add_option( "--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)") parser.add_option("--thld_err", type="float", default=1.0, help="threshold of pixel error (default = 1.732)") parser.add_option( "--grouping", type="string", default="GRP", help= "do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size" ) parser.add_option( "--delta", type="float", default=-1.0, help="angular step for reference projections (required for GEV method)" ) parser.add_option( "--fl", type="float", default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter") parser.add_option( "--aa", type="float", default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter") parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version") (options, args) = parser.parse_args() from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD from mpi import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT from applications import MPI_start_end, within_group_refinement, ali2d_ras from pixel_error import multi_align_stability from utilities import send_EMData, recv_EMData from utilities import get_image, bcast_number_to_all, set_params2D, get_params2D from utilities import group_proj_by_phitheta, model_circle, get_input_from_string sys.argv = mpi_init(len(sys.argv), sys.argv) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 2: stack = args[0] outdir = args[1] else: ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid) exit() if not options.MPI: ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid) exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True #if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid) #mpi_barrier(MPI_COMM_WORLD) img_per_grp = options.img_per_group radius = options.radius ite = options.iter num_ali = options.num_ali thld_err = options.thld_err xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else: yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) if radius == -1: radius = nx / 2 - 2 mask = model_circle(radius, nx, nx) st = time() if options.grouping == "GRP": if myid == main_node: print " A ", myid, " ", time() - st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") proj_params = [] for i in xrange(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) # Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta, # So I will briefly explain it here # proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers # except for the last one. Depending on the number of particles left, they will either form a # group or append themselves to the last group # angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi, # theta) is the projection angle of the center of the group, delta is the range of this group # mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates # whether it should take mirror position. # In this program angle_list and mirror list are not of interest. proj_list_all, angle_list, mirror_list = group_proj_by_phitheta( proj_params, img_per_grp=img_per_grp) del proj_params print " B number of groups ", myid, " ", len( proj_list_all), time() - st mpi_barrier(MPI_COMM_WORLD) # Number of groups, actually there could be one or two more groups, since the size of the remaining group varies # we will simply assign them to main node. n_grp = nima / img_per_grp - 1 # Divide proj_list_all equally to all nodes, and becomes proj_list proj_list = [] for i in xrange(n_grp): proc_to_stay = i % number_of_proc if proc_to_stay == main_node: if myid == main_node: proj_list.append(proj_list_all[i]) elif myid == main_node: mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) elif myid == proc_to_stay: img_per_grp = mpi_recv(1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) img_per_grp = int(img_per_grp[0]) temp = mpi_recv(img_per_grp, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) proj_list.append(map(int, temp)) del temp mpi_barrier(MPI_COMM_WORLD) print " C ", myid, " ", time() - st if myid == main_node: # Assign the remaining groups to main_node for i in xrange(n_grp, len(proj_list_all)): proj_list.append(proj_list_all[i]) del proj_list_all, angle_list, mirror_list # Compute stability per projection projection direction, equal number assigned, thus overlaps elif options.grouping == "GEV": if options.delta == -1.0: ERROR( "Angular step for reference projections is required for GEV method", "sxproj_stability", 1) from utilities import even_angles, nearestk_to_refdir, getvec refproj = even_angles(options.delta) img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid) # Now each processor keeps its own share of reference projections refprojdir = refproj[img_begin:img_end] del refproj ref_ang = [0.0] * (len(refprojdir) * 2) for i in xrange(len(refprojdir)): ref_ang[i * 2] = refprojdir[0][0] ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1 print " A ", myid, " ", time() - st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") # the solution below is very slow, do not use it unless there is a problem with the i/O """ for i in xrange(number_of_proc): if myid == i: proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") mpi_barrier(MPI_COMM_WORLD) """ print " B ", myid, " ", time() - st proj_ang = [0.0] * (nima * 2) for i in xrange(nima): dp = proj_attr[i].get_params("spider") proj_ang[i * 2] = dp["phi"] proj_ang[i * 2 + 1] = dp["theta"] print " C ", myid, " ", time() - st asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp) del proj_ang, ref_ang proj_list = [] for i in xrange(len(refprojdir)): proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp]) del asi print " D ", myid, " ", time() - st #from sys import exit #exit() # Compute stability per projection elif options.grouping == "PPR": print " A ", myid, " ", time() - st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") print " B ", myid, " ", time() - st proj_params = [] for i in xrange(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) print " C ", myid, " ", time() - st from utilities import nearest_proj proj_list, mirror_list = nearest_proj( proj_params, img_per_grp, range(img_begin, img_begin + 1)) #range(img_begin, img_end)) refprojdir = proj_params[img_begin:img_end] del proj_params, mirror_list print " D ", myid, " ", time() - st else: ERROR("Incorrect projection grouping option", "sxproj_stability", 1) """ from utilities import write_text_file for i in xrange(len(proj_list)): write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid)) """ ########################################################################################################### # Begin stability test from utilities import get_params_proj, read_text_file #if myid == 0: # from utilities import read_text_file # proj_list[0] = map(int, read_text_file("lggrpp0.txt")) from utilities import model_blank aveList = [model_blank(nx, ny)] * len(proj_list) if options.grouping == "GRP": refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list) for i in xrange(len(proj_list)): print " E ", myid, " ", time() - st class_data = EMData.read_images(stack, proj_list[i]) #print " R ",myid," ",time()-st if options.CTF: from filter import filt_ctf for im in xrange(len(class_data)): # MEM LEAK!! atemp = class_data[im].copy() btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1) class_data[im] = btemp #class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: try: t = im.get_attr( "xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) #print " F ",myid," ",time()-st # Here, we perform realignment num_ali times all_ali_params = [] for j in xrange(num_ali): if (xrng[0] == 0.0 and yrng[0] == 0.0): avet = ali2d_ras(class_data, randomize=True, ir=1, ou=radius, rs=1, step=1.0, dst=90.0, maxit=ite, check_mirror=True, FH=options.fl, FF=options.aa) else: avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa) ali_params = [] for im in xrange(len(class_data)): alpha, sx, sy, mirror, scale = get_params2D(class_data[im]) ali_params.extend([alpha, sx, sy, mirror]) all_ali_params.append(ali_params) #aveList[i] = avet #print " G ",myid," ",time()-st del ali_params # We determine the stability of this group here. # stable_set contains all particles deemed stable, it is a list of list # each list has two elements, the first is the pixel error, the second is the image number # stable_set is sorted based on pixel error #from utilities import write_text_file #write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid) stable_set, mir_stab_rate, average_pix_err = multi_align_stability( all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1) #print " H ",myid," ",time()-st if (len(stable_set) > 5): stable_set_id = [] members = [] pix_err = [] # First put the stable members into attr 'members' and 'pix_err' for s in stable_set: # s[1] - number in this subset stable_set_id.append(s[1]) # the original image number members.append(proj_list[i][s[1]]) pix_err.append(s[0]) # Then put the unstable members into attr 'members' and 'pix_err' from fundamentals import rot_shift2D avet.to_zero() if options.grouping == "GRP": aphi = 0.0 atht = 0.0 vphi = 0.0 vtht = 0.0 l = -1 for j in xrange(len(proj_list[i])): # Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses if j in stable_set_id: l += 1 avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3]) if options.grouping == "GRP": phi, theta, psi, sxs, sys = get_params_proj( class_data[j]) if (theta > 90.0): phi = (phi + 540.0) % 360.0 theta = 180.0 - theta aphi += phi atht += theta vphi += phi * phi vtht += theta * theta else: members.append(proj_list[i][j]) pix_err.append(99999.99) aveList[i] = avet.copy() if l > 1: l += 1 aveList[i] /= l if options.grouping == "GRP": aphi /= l atht /= l vphi = (vphi - l * aphi * aphi) / l vtht = (vtht - l * atht * atht) / l from math import sqrt refprojdir[i] = [ aphi, atht, (sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0 ] # Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION aveList[i].set_attr('members', members) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', pix_err) else: print " empty group ", i, refprojdir[i] aveList[i].set_attr('members', [-1]) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', [99999.]) del class_data if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node: for im in xrange(len(aveList)): aveList[im].write_image(args[1], km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im + i + 70000) nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('members', map(int, members)) members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('pixerr', map(float, members)) members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('refprojdir', map(float, members)) ave.write_image(args[1], km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in xrange(len(aveList)): send_EMData(aveList[im], main_node, im + myid + 70000) members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) members = aveList[im].get_attr('pixerr') mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) except: mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) global_def.BATCH = False mpi_barrier(MPI_COMM_WORLD) from mpi import mpi_finalize mpi_finalize()
def main(): from utilities import get_input_from_string progname = os.path.basename(sys.argv[0]) usage = progname + " stack output_average --radius=particle_radius --xr=xr --yr=yr --ts=ts --thld_err=thld_err --num_ali=num_ali --fl=fl --aa=aa --CTF --verbose --stables" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--radius", type="int", default=-1, help=" particle radius for alignment") parser.add_option("--xr", type="string" , default="2 1", help="range for translation search in x direction, search is +/xr (default 2,1)") parser.add_option("--yr", type="string" , default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)") parser.add_option("--ts", type="string" , default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default: 1,0.5)") parser.add_option("--thld_err", type="float", default=0.75, help="threshld of pixel error (default = 0.75)") parser.add_option("--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)") parser.add_option("--maxit", type="int", default=30, help="number of iterations for each xr (default = 30)") parser.add_option("--fl", type="float" , default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter (default = 0.3)") parser.add_option("--aa", type="float" , default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter (default = 0.2)") parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction during the alignment ") parser.add_option("--verbose", action="store_true", default=False, help="print individual pixel error (default = False)") parser.add_option("--stables", action="store_true", default=False, help="output the stable particles number in file (default = False)") parser.add_option("--method", type="string" , default=" ", help="SHC (standard method is default when flag is ommitted)") (options, args) = parser.parse_args() if len(args) != 1 and len(args) != 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" else: if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() from applications import within_group_refinement, ali2d_ras from pixel_error import multi_align_stability from utilities import write_text_file, write_text_row global_def.BATCH = True xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else : yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) class_data = EMData.read_images(args[0]) nx = class_data[0].get_xsize() ou = options.radius num_ali = options.num_ali if ou == -1: ou = nx/2-2 from utilities import model_circle, get_params2D, set_params2D mask = model_circle(ou, nx, nx) if options.CTF : from filter import filt_ctf for im in xrange(len(class_data)): # Flip phases class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: im.set_attr("previousmax", -1.0e10) try: t = im.get_attr("xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) all_ali_params = [] for ii in xrange(num_ali): ali_params = [] if options.verbose: ALPHA = [] SX = [] SY = [] MIRROR = [] if( xrng[0] == 0.0 and yrng[0] == 0.0 ): avet = ali2d_ras(class_data, randomize = True, ir = 1, ou = ou, rs = 1, step = 1.0, dst = 90.0, \ maxit = options.maxit, check_mirror = True, FH=options.fl, FF=options.aa) else: avet = within_group_refinement(class_data, mask, True, 1, ou, 1, xrng, yrng, step, 90.0, \ maxit = options.maxit, FH=options.fl, FF=options.aa, method = options.method) from utilities import info #print " avet ",info(avet) for im in class_data: alpha, sx, sy, mirror, scale = get_params2D(im) ali_params.extend([alpha, sx, sy, mirror]) if options.verbose: ALPHA.append(alpha) SX.append(sx) SY.append(sy) MIRROR.append(mirror) all_ali_params.append(ali_params) if options.verbose: write_text_file([ALPHA, SX, SY, MIRROR], "ali_params_run_%d"%ii) """ avet = class_data[0] from utilities import read_text_file all_ali_params = [] for ii in xrange(5): temp = read_text_file( "ali_params_run_%d"%ii,-1) uuu = [] for k in xrange(len(temp[0])): uuu.extend([temp[0][k],temp[1][k],temp[2][k],temp[3][k]]) all_ali_params.append(uuu) """ stable_set, mir_stab_rate, pix_err = multi_align_stability(all_ali_params, 0.0, 10000.0, options.thld_err, options.verbose, 2*ou+1) print "%4s %20s %20s %20s %30s %6.2f"%("", "Size of set", "Size of stable set", "Mirror stab rate", "Pixel error prior to pruning the set above threshold of",options.thld_err) print "Average stat: %10d %20d %20.2f %15.2f"%( len(class_data), len(stable_set), mir_stab_rate, pix_err) if( len(stable_set) > 0): if options.stables: stab_mem = [[0,0.0,0] for j in xrange(len(stable_set))] for j in xrange(len(stable_set)): stab_mem[j] = [int(stable_set[j][1]), stable_set[j][0], j] write_text_row(stab_mem, "stable_particles.txt") stable_set_id = [] particle_pixerr = [] for s in stable_set: stable_set_id.append(s[1]) particle_pixerr.append(s[0]) from fundamentals import rot_shift2D avet.to_zero() l = -1 print "average parameters: angle, x-shift, y-shift, mirror" for j in stable_set_id: l += 1 print " %4d %4d %12.2f %12.2f %12.2f %1d"%(l,j, stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], int(stable_set[l][2][3])) avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] ) avet /= (l+1) avet.set_attr('members', stable_set_id) avet.set_attr('pix_err', pix_err) avet.set_attr('pixerr', particle_pixerr) avet.write_image(args[1]) global_def.BATCH = False
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Forms chains of 2D images based on their similarities. Functionality: 4. Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent). Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers. The ways to use the program: 4.1 Use option initial to specify which image will be used as an initial seed to form the chain. sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25 4.2 If options initial is omitted, the program will determine which image best serves as initial seed to form the chain sxprocess.py input_stack.hdf output_stack.hdf --radius=25 4.3 Use option circular to form a circular chain. sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25 4.4 New circular code based on pairwise alignments sxprocess.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt 4.5 Circular ordering based on pairwise alignments sxprocess.py vols.hdf chain.hdf mask.hdf --dd --radius=25 """ parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False) parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False) parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False) parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)") parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") # import params for 2D alignment parser.add_option("--ou", type="int", default=-1, help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)") parser.add_option("--xr", type="int", default=0, help="range for translation search in x direction, search is +/xr (0)") parser.add_option("--yr", type="int", default=0, help="range for translation search in y direction, search is +/yr (0)") #parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)") parser.add_option("--pairwiseccc", type="string", default= None, help="Input/output pairwise ccc file") (options, args) = parser.parse_args() global_def.BATCH = True if options.dd: nargs = len(args) if nargs != 3: print "must provide name of input and two output files!" return stack = args[0] new_stack = args[1] from utilities import model_circle from statistics import ccc from statistics import mono lend = EMUtil.get_image_count(stack) lccc = [None]*(lend*(lend-1)/2) for i in xrange(lend-1): v1 = get_im( stack, i ) if( i == 0 and nargs == 2): nx = v1.get_xsize() ny = v1.get_ysize() nz = v1.get_ysize() if options.ou < 1 : radius = nx//2-2 else: radius = options.ou mask = model_circle(radius, nx, ny, nz) else: mask = get_im(args[2]) for j in xrange(i+1, lend): lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0] order = tsp(lccc) if(len(order) != lend): print " problem with data length" from sys import exit exit() print "Total sum of cccs :",TotalDistance(order, lccc) print "ordering :",order for i in xrange(lend): get_im(stack, order[i]).write_image( new_stack, i ) elif options.align: nargs = len(args) if nargs != 3: print "must provide name of input and two output files!" return from utilities import get_params2D, model_circle from fundamentals import rot_shift2D from statistics import ccc from time import time from alignment import align2d, align2d_scf from multi_shc import mult_transform stack = args[0] new_stack = args[1] d = EMData.read_images(stack) """ # will align anyway try: ttt = d[0].get_attr('xform.params2d') for i in xrange(len(d)): alpha, sx, sy, mirror, scale = get_params2D(d[i]) d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror) except: pass """ nx = d[0].get_xsize() ny = d[0].get_ysize() if options.ou < 1 : radius = nx//2-2 else: radius = options.ou mask = model_circle(radius, nx, ny) if(options.xr < 0): xrng = 0 else: xrng = options.xr if(options.yr < 0): yrng = xrng else: yrng = options.yr initial = max(options.initial, 0) from statistics import mono lend = len(d) lccc = [None]*(lend*(lend-1)/2) from utilities import read_text_row if options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) : st = time() for i in xrange(lend-1): for j in xrange(i+1, lend): # j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger) #alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F") alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius) lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir] #print " %4d %10.1f"%(i,time()-st) if(not os.path.exists(options.pairwiseccc)): from utilities import write_text_row write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc) elif(os.path.exists(options.pairwiseccc)): lccc = read_text_row(options.pairwiseccc) initial = int(lccc[0][0] + 0.1) del lccc[0] for i in xrange(len(lccc)): T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)}) lccc[i] = [lccc[i][0],T] tdummy = Transform({"type":"2D"}) maxsum = -1.023 for m in xrange(0,lend):#initial, initial+1): indc = range( lend ) lsnake = [[m, tdummy, 0.0]] del indc[m] lsum = 0.0 while len(indc) > 1: maxcit = -111. for i in xrange(len(indc)): cuc = lccc[mono(indc[i], lsnake[-1][0])][0] if cuc > maxcit: maxcit = cuc qi = indc[i] # Here we need transformation from the current to the previous, # meaning indc[i] -> lsnake[-1][0] T = lccc[mono(indc[i], lsnake[-1][0])][1] # If direction is from larger to smaller index, the transformation has to be inverted if( indc[i] > lsnake[-1][0] ): T = T.inverse() lsnake.append([qi,T, maxcit]) lsum += maxcit del indc[indc.index(qi)] T = lccc[mono(indc[-1], lsnake[-1][0])][1] if( indc[-1] > lsnake[-1][0]): T = T.inverse() lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]]) print " initial image and lsum ",m,lsum #print lsnake if(lsum > maxsum): maxsum = lsum init = m snake = [lsnake[i] for i in xrange(lend)] print " Initial image selected : ",init,maxsum," ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc) #for q in snake: print q from copy import deepcopy trans=deepcopy([snake[i][1] for i in xrange(len(snake))]) print [snake[i][0] for i in xrange(len(snake))] """ for m in xrange(lend): prms = trans[m].get_params("2D") print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2]) """ for k in xrange(lend-2,0,-1): T = snake[k][1] for i in xrange(k+1, lend): trans[i] = T*trans[i] # To add - apply all transformations and do the overall centering. for m in xrange(lend): prms = trans[m].get_params("2D") #print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2]) #rot_shift2D(d[snake[m][0]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image(new_stack, m) rot_shift2D(d[snake[m][0]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(new_stack, m) order = tsp(lccc) if(len(order) != lend): print " problem with data length" from sys import exit exit() print TotalDistance(order, lccc) print order ibeg = order.index(init) order = [order[(i+ibeg)%lend] for i in xrange(lend)] print TotalDistance(order, lccc) print order snake = [tdummy] for i in xrange(1,lend): # Here we need transformation from the current to the previous, # meaning order[i] -> order[i-1]] T = lccc[mono(order[i], order[i-1])][1] # If direction is from larger to smaller index, the transformation has to be inverted if( order[i] > order[i-1] ): T = T.inverse() snake.append(T) assert(len(snake) == lend) from copy import deepcopy trans = deepcopy(snake) for k in xrange(lend-2,0,-1): T = snake[k] for i in xrange(k+1, lend): trans[i] = T*trans[i] # Try to smooth the angles - complicated, I am afraid one would have to use angles forward and backwards # and find their average?? # In addition, one would have to recenter them """ trms = [] for m in xrange(lend): prms = trans[m].get_params("2D") trms.append([prms["alpha"], prms["mirror"]]) for i in xrange(3): for m in xrange(lend): mb = (m-1)%lend me = (m+1)%lend # angles order mb,m,me # calculate predicted angles mb->m """ for m in xrange(lend): prms = trans[m].get_params("2D") #rot_shift2D(d[order[m]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image("metro.hdf", m) rot_shift2D(d[order[m]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(args[2], m) """ # This was an effort to get number of loops, inconclusive, to say the least from numpy import outer, zeros, float32, sqrt lend = len(d) cor = zeros(lend,float32) cor = outer(cor, cor) for i in xrange(lend): cor[i][i] = 1.0 for i in xrange(lend-1): for j in xrange(i+1, lend): cor[i,j] = lccc[mono(i,j)][0] cor[j,i] = cor[i,j] lmbd, eigvec = pca(cor) from utilities import write_text_file nvec=20 print [lmbd[j] for j in xrange(nvec)] print " G" mm = [-1]*lend for i in xrange(lend): # row mi = -1.0e23 for j in xrange(nvec): qt = eigvec[j][i] if(abs(qt)>mi): mi = abs(qt) mm[i] = j for j in xrange(nvec): qt = eigvec[j][i] print round(qt,3), # eigenvector print mm[i] print for j in xrange(nvec): qt = [] for i in xrange(lend): if(mm[i] == j): qt.append(i) if(len(qt)>0): write_text_file(qt,"loop%02d.txt"%j) """ """ print [lmbd[j] for j in xrange(nvec)] print " B" mm = [-1]*lend for i in xrange(lend): # row mi = -1.0e23 for j in xrange(nvec): qt = eigvec[j][i]/sqrt(lmbd[j]) if(abs(qt)>mi): mi = abs(qt) mm[i] = j for j in xrange(nvec): qt = eigvec[j][i]/sqrt(lmbd[j]) print round(qt,3), # eigenvector print mm[i] print """ """ lend=3 cor = zeros(lend,float32) cor = outer(cor, cor) cor[0][0] =136.77 cor[0][1] = 79.15 cor[0][2] = 37.13 cor[1][0] = 79.15 cor[2][0] = 37.13 cor[1][1] = 50.04 cor[1][2] = 21.65 cor[2][1] = 21.65 cor[2][2] = 13.26 lmbd, eigvec = pca(cor) print lmbd print eigvec for i in xrange(lend): # row for j in xrange(lend): print eigvec[j][i], # eigenvector print print " B" for i in xrange(lend): # row for j in xrange(lend): print eigvec[j][i]/sqrt(lmbd[j]), # eigenvector print print " G" for i in xrange(lend): # row for j in xrange(lend): print eigvec[j][i]*sqrt(lmbd[j]), # eigenvector print """ else: nargs = len(args) if nargs != 2: print "must provide name of input and output file!" return from utilities import get_params2D, model_circle from fundamentals import rot_shift2D from statistics import ccc from time import time from alignment import align2d from multi_shc import mult_transform stack = args[0] new_stack = args[1] d = EMData.read_images(stack) try: ttt = d[0].get_attr('xform.params2d') for i in xrange(len(d)): alpha, sx, sy, mirror, scale = get_params2D(d[i]) d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror) except: pass nx = d[0].get_xsize() ny = d[0].get_ysize() if options.radius < 1 : radius = nx//2-2 else: radius = options.radius mask = model_circle(radius, nx, ny) init = options.initial if init > -1 : print " initial image: %d" % init temp = d[init].copy() temp.write_image(new_stack, 0) del d[init] k = 1 lsum = 0.0 while len(d) > 1: maxcit = -111. for i in xrange(len(d)): cuc = ccc(d[i], temp, mask) if cuc > maxcit: maxcit = cuc qi = i # print k, maxcit lsum += maxcit temp = d[qi].copy() del d[qi] temp.write_image(new_stack, k) k += 1 print lsum d[0].write_image(new_stack, k) else: if options.circular : print "Using options.circular" # figure the "best circular" starting image maxsum = -1.023 for m in xrange(len(d)): indc = range(len(d) ) lsnake = [-1]*(len(d)+1) lsnake[0] = m lsnake[-1] = m del indc[m] temp = d[m].copy() lsum = 0.0 direction = +1 k = 1 while len(indc) > 1: maxcit = -111. for i in xrange(len(indc)): cuc = ccc(d[indc[i]], temp, mask) if cuc > maxcit: maxcit = cuc qi = indc[i] lsnake[k] = qi lsum += maxcit del indc[indc.index(qi)] direction = -direction for i in xrange( 1,len(d) ): if( direction > 0 ): if(lsnake[i] == -1): temp = d[lsnake[i-1]].copy() #print " forw ",lsnake[i-1] k = i break else: if(lsnake[len(d) - i] == -1): temp = d[lsnake[len(d) - i +1]].copy() #print " back ",lsnake[len(d) - i +1] k = len(d) - i break lsnake[lsnake.index(-1)] = indc[-1] #print " initial image and lsum ",m,lsum #print lsnake if(lsum > maxsum): maxsum = lsum init = m snake = [lsnake[i] for i in xrange(len(d))] print " Initial image selected : ",init,maxsum print lsnake for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m) else: # figure the "best" starting image maxsum = -1.023 for m in xrange(len(d)): indc = range(len(d) ) lsnake = [m] del indc[m] temp = d[m].copy() lsum = 0.0 while len(indc) > 1: maxcit = -111. for i in xrange(len(indc)): cuc = ccc(d[indc[i]], temp, mask) if cuc > maxcit: maxcit = cuc qi = indc[i] lsnake.append(qi) lsum += maxcit temp = d[qi].copy() del indc[indc.index(qi)] lsnake.append(indc[-1]) #print " initial image and lsum ",m,lsum #print lsnake if(lsum > maxsum): maxsum = lsum init = m snake = [lsnake[i] for i in xrange(len(d))] print " Initial image selected : ",init,maxsum print lsnake for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m)
from utilities import get_params2D, model_circle from fundamentals import rot_shift2D from statistics import ccc from time import time from alignment import align2d from multi_shc import mult_transform stack = args[0] new_stack = args[1] d = EMData.read_images(stack) try: ttt = d[0].get_attr('xform.params2d') for i in xrange(len(d)): alpha, sx, sy, mirror, scale = get_params2D(d[i]) d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror) except: pass nx = d[0].get_xsize() ny = d[0].get_ysize() if options.radius < 1 : radius = nx//2-2 else: radius = options.radius mask = model_circle(radius, nx, ny) init = options.initial if init > -1 : print " initial image: %d" % init temp = d[init].copy()
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + """ Input Output [options] Generate three micrographs, each micrograph contains one projection of a long filament. Input: Reference Volume, output directory Output: Three micrographs stored in output directory sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84 Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200 sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35 Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60 Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams Generate run through example script for helicon sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35 """ parser = OptionParser(usage,version=SPARXVERSION) # helicise the Atom coordinates # generate micrographs of helical filament parser.add_option("--generate_micrograph", action="store_true", default=False, help="Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory") parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction") parser.add_option("--apix", type="float", default= -1, help="pixel size in Angstroms") parser.add_option("--rand_seed", type="int", default=14567, help="the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs.") parser.add_option("--Cs", type="float", default= 2.0, help="Microscope Cs (spherical aberation)") parser.add_option("--voltage", type="float", default=200.0, help="Microscope voltage in KV") parser.add_option("--ac", type="float", default=10.0, help="Amplitude contrast (percentage, default=10)") parser.add_option("--nonoise", action="store_true", default=False, help="Do not add noise to the micrograph.") # generate initial volume parser.add_option("--generate_noisycyl", action="store_true", default=False, help="Generate initial volume of noisy cylinder.") parser.add_option("--boxsize", type="string", default="100,100,200", help="String containing x , y, z dimensions (separated by comma) in pixels") parser.add_option("--rad", type="int", default=35, help="Radius of initial volume in pixels") # generate 2D mask parser.add_option("--generate_mask", action="store_true", default=False, help="Generate 2D rectangular mask.") parser.add_option("--masksize", type="string", default="200,200", help="String containing x and y dimensions (separated by comma) in pixels") parser.add_option("--maskwidth", type="int", default=60, help="Width of rectangular mask") # Apply 2D alignment parameters to input stack and output new images to output stack parser.add_option("--applyparams", action="store_true", default=False, help="Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack") # Generate run script parser.add_option("--generate_script", action="store_true", default=False, help="Generate script for helicon run through example") parser.add_option("--filename", type="string", default="runhelicon", help="Name of run script to generate") parser.add_option("--seg_ny", type="int", default=180, help="y-dimension of segment used for refinement") parser.add_option("--ptcl_dist", type="int", default=15, help="Distance in pixels between adjacent segments windowed from same filament") parser.add_option("--fract", type="float", default=0.35, help="Fraction of the volume used for applying helical symmetry.") (options, args) = parser.parse_args() if len(args) > 3: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" else: if options.generate_script: generate_runscript(options.filename, options.seg_ny, options.ptcl_dist, options.fract) if options.generate_micrograph: if options.apix <= 0: print "Please enter pixel size." sys.exit() generate_helimic(args[0], args[1], options.apix, options.CTF, options.Cs, options.voltage, options.ac, options.nonoise, options.rand_seed) if options.generate_noisycyl: from utilities import model_cylinder, model_gauss_noise outvol = args[0] boxdims = options.boxsize.split(',') if len(boxdims) < 1 or len(boxdims) > 3: print "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'" sys.exit() nx= int(boxdims[0]) if len(boxdims) == 1: ny = nx nz = nx else: ny = int(boxdims[1]) if len(boxdims) == 3: nz = int(boxdims[2]) (model_cylinder(options.rad,nx, ny, nz)*model_gauss_noise(1.0, nx, ny, nz) ).write_image(outvol) if options.generate_mask: from utilities import model_blank, pad outvol = args[0] maskdims = options.masksize.split(',') if len(maskdims) < 1 or len(maskdims) > 2: print "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'" sys.exit() nx= int(maskdims[0]) if len(maskdims) == 1: ny = nx else: ny = int(maskdims[1]) mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1, 0.0) mask.write_image(outvol) if options.applyparams: from utilities import get_im, get_params2D, set_params2D from fundamentals import cyclic_shift stack = args[0] newstack = args[1] mask = get_im(args[2]) nima = EMUtil.get_image_count(stack) for im in xrange(nima): prj = get_im(stack,im) alpha, sx, sy, mirror, scale = get_params2D(prj) prj = cyclic_shift(prj, int(sx)) set_params2D(prj, [0.0,0.,0.0,0,1]) stat = Util.infomask(prj , mask, False ) prj= (prj-stat[0])/stat[1] ctf_params = prj.get_attr("ctf") prj.set_attr('ctf_applied', 0) prj.write_image(newstack, im)
def compare(compare_ref_free, outfile_repro,ref_free_output,yrng, xrng, rstep,nx,apix,ref_free_cutoff, nproc, myid, main_node): from alignment import Numrinit, ringwe, Applyws from random import seed, randint from utilities import get_params2D, set_params2D, model_circle, inverse_transform2, combine_params2 from fundamentals import rot_shift2D from mpi import MPI_COMM_WORLD, mpi_barrier, mpi_bcast, MPI_INT from statistics import fsc_mask from filter import fit_tanh from numpy import array fout = "%s.hdf" % ref_free_output frc_out = "%s_frc" % ref_free_output res_out = "%s_res" % ref_free_output nima = EMUtil.get_image_count(compare_ref_free) image_start, image_end = MPI_start_end(nima, nproc, myid) ima = EMData() ima.read_image(compare_ref_free, image_start) last_ring = nx/2-2 first_ring = 1 mask = model_circle(last_ring, nx, nx) refi = [] numref = EMUtil.get_image_count(outfile_repro) cnx = nx/2 +1 cny = cnx mode = "F" numr = Numrinit(first_ring, last_ring, rstep, mode) wr = ringwe(numr, mode) ima.to_zero() for j in xrange(numref): temp = EMData() temp.read_image(outfile_repro, j) # even, odd, numer of even, number of images. After frc, totav refi.append(temp) # for each node read its share of data data = EMData.read_images(compare_ref_free, range(image_start, image_end)) for im in xrange(image_start, image_end): data[im-image_start].set_attr('ID', im) set_params2D(data[im-image_start],[0,0,0,0,1]) ringref = [] for j in xrange(numref): refi[j].process_inplace("normalize.mask", {"mask":mask, "no_sigma":1}) # normalize reference images to N(0,1) cimage = Util.Polar2Dm(refi[j], cnx, cny, numr, mode) Util.Frngs(cimage, numr) Applyws(cimage, numr, wr) ringref.append(cimage) if myid == main_node: seed(1000) data_shift = [] frc = [] res = [] for im in xrange(image_start, image_end): alpha, sx, sy, mirror, scale = get_params2D(data[im-image_start]) alphai, sxi, syi, scalei = inverse_transform2(alpha, sx, sy, 1.0) # normalize data[im-image_start].process_inplace("normalize.mask", {"mask":mask, "no_sigma":1}) # subtract average under the mask # align current image to the reference [angt, sxst, syst, mirrort, xiref, peakt] = Util.multiref_polar_ali_2d(data[im-image_start], ringref, xrng, yrng, 1, mode, numr, cnx+sxi, cny+syi) iref = int(xiref) [alphan, sxn, syn, mn] = combine_params2(0.0, -sxi, -syi, 0, angt, sxst, syst, (int)(mirrort)) set_params2D(data[im-image_start], [alphan, sxn, syn, int(mn), scale]) temp = rot_shift2D(data[im-image_start], alphan, sxn, syn, mn) temp.set_attr('assign',iref) tfrc = fsc_mask(temp,refi[iref],mask = mask) temp.set_attr('frc',tfrc[1]) res = fit_tanh(tfrc) temp.set_attr('res',res) data_shift.append(temp) for node in xrange(nproc): if myid == node: for image in data_shift: image.write_image(fout,-1) refindex = image.get_attr('assign') refi[refindex].write_image(fout,-1) mpi_barrier(MPI_COMM_WORLD) rejects = [] if myid == main_node: a = EMData() index = 0 frc = [] res = [] temp = [] classes = [] for im in xrange(nima): a.read_image(fout, index) frc.append(a.get_attr("frc")) if ref_free_cutoff != -1: classes.append(a.get_attr("class_ptcl_idxs")) tmp = a.get_attr("res") temp.append(tmp[0]) res.append("%12f" %(apix/tmp[0])) res.append("\n") index = index + 2 res_num = array(temp) mean_score = res_num.mean(axis=0) std_score = res_num.std(axis=0) std = std_score / 2 if ref_free_cutoff !=-1: cutoff = mean_score - std * ref_free_cutoff reject = res_num < cutoff index = 0 for i in reject: if i: rejects.extend(classes[index]) index = index + 1 rejects.sort() length = mpi_bcast(len(rejects),1,MPI_INT,main_node, MPI_COMM_WORLD) rejects = mpi_bcast(rejects,length , MPI_INT, main_node, MPI_COMM_WORLD) del a fout_frc = open(frc_out,'w') fout_res = open(res_out,'w') fout_res.write("".join(res)) temp = zip(*frc) datstrings = [] for i in temp: for j in i: datstrings.append(" %12f" % (j)) datstrings.append("\n") fout_frc.write("".join(datstrings)) fout_frc.close() del refi del ringref return rejects
def main(): import global_def from optparse import OptionParser from EMAN2 import EMUtil import os import sys from time import time progname = os.path.basename(sys.argv[0]) usage = progname + " proj_stack output_averages --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--img_per_group",type="int" , default=100 , help="number of images per group" ) parser.add_option("--radius", type="int" , default=-1 , help="radius for alignment" ) parser.add_option("--xr", type="string" , default="2 1", help="range for translation search in x direction, search is +/xr") parser.add_option("--yr", type="string" , default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)") parser.add_option("--ts", type="string" , default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional") parser.add_option("--iter", type="int" , default=30, help="number of iterations within alignment (default = 30)" ) parser.add_option("--num_ali", type="int" , default=5, help="number of alignments performed for stability (default = 5)" ) parser.add_option("--thld_err", type="float" , default=1.0, help="threshold of pixel error (default = 1.732)" ) parser.add_option("--grouping" , type="string" , default="GRP", help="do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size") parser.add_option("--delta", type="float" , default=-1.0, help="angular step for reference projections (required for GEV method)") parser.add_option("--fl", type="float" , default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter") parser.add_option("--aa", type="float" , default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter") parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ") parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version") (options,args) = parser.parse_args() from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD, MPI_TAG_UB from mpi import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT from applications import MPI_start_end, within_group_refinement, ali2d_ras from pixel_error import multi_align_stability from utilities import send_EMData, recv_EMData from utilities import get_image, bcast_number_to_all, set_params2D, get_params2D from utilities import group_proj_by_phitheta, model_circle, get_input_from_string sys.argv = mpi_init(len(sys.argv), sys.argv) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 2: stack = args[0] outdir = args[1] else: ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid) exit() if not options.MPI: ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid) exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True #if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid) #mpi_barrier(MPI_COMM_WORLD) img_per_grp = options.img_per_group radius = options.radius ite = options.iter num_ali = options.num_ali thld_err = options.thld_err xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else : yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) if radius == -1: radius = nx/2-2 mask = model_circle(radius, nx, nx) st = time() if options.grouping == "GRP": if myid == main_node: print " A ",myid," ",time()-st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") proj_params = [] for i in xrange(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) # Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta, # So I will briefly explain it here # proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers # except for the last one. Depending on the number of particles left, they will either form a # group or append themselves to the last group # angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi, # theta) is the projection angle of the center of the group, delta is the range of this group # mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates # whether it should take mirror position. # In this program angle_list and mirror list are not of interest. proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(proj_params, img_per_grp=img_per_grp) del proj_params print " B number of groups ",myid," ",len(proj_list_all),time()-st mpi_barrier(MPI_COMM_WORLD) # Number of groups, actually there could be one or two more groups, since the size of the remaining group varies # we will simply assign them to main node. n_grp = nima/img_per_grp-1 # Divide proj_list_all equally to all nodes, and becomes proj_list proj_list = [] for i in xrange(n_grp): proc_to_stay = i%number_of_proc if proc_to_stay == main_node: if myid == main_node: proj_list.append(proj_list_all[i]) elif myid == main_node: mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD) mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD) elif myid == proc_to_stay: img_per_grp = mpi_recv(1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) img_per_grp = int(img_per_grp[0]) temp = mpi_recv(img_per_grp, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) proj_list.append(map(int, temp)) del temp mpi_barrier(MPI_COMM_WORLD) print " C ",myid," ",time()-st if myid == main_node: # Assign the remaining groups to main_node for i in xrange(n_grp, len(proj_list_all)): proj_list.append(proj_list_all[i]) del proj_list_all, angle_list, mirror_list # Compute stability per projection projection direction, equal number assigned, thus overlaps elif options.grouping == "GEV": if options.delta == -1.0: ERROR("Angular step for reference projections is required for GEV method","sxproj_stability",1) from utilities import even_angles, nearestk_to_refdir, getvec refproj = even_angles(options.delta) img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid) # Now each processor keeps its own share of reference projections refprojdir = refproj[img_begin: img_end] del refproj ref_ang = [0.0]*(len(refprojdir)*2) for i in xrange(len(refprojdir)): ref_ang[i*2] = refprojdir[0][0] ref_ang[i*2+1] = refprojdir[0][1]+i*0.1 print " A ",myid," ",time()-st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") # the solution below is very slow, do not use it unless there is a problem with the i/O """ for i in xrange(number_of_proc): if myid == i: proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") mpi_barrier(MPI_COMM_WORLD) """ print " B ",myid," ",time()-st proj_ang = [0.0]*(nima*2) for i in xrange(nima): dp = proj_attr[i].get_params("spider") proj_ang[i*2] = dp["phi"] proj_ang[i*2+1] = dp["theta"] print " C ",myid," ",time()-st asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp) del proj_ang, ref_ang proj_list = [] for i in xrange(len(refprojdir)): proj_list.append(asi[i*img_per_grp:(i+1)*img_per_grp]) del asi print " D ",myid," ",time()-st #from sys import exit #exit() # Compute stability per projection elif options.grouping == "PPR": print " A ",myid," ",time()-st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") print " B ",myid," ",time()-st proj_params = [] for i in xrange(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) print " C ",myid," ",time()-st from utilities import nearest_proj proj_list, mirror_list = nearest_proj(proj_params, img_per_grp, range(img_begin, img_begin+1))#range(img_begin, img_end)) refprojdir = proj_params[img_begin: img_end] del proj_params, mirror_list print " D ",myid," ",time()-st else: ERROR("Incorrect projection grouping option","sxproj_stability",1) """ from utilities import write_text_file for i in xrange(len(proj_list)): write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid)) """ ########################################################################################################### # Begin stability test from utilities import get_params_proj, read_text_file #if myid == 0: # from utilities import read_text_file # proj_list[0] = map(int, read_text_file("lggrpp0.txt")) from utilities import model_blank aveList = [model_blank(nx,ny)]*len(proj_list) if options.grouping == "GRP": refprojdir = [[0.0,0.0,-1.0]]*len(proj_list) for i in xrange(len(proj_list)): print " E ",myid," ",time()-st class_data = EMData.read_images(stack, proj_list[i]) #print " R ",myid," ",time()-st if options.CTF : from filter import filt_ctf for im in xrange(len(class_data)): # MEM LEAK!! atemp = class_data[im].copy() btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1) class_data[im] = btemp #class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: try: t = im.get_attr("xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0,-d["tx"],-d["ty"],0,1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) #print " F ",myid," ",time()-st # Here, we perform realignment num_ali times all_ali_params = [] for j in xrange(num_ali): if( xrng[0] == 0.0 and yrng[0] == 0.0 ): avet = ali2d_ras(class_data, randomize = True, ir = 1, ou = radius, rs = 1, step = 1.0, dst = 90.0, maxit = ite, check_mirror = True, FH=options.fl, FF=options.aa) else: avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa) ali_params = [] for im in xrange(len(class_data)): alpha, sx, sy, mirror, scale = get_params2D(class_data[im]) ali_params.extend( [alpha, sx, sy, mirror] ) all_ali_params.append(ali_params) #aveList[i] = avet #print " G ",myid," ",time()-st del ali_params # We determine the stability of this group here. # stable_set contains all particles deemed stable, it is a list of list # each list has two elements, the first is the pixel error, the second is the image number # stable_set is sorted based on pixel error #from utilities import write_text_file #write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid) stable_set, mir_stab_rate, average_pix_err = multi_align_stability(all_ali_params, 0.0, 10000.0, thld_err, False, 2*radius+1) #print " H ",myid," ",time()-st if(len(stable_set) > 5): stable_set_id = [] members = [] pix_err = [] # First put the stable members into attr 'members' and 'pix_err' for s in stable_set: # s[1] - number in this subset stable_set_id.append(s[1]) # the original image number members.append(proj_list[i][s[1]]) pix_err.append(s[0]) # Then put the unstable members into attr 'members' and 'pix_err' from fundamentals import rot_shift2D avet.to_zero() if options.grouping == "GRP": aphi = 0.0 atht = 0.0 vphi = 0.0 vtht = 0.0 l = -1 for j in xrange(len(proj_list[i])): # Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses if j in stable_set_id: l += 1 avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] ) if options.grouping == "GRP": phi, theta, psi, sxs, sys = get_params_proj(class_data[j]) if( theta > 90.0): phi = (phi+540.0)%360.0 theta = 180.0 - theta aphi += phi atht += theta vphi += phi*phi vtht += theta*theta else: members.append(proj_list[i][j]) pix_err.append(99999.99) aveList[i] = avet.copy() if l>1 : l += 1 aveList[i] /= l if options.grouping == "GRP": aphi /= l atht /= l vphi = (vphi - l*aphi*aphi)/l vtht = (vtht - l*atht*atht)/l from math import sqrt refprojdir[i] = [aphi, atht, (sqrt(max(vphi,0.0))+sqrt(max(vtht,0.0)))/2.0] # Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION aveList[i].set_attr('members', members) aveList[i].set_attr('refprojdir',refprojdir[i]) aveList[i].set_attr('pixerr', pix_err) else: print " empty group ",i, refprojdir[i] aveList[i].set_attr('members',[-1]) aveList[i].set_attr('refprojdir',refprojdir[i]) aveList[i].set_attr('pixerr', [99999.]) del class_data if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node : for im in xrange(len(aveList)): aveList[im].write_image(args[1], km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im+i+70000) nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('members', map(int, members)) members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('pixerr', map(float, members)) members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('refprojdir', map(float, members)) ave.write_image(args[1], km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) for im in xrange(len(aveList)): send_EMData(aveList[im], main_node,im+myid+70000) members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) members = aveList[im].get_attr('pixerr') mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) except: mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) global_def.BATCH = False mpi_barrier(MPI_COMM_WORLD) from mpi import mpi_finalize mpi_finalize()
def compare(compare_ref_free, outfile_repro, ref_free_output, yrng, xrng, rstep, nx, apix, ref_free_cutoff, nproc, myid, main_node): from alignment import Numrinit, ringwe, Applyws from random import seed, randint from utilities import get_params2D, set_params2D, model_circle, inverse_transform2, combine_params2 from fundamentals import rot_shift2D from mpi import MPI_COMM_WORLD, mpi_barrier, mpi_bcast, MPI_INT from statistics import fsc_mask from filter import fit_tanh from numpy import array fout = "%s.hdf" % ref_free_output frc_out = "%s_frc" % ref_free_output res_out = "%s_res" % ref_free_output nima = EMUtil.get_image_count(compare_ref_free) image_start, image_end = MPI_start_end(nima, nproc, myid) ima = EMData() ima.read_image(compare_ref_free, image_start) last_ring = nx / 2 - 2 first_ring = 1 mask = model_circle(last_ring, nx, nx) refi = [] numref = EMUtil.get_image_count(outfile_repro) cnx = nx / 2 + 1 cny = cnx mode = "F" numr = Numrinit(first_ring, last_ring, rstep, mode) wr = ringwe(numr, mode) ima.to_zero() for j in xrange(numref): temp = EMData() temp.read_image(outfile_repro, j) # even, odd, numer of even, number of images. After frc, totav refi.append(temp) # for each node read its share of data data = EMData.read_images(compare_ref_free, range(image_start, image_end)) for im in xrange(image_start, image_end): data[im - image_start].set_attr('ID', im) set_params2D(data[im - image_start], [0, 0, 0, 0, 1]) ringref = [] for j in xrange(numref): refi[j].process_inplace("normalize.mask", { "mask": mask, "no_sigma": 1 }) # normalize reference images to N(0,1) cimage = Util.Polar2Dm(refi[j], cnx, cny, numr, mode) Util.Frngs(cimage, numr) Applyws(cimage, numr, wr) ringref.append(cimage) if myid == main_node: seed(1000) data_shift = [] frc = [] res = [] for im in xrange(image_start, image_end): alpha, sx, sy, mirror, scale = get_params2D(data[im - image_start]) alphai, sxi, syi, scalei = inverse_transform2(alpha, sx, sy, 1.0) # normalize data[im - image_start].process_inplace("normalize.mask", { "mask": mask, "no_sigma": 1 }) # subtract average under the mask # align current image to the reference [angt, sxst, syst, mirrort, xiref, peakt] = Util.multiref_polar_ali_2d(data[im - image_start], ringref, xrng, yrng, 1, mode, numr, cnx + sxi, cny + syi) iref = int(xiref) [alphan, sxn, syn, mn] = combine_params2(0.0, -sxi, -syi, 0, angt, sxst, syst, (int)(mirrort)) set_params2D(data[im - image_start], [alphan, sxn, syn, int(mn), scale]) temp = rot_shift2D(data[im - image_start], alphan, sxn, syn, mn) temp.set_attr('assign', iref) tfrc = fsc_mask(temp, refi[iref], mask=mask) temp.set_attr('frc', tfrc[1]) res = fit_tanh(tfrc) temp.set_attr('res', res) data_shift.append(temp) for node in xrange(nproc): if myid == node: for image in data_shift: image.write_image(fout, -1) refindex = image.get_attr('assign') refi[refindex].write_image(fout, -1) mpi_barrier(MPI_COMM_WORLD) rejects = [] if myid == main_node: a = EMData() index = 0 frc = [] res = [] temp = [] classes = [] for im in xrange(nima): a.read_image(fout, index) frc.append(a.get_attr("frc")) if ref_free_cutoff != -1: classes.append(a.get_attr("class_ptcl_idxs")) tmp = a.get_attr("res") temp.append(tmp[0]) res.append("%12f" % (apix / tmp[0])) res.append("\n") index = index + 2 res_num = array(temp) mean_score = res_num.mean(axis=0) std_score = res_num.std(axis=0) std = std_score / 2 if ref_free_cutoff != -1: cutoff = mean_score - std * ref_free_cutoff reject = res_num < cutoff index = 0 for i in reject: if i: rejects.extend(classes[index]) index = index + 1 rejects.sort() length = mpi_bcast(len(rejects), 1, MPI_INT, main_node, MPI_COMM_WORLD) rejects = mpi_bcast(rejects, length, MPI_INT, main_node, MPI_COMM_WORLD) del a fout_frc = open(frc_out, 'w') fout_res = open(res_out, 'w') fout_res.write("".join(res)) temp = zip(*frc) datstrings = [] for i in temp: for j in i: datstrings.append(" %12f" % (j)) datstrings.append("\n") fout_frc.write("".join(datstrings)) fout_frc.close() del refi del ringref return rejects
def main(): from utilities import get_input_from_string progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack output_average --radius=particle_radius --xr=xr --yr=yr --ts=ts --thld_err=thld_err --num_ali=num_ali --fl=fl --aa=aa --CTF --verbose --stables" ) parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--radius", type="int", default=-1, help=" particle radius for alignment") parser.add_option( "--xr", type="string", default="2 1", help="range for translation search in x direction, search is +/xr (default 2,1)", ) parser.add_option( "--yr", type="string", default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)", ) parser.add_option( "--ts", type="string", default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default: 1,0.5)", ) parser.add_option("--thld_err", type="float", default=0.75, help="threshld of pixel error (default = 0.75)") parser.add_option( "--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)" ) parser.add_option("--maxit", type="int", default=30, help="number of iterations for each xr (default = 30)") parser.add_option( "--fl", type="float", default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter (default = 0.3)", ) parser.add_option( "--aa", type="float", default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter (default = 0.2)" ) parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction during the alignment ") parser.add_option( "--verbose", action="store_true", default=False, help="print individual pixel error (default = False)" ) parser.add_option( "--stables", action="store_true", default=False, help="output the stable particles number in file (default = False)", ) parser.add_option( "--method", type="string", default=" ", help="SHC (standard method is default when flag is ommitted)" ) (options, args) = parser.parse_args() if len(args) != 1 and len(args) != 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" else: if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() from applications import within_group_refinement, ali2d_ras from pixel_error import multi_align_stability from utilities import write_text_file, write_text_row global_def.BATCH = True xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else: yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) class_data = EMData.read_images(args[0]) nx = class_data[0].get_xsize() ou = options.radius num_ali = options.num_ali if ou == -1: ou = nx / 2 - 2 from utilities import model_circle, get_params2D, set_params2D mask = model_circle(ou, nx, nx) if options.CTF: from filter import filt_ctf for im in xrange(len(class_data)): # Flip phases class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: im.set_attr("previousmax", -1.0e10) try: t = im.get_attr("xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) all_ali_params = [] for ii in xrange(num_ali): ali_params = [] if options.verbose: ALPHA = [] SX = [] SY = [] MIRROR = [] if xrng[0] == 0.0 and yrng[0] == 0.0: avet = ali2d_ras( class_data, randomize=True, ir=1, ou=ou, rs=1, step=1.0, dst=90.0, maxit=options.maxit, check_mirror=True, FH=options.fl, FF=options.aa, ) else: avet = within_group_refinement( class_data, mask, True, 1, ou, 1, xrng, yrng, step, 90.0, maxit=options.maxit, FH=options.fl, FF=options.aa, method=options.method, ) from utilities import info # print " avet ",info(avet) for im in class_data: alpha, sx, sy, mirror, scale = get_params2D(im) ali_params.extend([alpha, sx, sy, mirror]) if options.verbose: ALPHA.append(alpha) SX.append(sx) SY.append(sy) MIRROR.append(mirror) all_ali_params.append(ali_params) if options.verbose: write_text_file([ALPHA, SX, SY, MIRROR], "ali_params_run_%d" % ii) """ avet = class_data[0] from utilities import read_text_file all_ali_params = [] for ii in xrange(5): temp = read_text_file( "ali_params_run_%d"%ii,-1) uuu = [] for k in xrange(len(temp[0])): uuu.extend([temp[0][k],temp[1][k],temp[2][k],temp[3][k]]) all_ali_params.append(uuu) """ stable_set, mir_stab_rate, pix_err = multi_align_stability( all_ali_params, 0.0, 10000.0, options.thld_err, options.verbose, 2 * ou + 1 ) print "%4s %20s %20s %20s %30s %6.2f" % ( "", "Size of set", "Size of stable set", "Mirror stab rate", "Pixel error prior to pruning the set above threshold of", options.thld_err, ) print "Average stat: %10d %20d %20.2f %15.2f" % (len(class_data), len(stable_set), mir_stab_rate, pix_err) if len(stable_set) > 0: if options.stables: stab_mem = [[0, 0.0, 0] for j in xrange(len(stable_set))] for j in xrange(len(stable_set)): stab_mem[j] = [int(stable_set[j][1]), stable_set[j][0], j] write_text_row(stab_mem, "stable_particles.txt") stable_set_id = [] particle_pixerr = [] for s in stable_set: stable_set_id.append(s[1]) particle_pixerr.append(s[0]) from fundamentals import rot_shift2D avet.to_zero() l = -1 print "average parameters: angle, x-shift, y-shift, mirror" for j in stable_set_id: l += 1 print " %4d %4d %12.2f %12.2f %12.2f %1d" % ( l, j, stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], int(stable_set[l][2][3]), ) avet += rot_shift2D( class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] ) avet /= l + 1 avet.set_attr("members", stable_set_id) avet.set_attr("pix_err", pix_err) avet.set_attr("pixerr", particle_pixerr) avet.write_image(args[1]) global_def.BATCH = False