def errors_per_image(params, avgtrans, thresherr=1.0, radius = 1.0): #print "errors per image" # We have average parameters and transformed parameters # Find errors per image ll = len(params) nn = len(params[0]) perr = [None]*nn for k in xrange(nn): # nas = [0.0,0.0,0.0] r1,r2,r3 = getfvec(avgtrans[k][0],avgtrans[k][1]) sacos = 0.0 sder = 0.0 ser3 = 0.0 for i in xrange(ll): #print i,fifi[i] d = max_3D_pixel_error(params[i][k], avgtrans[k], r=radius) #if(d>10.): print " LARGE ERROR",k,i,d,fifi[i], avgtrans[k] ser3 += d n1,n2,n3 = getfvec(params[i][k][0],params[i][k][1]) sacos += acos( min(1.0, n1*r1+n2*r2+n3*r3) ) sder += pixel_error_2D(params[i][k][2:], avgtrans[k][2:], r = 1.0) # average deviation in radians sacos /= ll sare = tan(sacos) sder /= ll ser3 /= ll #print k, ser3, sare, sder perr[k] = [ser3, sacos, sare, sder] #write_text_row(perr, 'per.txt') #perr = [perr[k][1] <= thresherr for k in xrange(nn)] return perr
def errors_per_image(params, avgtrans, thresherr=1.0, radius=1.0): #print "errors per image" # We have average parameters and transformed parameters # Find errors per image ll = len(params) nn = len(params[0]) perr = [None] * nn for k in xrange(nn): # nas = [0.0, 0.0, 0.0] r1, r2, r3 = getfvec(avgtrans[k][0], avgtrans[k][1]) sacos = 0.0 sder = 0.0 ser3 = 0.0 for i in xrange(ll): #print i,fifi[i] d = max_3D_pixel_error(params[i][k], avgtrans[k], r=radius) #if(d>10.): print " LARGE ERROR",k,i,d,fifi[i], avgtrans[k] ser3 += d n1, n2, n3 = getfvec(params[i][k][0], params[i][k][1]) sacos += lacos(n1 * r1 + n2 * r2 + n3 * r3) sder += pixel_error_2D(params[i][k][2:], avgtrans[k][2:], r=1.0) # average deviation in radians sacos /= ll sare = tan(sacos) sder /= ll ser3 /= ll #print k, ser3, sare, sder perr[k] = [ser3, sacos, sare, sder] #write_text_row(perr, 'per.txt') #perr = [perr[k][1] <= thresherr for k in xrange(nn)] return perr
def comparetwoalis(params1, params2, thresherr=1.0, radius = 1.0): # Find errors per image nn = len(params1) perr = 0 for k in xrange(nn): if(max_3D_pixel_error(params1[k], params2[k], r=radius) < thresherr): perr += 1 return perr/float(nn)*100.0
def proj_ali_incore_local(data, refrings, numr, xrng, yrng, step, an, finfo=None): from utilities import compose_transform2 #from utilities import set_params_proj, get_params_proj from math import cos, sin, pi ID = data.get_attr("ID") mode = "F" nx = data.get_xsize() ny = data.get_ysize() # center is in SPIDER convention cnx = nx//2 + 1 cny = ny//2 + 1 ant = cos(an*pi/180.0) #phi, theta, psi, sxo, syo = get_params_proj(data) t1 = data.get_attr("xform.projection") dp = t1.get_params("spider") # get translations from data tx = dp["tx"] ty = dp["ty"] if finfo: finfo.write("Image id: %6d\n"%(ID)) finfo.write("Old parameters: %9.4f %9.4f %9.4f %9.4f %9.4f\n"%(dp["phi"], dp["theta"], dp["psi"], -tx, -ty)) finfo.flush() [ang, sxs, sys, mirror, iref, peak] = Util.multiref_polar_ali_2d_local(data, refrings, xrng, yrng, step, ant, mode, numr, cnx+tx, cny+ty) iref=int(iref) data.set_attr("assign",iref) if iref > -1: # The ormqip returns parameters such that the transformation is applied first, the mirror operation second. # What that means is that one has to change the the Eulerian angles so they point into mirrored direction: phi+180, 180-theta, 180-psi angb, sxb, syb, ct = compose_transform2(0.0, sxs, sys, 1, -ang, 0.0, 0.0, 1) if mirror: phi = (refrings[iref].get_attr("phi")+540.0)%360.0 theta = 180.0-refrings[iref].get_attr("theta") psi = (540.0-refrings[iref].get_attr("psi")+angb)%360.0 s2x = sxb - tx s2y = syb - ty else: phi = refrings[iref].get_attr("phi") theta = refrings[iref].get_attr("theta") psi = (refrings[iref].get_attr("psi")+angb+360.0)%360.0 s2x = sxb - tx s2y = syb - ty t2 = Transform({"type":"spider","phi":phi,"theta":theta,"psi":psi}) t2.set_trans(Vec2f(-s2x, -s2y)) from pixel_error import max_3D_pixel_error pixel_error = max_3D_pixel_error(t1, t2, numr[-3]) if finfo: finfo.write( "New parameters: %9.4f %9.4f %9.4f %9.4f %9.4f %10.5f %11.3e\n\n" %(phi, theta, psi, s2x, s2y, peak, pixel_error)) finfo.flush() return t2, peak, pixel_error else: return -1.0e23, 0.0
def errors_per_image_original(nn, qt, asi, avgtrans, thresherr=1.0, radius=1.0): #print "errors per image" # We have average parameters and transformed parameters # Find errors per image # Find outliers, remove them, repeat calculations perr = [None] * nn for k in xrange(nn): lin = k // (nn // 4) # fifi = [] for j in xrange(6): if (asi[lin][j] != -10): #print " fff ",k,lin,j,asi[lin][j],qt[j][k] fifi.append(qt[j][k]) assert (len(fifi) == 3) nas = [0.0, 0.0, 0.0] r1, r2, r3 = getfvec(avgtrans[k][0], avgtrans[k][1]) sacos = 0.0 sder = 0.0 ser3 = 0.0 for i in xrange(3): #print i,fifi[i] d = max_3D_pixel_error(fifi[i], avgtrans[k], r=radius) #if(d>10.): print " LARGE ERROR",k,i,d,fifi[i], avgtrans[k] ser3 += d n1, n2, n3 = getfvec(fifi[i][0], fifi[i][1]) sacos += lacos(n1 * r1 + n2 * r2 + n3 * r3) sder += pixel_error_2D(fifi[i][2:], avgtrans[k][2:], r=1.0) # average deviation in radians sacos /= 3.0 sare = tan(sacos) sder /= 3.0 ser3 /= 3 #print k, ser3, sare, sder perr[k] = [k, ser3, sacos, sare, sder] #write_text_row(perr, 'per.txt') perr = [perr[k][1] <= thresherr for k in xrange(nn)] return perr
def errors_per_image_original(nn, qt, asi, avgtrans, thresherr=1.0, radius = 1.0): #print "errors per image" # We have average parameters and transformed parameters # Find errors per image # Find outliers, remove them, repeat calculations perr = [None]*nn for k in xrange(nn): lin = k//(nn//4) # fifi = [] for j in xrange(6): if(asi[lin][j] != -10): #print " fff ",k,lin,j,asi[lin][j],qt[j][k] fifi.append(qt[j][k]) assert(len(fifi) == 3) nas = [0.0,0.0,0.0] r1,r2,r3 = getfvec(avgtrans[k][0],avgtrans[k][1]) sacos = 0.0 sder = 0.0 ser3 = 0.0 for i in xrange(3): #print i,fifi[i] d = max_3D_pixel_error(fifi[i], avgtrans[k], r=radius) #if(d>10.): print " LARGE ERROR",k,i,d,fifi[i], avgtrans[k] ser3 += d n1,n2,n3 = getfvec(fifi[i][0],fifi[i][1]) sacos += acos(min(1.0,n1*r1+n2*r2+n3*r3)) sder += pixel_error_2D(fifi[i][2:], avgtrans[k][2:], r = 1.0) # average deviation in radians sacos /= 3.0 sare = tan(sacos) sder /= 3.0 ser3 /= 3 #print k, ser3, sare, sder perr[k] = [k, ser3, sacos, sare, sder] #write_text_row(perr, 'per.txt') perr = [perr[k][1] <= thresherr for k in xrange(nn)] return perr
def main(): from utilities import write_text_row, drop_image, model_gauss_noise, get_im, set_params_proj, wrap_mpi_bcast, model_circle import user_functions from applications import MPI_start_end from optparse import OptionParser from global_def import SPARXVERSION from EMAN2 import EMData from multi_shc import multi_shc, do_volume from logger import Logger, BaseLogger_Files import sys import os import time import socket progname = os.path.basename(sys.argv[0]) usage = progname + " stack [output_directory] initial_volume --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --center=center_type --fl --aa --ref_a=S --sym=c1" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--ir", type= "int", default= 1, help="inner radius for rotational correlation > 0 (set to 1)") parser.add_option("--ou", type= "int", default= -1, help="outer radius for rotational correlation < int(nx/2)-1 (set to the radius of the particle)") parser.add_option("--rs", type= "int", default= 1, help="step between rings in rotational correlation >0 (set to 1)" ) parser.add_option("--xr", type="string", default= "-1", help="range for translation search in x direction, search is +/xr (default 0)") parser.add_option("--yr", type="string", default= "-1", help="range for translation search in y direction, search is +/yr (default = same as xr)") parser.add_option("--ts", type="string", default= "1", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional") parser.add_option("--delta", type="string", default= "-1", help="angular step of reference projections during initialization step (default automatically selected based on radius of the structure.)") #parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches (phi and theta)") parser.add_option("--center", type="float", default= -1, help="-1: average shift method; 0: no centering; 1: center of gravity (default=-1)") parser.add_option("--maxit", type="int", default= 400, help="maximum number of iterations performed for the GA part (set to 400) ") parser.add_option("--outlier_percentile",type="float", default= 95, help="percentile above which outliers are removed every iteration") parser.add_option("--iteration_start",type="int", default= 0, help="starting iteration for rviper, 0 means go to the most recent one (default).") parser.add_option("--CTF", action="store_true", default=False, help="Use CTF (Default no CTF correction)") parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data (default 1.0)") parser.add_option("--ref_a", type="string", default= "S", help="method for generating the quasi-uniformly distributed projection directions (default S)") parser.add_option("--sym", type="string", default= "c1", help="symmetry of the refined structure") parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default=2)") parser.add_option("--startangles", action="store_true", default=False, help="Use orientation parameters in the input file header to jumpstart the procedure") #options introduced for the do_volume function parser.add_option("--fl", type="float", default=0.12, help="cut-off frequency of hyperbolic tangent low-pass Fourier filte (default 0.12)") parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass Fourier filter (default 0.1)") parser.add_option("--pwreference", type="string", default="", help="text file with a reference power spectrum (default no power spectrum adjustment)") parser.add_option("--mask3D", type="string", default=None, help="3D mask file (default a sphere WHAT RADIUS??)") (options, args) = parser.parse_args(sys.argv[1:]) #print( " args ",args) if( len(args) == 3): volinit = args[2] masterdir = args[1] elif(len(args) == 2): volinit = args[1] masterdir = "" else: print( "usage: " + usage) print( "Please run '" + progname + " -h' for detailed options") return 1 orgstack = args[0] #print( orgstack,masterdir,volinit ) # INPUT PARAMETERS radi = options.ou global_def.BATCH = True ali3d_options.ir = options.ir ali3d_options.rs = options.rs ali3d_options.ou = options.ou ali3d_options.xr = options.xr ali3d_options.yr = options.yr ali3d_options.ts = options.ts ali3d_options.an = "-1" ali3d_options.sym = options.sym ali3d_options.delta = options.delta ali3d_options.npad = options.npad ali3d_options.center = options.center ali3d_options.CTF = options.CTF ali3d_options.ref_a = options.ref_a ali3d_options.snr = options.snr ali3d_options.mask3D = options.mask3D ali3d_options.pwreference = options.pwreference ali3d_options.fl = 0.4 ali3d_options.aa = 0.1 if( ali3d_options.xr == "-1" ): ali3d_options.xr = "2" """ print( options) print( 'ali3d_options', ali3d_options.ir ,\ ali3d_options.rs ,\ ali3d_options.ou ,\ ali3d_options.xr ,\ ali3d_options.yr ,\ ali3d_options.ts ,\ ali3d_options.an ,\ ali3d_options.sym ,\ ali3d_options.delta ,\ ali3d_options.npad ,\ ali3d_options.center ,\ ali3d_options.CTF ,\ ali3d_options.ref_a ,\ ali3d_options.snr ,\ ali3d_options.mask3D ,\ ali3d_options.fl ,\ ali3d_options.aa \ ) #exit() """ mpi_init(0, []) nproc = mpi_comm_size(MPI_COMM_WORLD) myid = mpi_comm_rank(MPI_COMM_WORLD) main_node = 0 #mpi_finalize() #exit() nxinit = -1 #int(280*0.3*2) nsoft = 0 mempernode = 4.0e9 # PARAMETERS OF THE PROCEDURE # threshold error thresherr = 0 fq = 0.11 # low-freq limit to which fuse ref volumes. Should it be estimated somehow? # Get the pixel size, if none set to 1.0, and the original image size if(myid == main_node): a = get_im(orgstack) nnxo = a.get_xsize() if ali3d_options.CTF: i = a.get_attr('ctf') pixel_size = i.apix else: pixel_size = 1.0 del a else: nnxo = 0 pixel_size = 1.0 pixel_size = bcast_number_to_all(pixel_size, source_node = main_node) nnxo = bcast_number_to_all(nnxo, source_node = main_node) if(radi < 1): radi = nnxo//2-2 elif((2*radi+2)>nnxo): ERROR("HERE","particle radius set too large!",1) ali3d_options.ou = radi if(nxinit < 0): nxinit = min(32, nnxo) nxshrink = nxinit minshrink = 32.0/float(nnxo) shrink = max(float(nxshrink)/float(nnxo),minshrink) # MASTER DIRECTORY if(myid == main_node): print( " masterdir ",masterdir) if( masterdir == ""): timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime()) masterdir = "master"+timestring li = len(masterdir) cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) keepchecking = 0 else: li = 0 keepchecking = 1 else: li = 0 keepchecking = 1 li = mpi_bcast(li,1,MPI_INT,main_node,MPI_COMM_WORLD)[0] if( li > 0 ): masterdir = mpi_bcast(masterdir,li,MPI_CHAR,main_node,MPI_COMM_WORLD) masterdir = string.join(masterdir,"") # create a vstack from input stack to the local stack in masterdir # Stack name set to default stack = "bdb:"+masterdir+"/rdata" # Initialization of stacks if(myid == main_node): if keepchecking: if(os.path.exists(os.path.join(masterdir,"EMAN2DB/rdata.bdb"))): doit = False else: doit = True else: doit = True if doit: if(orgstack[:4] == "bdb:"): cmd = "{} {} {}".format("e2bdb.py", orgstack,"--makevstack="+stack) else: cmd = "{} {} {}".format("sxcpy.py", orgstack, stack) cmdexecute(cmd) cmd = "{} {}".format("sxheader.py --consecutive --params=originalid", stack) cmdexecute(cmd) keepchecking = False total_stack = EMUtil.get_image_count(stack) junk = get_im(stack) nnxo = junk.get_xsize() del junk else: total_stack = 0 nnxo = 0 total_stack = bcast_number_to_all(total_stack, source_node = main_node) nnxo = bcast_number_to_all(nnxo, source_node = main_node) # INITIALIZATION # Run exhaustive projection matching to get initial orientation parameters # Estimate initial resolution initdir = os.path.join(masterdir,"main000") # make sure the initial volume is not set to zero outside of a mask, as if it is it will crach the program if( myid == main_node and (not options.startangles)): viv = get_im(volinit) if(options.mask3D == None): mask33d = model_circle(radi,nnxo,nnxo,nnxo) else: mask33d = (options.mask3D).copy() st = Util.infomask(viv, mask33d, False) if( st[0] == 0.0 ): viv += (model_blank(nnxo,nnxo,nnxo,1.0) - mask33d)*model_gauss_noise(st[1]/1000.0,nnxo,nnxo,nnxo) viv.write_image(volinit) del mask33d, viv doit, keepchecking = checkstep(initdir, keepchecking, myid, main_node) if doit: partids = os.path.join(masterdir, "ids.txt") partstack = os.path.join(masterdir, "paramszero.txt") xr = min(8,(nnxo - (2*radi+1))//2) if(xr > 3): ts = "2" else: ts = "1" delta = int(options.delta) if(delta <= 0.0): delta = "%f"%round(degrees(atan(1.0/float(radi))), 2) paramsdict = { "stack":stack,"delta":"2.0", "ts":ts, "xr":"%f"%xr, "an":"-1", "center":options.center, "maxit":1, \ "currentres":0.4, "aa":0.1, "radius":radi, "nsoft":0, "delpreviousmax":True, "shrink":1.0, "saturatecrit":1.0, \ "refvol":volinit, "mask3D":options.mask3D} if(options.startangles): if( myid == main_node ): cmd = "mkdir "+initdir cmdexecute(cmd) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print(line,"INITIALIZATION") cmd = "{} {}".format("sxheader.py --params=xform.projection --export="+os.path.join(initdir,"params-chunk0.txt"), stack) cmdexecute(cmd) print(line,"Executed successfully: ","Imported initial parameters from the input stack") else: if( myid == main_node ): line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print(line,"INITIALIZATION") write_text_file(range(total_stack), partids) write_text_row([[0.0,0.0,0.0,0.0,0.0] for i in xrange(total_stack) ], partstack) metamove(paramsdict, partids, partstack, initdir, 0, myid, main_node, nproc) if(myid == main_node): print(line,"Executed successfully: ","initialization ali3d_base_MPI %d"%nsoft) # store params partids = [None]*2 for procid in xrange(2): partids[procid] = os.path.join(initdir,"chunk%01d.txt"%procid) partstack = [None]*2 for procid in xrange(2): partstack[procid] = os.path.join(initdir,"params-chunk%01d.txt"%procid) from random import shuffle if(myid == main_node): # split randomly params = read_text_row(os.path.join(initdir,"params-chunk0.txt")) assert(len(params) == total_stack) ll = range(total_stack) shuffle(ll) l1 = ll[:total_stack//2] l2 = ll[total_stack//2:] del ll l1.sort() l2.sort() write_text_file(l1,partids[0]) write_text_file(l2,partids[1]) write_text_row([params[i] for i in l1], partstack[0]) write_text_row([params[i] for i in l2], partstack[1]) del params, l1, l2 mpi_barrier(MPI_COMM_WORLD) # Now parallel vol = [None]*2 for procid in xrange(2): projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc) if ali3d_options.CTF: vol[procid] = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) else: vol[procid] = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) del projdata if( myid == main_node): vol[procid].write_image(os.path.join(initdir,"vol%01d.hdf"%procid) ) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print( line,"Generated inivol #%01d "%procid) if(myid == main_node): currentres = get_resolution(vol, radi, nnxo, initdir) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print( line,"Initial resolution %6.4f"%currentres) write_text_file([currentres],os.path.join(initdir,"current_resolution.txt")) else: currentres = 0.0 currentres = bcast_number_to_all(currentres, source_node = main_node) else: if(myid == main_node): currentres = read_text_file(os.path.join(initdir,"current_resolution.txt"))[0] else: currentres = 0.0 currentres = bcast_number_to_all(currentres, source_node = main_node) # set for the first iteration nxshrink = min(max(32, int((currentres+paramsdict["aa"]/2.)*2*nnxo + 0.5)), nnxo) shrink = float(nxshrink)/nnxo tracker = {"previous-resolution":currentres, "movedup":False,"eliminated-outliers":False,\ "previous-nx":nxshrink, "previous-shrink":shrink, "extension":0, "bestsolution":0} previousoutputdir = initdir # MAIN ITERATION mainiteration = 0 keepgoing = 1 while(keepgoing): mainiteration += 1 # prepare output directory mainoutputdir = os.path.join(masterdir,"main%03d"%mainiteration) if(myid == main_node): line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print(line,"MAIN ITERATION #",mainiteration, shrink, nxshrink) if keepchecking: if(os.path.exists(mainoutputdir)): doit = 0 print("Directory ",mainoutputdir," exists!") else: doit = 1 keepchecking = False else: doit = 1 if doit: cmd = "{} {}".format("mkdir", mainoutputdir) cmdexecute(cmd) # prepare names of input file names, they are in main directory, # log subdirectories contain outputs from specific refinements partids = [None]*2 for procid in xrange(2): partids[procid] = os.path.join(previousoutputdir,"chunk%01d.txt"%procid) partstack = [None]*2 for procid in xrange(2): partstack[procid] = os.path.join(previousoutputdir,"params-chunk%01d.txt"%procid) mpi_barrier(MPI_COMM_WORLD) #mpi_finalize() #exit() #print("RACING A ",myid) outvol = [os.path.join(previousoutputdir,"vol%01d.hdf"%procid) for procid in xrange(2)] for procid in xrange(2): doit, keepchecking = checkstep(outvol[procid], keepchecking, myid, main_node) if doit: from multi_shc import do_volume projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc) if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) del projdata if( myid == main_node): vol.write_image(outvol[procid]) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print( line,"Generated inivol #%01d "%procid) del vol if(myid == main_node): if keepchecking: procid = 1 if(os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid)): doit = 0 else: doit = 1 keepchecking = False else: doit = 1 if doit: vol = [get_im(outvol[procid]) for procid in xrange(2) ] fq = 0.11 # which part to fuse fuselowf(vol, fq) for procid in xrange(2): vol[procid].write_image(os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid) ) del vol else: doit = 0 mpi_barrier(MPI_COMM_WORLD) doit = bcast_number_to_all(doit, source_node = main_node) # Refine two groups at a current resolution lastring = int(shrink*radi + 0.5) if(lastring < 2): print( line,"ERROR!! lastring too small ", radi, shrink, lastring) break # REFINEMENT # Part "a" SHC for procid in xrange(2): coutdir = os.path.join(mainoutputdir,"loga%01d"%procid) doit, keepchecking = checkstep(coutdir , keepchecking, myid, main_node) paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":1500, \ "currentres":currentres, "aa":0.1, "radius":radi, "nsoft":1, "saturatecrit":0.75, "delpreviousmax":True, "shrink":shrink, \ "refvol":os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid),"mask3D":options.mask3D } if doit: metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc) partstack = [None]*2 for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir, "loga%01d"%procid, "params-chunk%01d.txt"%procid) for procid in xrange(2): outvol = os.path.join(mainoutputdir,"loga%01d"%procid,"shcvol%01d.hdf"%procid) doit, keepchecking = checkstep(outvol, keepchecking, myid, main_node) if doit: from multi_shc import do_volume projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc) if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) del projdata if( myid == main_node): vol.write_image(outvol) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print( line,"Generated shcvol #%01d "%procid) del vol if(myid == main_node): if keepchecking: procid = 1 if(os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid) ): doit = 0 else: doit = 1 keepchecking = False else: doit = 1 if doit: vol = [] for procid in xrange(2): vol.append(get_im(os.path.join(mainoutputdir,"loga%01d"%procid,"shcvol%01d.hdf"%procid) )) fq = 0.11 # which part to fuse fuselowf(vol, fq) for procid in xrange(2): vol[procid].write_image( os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid) ) del vol else: doit = 0 mpi_barrier(MPI_COMM_WORLD) doit = bcast_number_to_all(doit, source_node = main_node) # Part "b" deterministic partstack = [None]*2 for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir,"loga%01d"%procid,"params-chunk%01d.txt"%procid) for procid in xrange(2): coutdir = os.path.join(mainoutputdir,"logb%01d"%procid) doit, keepchecking = checkstep(coutdir, keepchecking, myid, main_node) # Run exhaustive to finish up matching paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":10, \ "currentres":currentres, "aa":0.1, "radius":radi, "nsoft":0, "saturatecrit":0.95, "delpreviousmax":True, "shrink":shrink, \ "refvol":os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid), "mask3D":options.mask3D } if doit: metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc) partstack = [None]*2 for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir,"logb%01d"%procid,"params-chunk%01d.txt"%procid) # Compute current resolution, store result in main directory doit, keepchecking = checkstep(os.path.join(mainoutputdir,"current_resolution.txt"), keepchecking, myid, main_node) newres = 0.0 if doit: newres = compute_resolution(stack, mainoutputdir, partids, partstack, radi, nnxo, ali3d_options.CTF, myid, main_node, nproc) else: if(myid == main_node): newres = read_text_file( os.path.join(mainoutputdir,"current_resolution.txt") )[0] newres = bcast_number_to_all(newres, source_node = main_node) # Here I have code to generate presentable results. IDs and params have to be merged and stored and an overall volume computed. doit, keepchecking = checkstep(os.path.join(mainoutputdir,"volf.hdf"), keepchecking, myid, main_node) if doit: if( myid == main_node ): pinids = map(int, read_text_file(partids[0]) ) + map(int, read_text_file(partids[1]) ) params = read_text_row(partstack[0]) + read_text_row(partstack[1]) assert(len(pinids) == len(params)) for i in xrange(len(pinids)): pinids[i] = [ pinids[i], params[i] ] del params pinids.sort() write_text_file([pinids[i][0] for i in xrange(len(pinids))], os.path.join(mainoutputdir,"indexes.txt")) write_text_row( [pinids[i][1] for i in xrange(len(pinids))], os.path.join(mainoutputdir,"params.txt")) mpi_barrier(MPI_COMM_WORLD) ali3d_options.fl = newres ali3d_options.ou = radi projdata = getindexdata(stack, os.path.join(mainoutputdir,"indexes.txt"), os.path.join(mainoutputdir,"params.txt"), myid, nproc) volf = do_volume(projdata, ali3d_options, mainiteration, mpi_comm = MPI_COMM_WORLD) if(myid == main_node): volf.write_image(os.path.join(mainoutputdir,"volf.hdf")) mpi_barrier(MPI_COMM_WORLD) #print("RACING X ",myid) if(newres == currentres): for procid in xrange(2): coutdir = os.path.join(mainoutputdir,"logc%01d"%procid) doit, keepchecking = checkstep(coutdir, keepchecking, myid, main_node) if doit: # Do cross-check of the results paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":1, \ "currentres":newres, "aa":0.1, "radius":radi, "nsoft":0, "saturatecrit":0.95, "delpreviousmax":True, "shrink":shrink, \ "refvol":os.path.join(mainoutputdir,"vol%01d.hdf"%(1-procid)), "mask3D":options.mask3D } # The cross-check uses parameters from step "b" to make sure shifts are correct. # As the check is exhaustive, angles are ignored metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc) # identify bad apples doit, keepchecking = checkstep(os.path.join(mainoutputdir,"badapples.txt"), keepchecking, myid, main_node) if doit: if(myid == main_node): from utilities import get_symt from pixel_error import max_3D_pixel_error ts = get_symt(ali3d_options.sym) badapples = [] deltaerror = 2.0 total_images_now = 0 for procid in xrange(2): bad = [] ids = map(int,read_text_file( partids[procid] )) total_images_now += len(ids) oldp = read_text_row(partstack[procid]) newp = read_text_row(os.path.join(mainoutputdir,"logc%01d"%procid,"params-chunk%01d.txt"%procid)) for i in xrange(len(ids)): t1 = Transform({"type":"spider","phi":oldp[i][0],"theta":oldp[i][1],"psi":oldp[i][2]}) t1.set_trans(Vec2f(-oldp[i][3]*shrink, -oldp[i][4]*shrink)) t2 = Transform({"type":"spider","phi":newp[i][0],"theta":newp[i][1],"psi":newp[i][2]}) t2.set_trans(Vec2f(-newp[i][3]*shrink, -newp[i][4]*shrink)) if(len(ts) > 1): # only do it if it is not c1 pixel_error = +1.0e23 for kts in ts: ut = t2*kts # we do not care which position minimizes the error pixel_error = min(max_3D_pixel_error(t1, ut, lastring), pixel_error) else: pixel_error = max_3D_pixel_error(t1, t2, lastring) if(pixel_error > deltaerror): bad.append(i) if(len(bad)>0): badapples += [ids[bad[i]] for i in xrange(len(bad))] for i in xrange(len(bad)-1,-1,-1): del oldp[bad[i]],ids[bad[i]] if(len(ids) == 0): ERROR("sxpetite","program diverged, all images have large angular errors, most likely the initial model is badly off",1) else: # This generate new parameters, hopefully to be used as starting ones in the new iteration write_text_file(ids,os.path.join(mainoutputdir,"chunk%01d.txt"%procid)) write_text_row(oldp,os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid)) if(len(badapples)>0): badapples.sort() write_text_file(badapples,os.path.join(mainoutputdir,"badapples.txt")) eli = 100*float(len(badapples))/float(total_images_now) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print(line,"Elimination of outliers: %5.1f percent"%eli ) else: eli = 0.0 del badapples, oldp,ids,bad,newp,ts else: eli =0.0 eli = bcast_number_to_all(eli, source_node = main_node) # This part under MPI if(eli > 0.0): # Compute current resolution depres = compute_resolution(stack, mainoutputdir, \ [os.path.join(mainoutputdir,"chunk%01d.txt"%procid) for procid in xrange(2)], \ [os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid) for procid in xrange(2)], \ radi, nnxo, ali3d_options.CTF, myid, main_node, nproc) depres = bcast_number_to_all(depres, source_node = main_node) if(depres < newres): # elimination of outliers decreased resolution, ignore the effort eliminated_outliers = False else: eliminated_outliers = True newres = depres """ # It does not seem to be needed, as data is there, we just point to the directory for procid in xrange(2): # set pointers to current parameters in main, which are for the reduced set stored above partids[procid] = os.path.join(mainoutputdir,"chunk%01d.txt"%procid partstack[procid] = os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid) """ else: eliminated_outliers = False else: eliminated_outliers = False if(myid == main_node and not eliminated_outliers): for procid in xrange(2): # This is standard path, copy parameters to be used to the main cmd = "{} {} {}".format("cp -p ", partids[procid] , os.path.join(mainoutputdir,"chunk%01d.txt"%procid)) cmdexecute(cmd) cmd = "{} {} {}".format("cp -p ", partstack[procid], os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid)) cmdexecute(cmd) keepgoing = 0 if( newres > currentres or (eliminated_outliers and not tracker["eliminated-outliers"])): if(myid == main_node): print(" Resolution improved, full steam ahead!") if( newres > currentres ): tracker["movedup"] = True else: tracker["movedup"] = False shrink = max(min(2*newres + paramsdict["aa"], 1.0),minshrink) tracker["extension"] = 4 nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo) tracker["previous-resolution"] = newres currentres = newres tracker["bestsolution"] = mainiteration bestoutputdir = mainoutputdir tracker["eliminated-outliers"] = eliminated_outliers keepgoing = 1 elif(newres < currentres): if(not tracker["movedup"] and tracker["extension"] < 2 and mainiteration > 1): keepgoing = 0 if(myid == main_node): print(" Cannot improve resolution, the best result is in the directory main%03d"%tracker["bestsolution"]) else: if(not tracker["movedup"] and tracker["extension"] > 1 and mainiteration > 1): if(myid == main_node): print(" Resolution decreased. Will decrease target resolution and will fall back on the best so far: main%03d"%tracker["bestsolution"]) bestoutputdir = os.path.join(masterdir,"main%03d"%tracker["bestsolution"]) elif( tracker["movedup"] and tracker["extension"] > 1 and mainiteration > 1): if(myid == main_node): print(" Resolution decreased. Will decrease target resolution and will try starting from previous stage: main%03d"%(mainiteration - 1)) bestoutputdir = os.path.join(masterdir,"main%03d"%(mainiteration-1)) elif( mainiteration == 1): if(myid == main_node): print(" Resolution decreased in the first iteration. It is expected, not to worry") bestoutputdir = mainoutputdir tracker["extension"] += 1 else: # missing something here? if(myid == main_node): print(" Should not be here, ERROR 175!") break mpi_finalize() exit() if( bestoutputdir != mainoutputdir ): # This is the key, we just reset the main to previous, so it will be eventually used as a starting in the next iteration mainoutputdir = bestoutputdir """ # Set data from the main previous best to the current. for procid in xrange(2): partids[procid] = os.path.join(bestoutputdir,"chunk%01d.txt"%procid) partstack[procid] = os.path.join(bestoutputdir,"params-chunk%01d.txt"%procid) """ if(myid == main_node): currentres = read_text_file( os.path.join(bestoutputdir,"current_resolution.txt") )[0] currentres = bcast_number_to_all(currentres, source_node = main_node) shrink = max(min(2*currentres + paramsdict["aa"], 1.0), minshrink) tracker["extension"] -= 1 nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo) tracker["previous-resolution"] = newres tracker["eliminated-outliers"] = eliminated_outliers tracker["movedup"] = False keepgoing = 1 elif(newres == currentres): if( tracker["extension"] > 0 ): if(myid == main_node): print("The resolution did not improve. This is look ahead move. Let's try to relax slightly and hope for the best") tracker["extension"] -= 1 tracker["movedup"] = False shrink = max(min(2*currentres + paramsdict["aa"], 1.0), minshrink) nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo) if( tracker["previous-nx"] == nnxo ): keepgoing = 0 else: tracker["previous-resolution"] = newres currentres = newres tracker["eliminated-outliers"] = eliminated_outliers tracker["movedup"] = False keepgoing = 1 else: if(myid == main_node): print("The resolution did not improve.") keepgoing = 0 if( keepgoing == 1 ): if(myid == main_node): print(" New shrink and image dimension :",shrink,nxshrink) """ # It does not look like it is necessary, we just have to point to the directory as the files should be there. # Will continue, so update the params files for procid in xrange(2): # partids ads partstack contain parameters to be used as starting in the next iteration if(not os.path.exists(os.path.join(mainoutputdir,"chunk%01d.txt"%procid))): cmd = "{} {} {}".format("cp -p ", partids[procid] , os.path.join(mainoutputdir,"chunk%01d.txt"%procid)) cmdexecute(cmd) if(not os.path.exists(os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))): cmd = "{} {} {}".format("cp -p ", partstack[procid], os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid)) cmdexecute(cmd) """ previousoutputdir = mainoutputdir tracker["previous-shrink"] = shrink tracker["previous-nx"] = nxshrink else: if(myid == main_node): print(" Terminating, the best solution is in the directory main%03d"%tracker["bestsolution"]) mpi_barrier(MPI_COMM_WORLD) mpi_finalize()
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " stack outdir --phase=1 --ou=outer_radius|sxconsistency.py --phase=3 newlocal/main000 --ou=133 --thresherr=3.0 --params=paramsa outgrouparms" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--phase", type="int", default=1, help= "Phase =1 prepares resampled stacks, =2 analyzes consistency of orientation parameters" ) parser.add_option("--ou", type="int", default=-1, help="outer radius for calculation of pixel error") parser.add_option("--sym", type="string", default="c1", help="symmetry of the refined structure") parser.add_option( "--thresherr", type="float", default=1.0, help="Threshold for accpetable orientation errors (in pixels)") parser.add_option( "--ndigits", type="int", default=1, help="Accuracy for checking whether parameters are identical") parser.add_option("--chunk", type="string", default="", help="Root of of four chunk files with indeces") parser.add_option( "--params", type="string", default="", help="Root of of six parameter file names with refinement results") (options, args) = parser.parse_args(arglist[1:]) global_def.BATCH = True if options.phase == 1 and len(args) == 2: inputbdb = args[0] outdir = args[1] nn = EMUtil.get_image_count(inputbdb) t = range(nn) shuffle(t) chunks = [] for i in xrange(4): # I use the MPI function here just to easily get the balanced load j, k = MPI_start_end(nn, 4, i) chunks.append(t[j:k]) chunks[i].sort() write_text_file(chunks[i], os.path.join(outdir, 'chunk%01d.txt' % i)) del t write_text_file([len(chunks[i]) for i in xrange(4)], os.path.join(outdir, 'chunklengths.txt')) """ pt = [[None]]*6 ll=0 for i in xrange(3): for j in xrange(i+1,4): pt[ll] = chunks[i]+chunks[j] ll+=1 for i in xrange(6): listfile = os.path.join(outdir,'lili%01d.txt'%i) write_text_file(pt[i],listfile) outbdb = "bdb:"+ os.path.join(outdir,"X%01d"%i) cmd = '{} {} {} {}'.format('e2bdb.py', inputbdb, '--makevstack='+outbdb, '--list='+listfile) subprocess.call(cmd, shell=True) """ # Run 6 programs elif options.phase == 2 and len(args) == 2: outdir = args[0] howmanythesame = args[1] ndigits = options.ndigits #for chc5 1, for ribo 4#4.0 prms = [] for i in xrange(6): prms.append( read_text_row( os.path.join(outdir, options.params + "%01d.txt" % i))) for j in xrange(len(prms[-1])): for k in xrange(5): prms[-1][j][k] = round(prms[-1][j][k], ndigits) nn = 2 * len(prms[0]) n4 = nn // 4 qt = [[[-1.0]] * nn for i in xrange(6)] ll = 0 for i in xrange(3): for j in xrange(i + 1, 4): qt[ll][i * n4:(i + 1) * n4] = prms[ll][:n4] qt[ll][j * n4:(j + 1) * n4] = prms[ll][n4:] ll += 1 thesame = 0 for ll in xrange(len(qt[0])): rw = [] for j in xrange(6): if (len(qt[j][ll]) > 1): rw.append(qt[j][ll]) isame = True for j in xrange(3): if (rw[0][j] != rw[1][j]): isame = False #print ll,rw[0][j], rw[1][j] break if (rw[0][j] != rw[2][j]): isame = False #print ll,rw[0][j], rw[2][j] break if isame: thesame += 1 qt = float(thesame) / nn print("Proportion of the same orientations ", qt) write_text_file([qt], os.path.join(outdir, howmanythesame)) ######### PHASE 3 elif options.phase == 3 and len(args) == 2: outdir = args[0] outgrouparms = args[1] radius = options.ou thresherr = options.thresherr #for chc5 1, for ribo 4#4.0 sym = int(options.sym[1:]) qsym = 360.0 / sym blocks = ['A', 'B', 'C', 'D'] params = {} ll = 0 for i in xrange(3): for j in xrange(i + 1, 4): params[chr(65 + i) + chr(48 + ll)] = [] params[chr(65 + j) + chr(48 + ll)] = [] ll += 1 chunks = {} chunklengths = {} for i in xrange(4): chunks[chr(65 + i)] = map( int, read_text_file( os.path.join(outdir, options.chunk + "%01d.txt" % i))) chunklengths[chr(65 + i)] = len(chunks[chr(65 + i)]) for i in xrange(6): prms = read_text_row( os.path.join(outdir, options.params + "%01d.txt" % i)) for q in blocks: if q + chr(48 + i) in params: params[q + chr(48 + i)] = prms[:chunklengths[q]] del prms[:chunklengths[q]] pairs = [["A0", "A1"], ["B3", "B4"], ["C1", "C5"], ["D2", "D4"]] lefts = ["A2", "B0", "C3", "D5"] # Compute average projection params and pixel errors avgtrans = {} pixer = {} for i, q in enumerate(pairs): avgtrans[q[0][0]] = [0.0] * chunklengths[q[0][0]] pixer[q[0][0]] = [0.0] * chunklengths[q[0][0]] for j in xrange(chunklengths[q[0][0]]): fifi = [params[q[0]][j], params[q[1]][j]] nas = [0.0, 0.0, 0.0] if (sym == 1): pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in xrange(2): n1, n2, n3 = getfvec(fifi[i][0], fifi[i][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1, m2, m3 = getfvec(fifi[0][0], fifi[0][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 #if(k == 2): # print "XXXX" # print fifi[0],nas for i in xrange(1, 2): qnom = -1.e10 for j in xrange(-1, 2, 1): t1, t2, t3 = getfvec(fifi[i][0] + j * qsym, fifi[i][1]) nom = t1 * m1 + t2 * m2 + t3 * m3 if (nom > qnom): qnom = nom n1 = t1 n2 = t2 n3 = t3 #if(k == 2): # print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 print(qnom, n1, n2, n3, nas) # To get the correct pixer phi angle has to be taken from the above!! pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if (nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2] / nom)) % 360.0 if (sym > 1 and ntheta > 90.0): nphi = (degrees(atan2(nas[1], nas[0])) - 180.0) % qsym + 180.0 else: nphi = degrees(atan2(nas[1], nas[0])) % qsym #print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform([fifi[0][2:], fifi[1][2:]]) avgtrans[q[0][0]][j] = [ nphi, ntheta, twod[0], twod[1], twod[2] ] perr = {} tgood = 0 for q in blocks: perr[q] = [True] * chunklengths[q] for k in xrange(chunklengths[q]): if (pixer[q][k] > thresherr): perr[q][k] = False if perr[q][k]: tgood += 1 if (tgood < 4): print( " No good images within the pixel error threshold specified" ) exit() print(" tgood ", tgood) hi = hist_list( [pixer[q][k] for q in blocks for k in xrange(chunklengths[q])], 16) for i in xrange(len(hi[0])): print("%4d %12.3f %12.0f " % (i, hi[0][i], hi[1][i])) # Finished, store average orientation params and table of good images # store lists of good images for each group # blocks is indexed by first letter good = [[] for i in xrange(4)] bad = [[] for i in xrange(4)] for i, q in enumerate(blocks): for k in xrange(chunklengths[q]): #deprt = max_3D_pixel_error(params[lefts[i]][k],avgtrans[q][k],r=radius) if perr[q][k]: good[i].append(chunks[q][k]) #[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k] ]) else: bad[i].append(chunks[q][k]) #[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k]]) write_text_file(good[i], os.path.join(outdir, "newgood%01d.txt" % i)) write_text_file(bad[i], os.path.join(outdir, "newbad%01d.txt" % i)) # write out parameters, for those in pairs write out average, for leftouts leave them as they were # These parameters refer to the original X files. ll = 0 for m in xrange(6): if q + chr(48 + m) in params: prmsgood = [] prmsbad = [] try: j = lefts.index(q + chr(48 + m)) for k in xrange(chunklengths[q]): if perr[q][k]: prmsgood.append(params[q + chr(48 + m)][k]) else: prmsbad.append(params[q + chr(48 + m)][k]) except: for k in xrange(chunklengths[q]): if perr[q][k]: prmsgood.append(avgtrans[q][k]) else: prmsbad.append(avgtrans[q][k]) write_text_row( prmsgood, os.path.join(outdir, "params-newgood%01d%01d.txt" % (i, ll))) write_text_row( prmsbad, os.path.join(outdir, "params-newbad%01d%01d.txt" % (i, ll))) ll += 1 # Generate newlili files from newgood, these contain original numbering of the total single file ll = 0 for i in xrange(3): for j in xrange(i + 1, 4): write_text_file(good[i] + good[j], os.path.join(outdir, "newlili%01d.txt" % ll)) ll += 1 # write out parameters, for those in pairs write out average, for leftouts leave them as they were # These parameters refer to the original X files. for i in xrange(6): prms = [] for q in blocks: if q + chr(48 + i) in params: try: j = lefts.index(q + chr(48 + i)) prms += params[q + chr(48 + i)] except: prms += avgtrans[q] write_text_row(prms, os.path.join(outdir, outgrouparms + "%01d.txt" % i)) # Write chunklengths chunklengths = [len(good[i]) for i in xrange(4)] write_text_file(chunklengths, os.path.join(outdir, "chunklengths.txt")) """ # We do not use consecutive numbering anymore # Generate newx files from newgood, these contain consecutive (with gaps) numbering that allows to generate truncated X files from the previous X files ll = 0 for i in xrange(3): firstblock = [] l = len(perr[blocks[i]]) for k in xrange(l): if perr[blocks[i]][k]: firstblock.append(k) for j in xrange(i+1,4): secondblock = [] for k in xrange(len(perr[blocks[j]])): if perr[blocks[j]][k]: secondblock.append(k+l) write_text_file( firstblock + secondblock, os.path.join(outdir,"goodX%01d.txt"%ll) ) ll += 1 del good,bad,firstblock,secondblock,perr """ """ 0 1 2 3 4 5 A A A = = = B = = B B = = C = C = C = = D = D D """ elif options.phase == 4 and len(args) == 1: outdir = args[0] bp = 'badparams' #outgrouparms= args[1] radius = options.ou thresherr = options.thresherr sym = int(options.sym[1:]) qsym = 360.0 / sym #params = [[None for i in xrange(3)] for j in xrange(4)] ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general. for jj in xrange(4): params = [None for ii in xrange(ll)] newbad = map(int, read_text_file(options.params + "%01d.txt" % jj)) nn = len(newbad) for ii in xrange(ll): params[ii] = read_text_row( os.path.join(outdir, bp + "%01d%01d.txt" % (jj, ii))) assert (nn == len(params[ii])) # Compute average projection params and pixel errors avgtrans = [None] * nn pixer = [0.0] * nn for j in xrange(nn): nas = [0.0, 0.0, 0.0] if (sym == 1): #pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in xrange(ll): n1, n2, n3 = getfvec(params[i][j][0], params[i][j][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1, m2, m3 = getfvec(params[0][j][0], params[0][j][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 for i in xrange(1, ll): qnom = -1.e10 for j in xrange(-1, 2, 1): t1, t2, t3 = getfvec(params[i][j][0] + j * qsym, params[i][j][1]) nom = t1 * m1 + t2 * m2 + t3 * m3 if (nom > qnom): qnom = nom n1 = t1 n2 = t2 n3 = t3 #if(k == 2): # print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 print(qnom, n1, n2, n3, nas) # To get the correct pixer phi angle has to be taken from the above!! nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if (nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2] / nom)) % 360.0 if (sym > 1 and ntheta > 90.0): nphi = (degrees(atan2(nas[1], nas[0])) - 180.0) % qsym + 180.0 else: nphi = degrees(atan2(nas[1], nas[0])) % qsym #print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform( [params[ii][j][2:] for ii in xrange(ll)]) avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]] perr = errors_per_image(params, avgtrans, thresherr, radius) rescued = [] rejects = [] for j in xrange(nn): #print chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)] if (perr[j][0] <= thresherr): rescued.append([newbad[j], j]) else: rejects.append([newbad[j], j]) if (len(rescued) == 0): write_text_row([-1, -1], os.path.join(outdir, "rescued%01d.txt" % jj)) else: write_text_row(rescued, os.path.join(outdir, "rescued%01d.txt" % jj)) # We also have to write params. if (len(rescued) != 0): for ii in xrange(ll): write_text_row( [ params[ii][rescued[k][1]] for k in xrange(len(rescued)) ], os.path.join(outdir, "params-rescued%01d%01d.txt" % (jj, ii))) if (len(rejects) == 0): write_text_row([-1, -1], os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) else: write_text_row( rejects, os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) if (len(rejects) != 0): for ii in xrange(ll): write_text_row( [ params[ii][rejects[k][1]] for k in xrange(len(rejects)) ], os.path.join(outdir, "params-rejects%01d%01d.txt" % (jj, ii))) hi = hist_list([perr[j][0] for j in xrange(nn)], 16) print("Pixel errors for BAD GROUP ", chr(65 + jj)) for ii in xrange(len(hi[0])): print("%4d %12.3f %12.0f " % (ii, hi[0][ii], hi[1][ii])) elif options.phase == 5 and len(args) == 1: # This version is for meridien refinement. There are simply three full sets of params. outdir = args[0] bp = 'badparams' #outgrouparms= args[1] radius = options.ou thresherr = options.thresherr sym = int(options.sym[1:]) qsym = 360.0 / sym #params = [[None for i in xrange(3)] for j in xrange(4)] ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general. for jj in xrange(1): params = [None for ii in xrange(ll)] #newbad = map(int, read_text_file(options.params+"%01d.txt"%jj) ) #nn = len(newbad) for ii in xrange(ll): params[ii] = read_text_row( os.path.join(outdir, "params%01d.txt" % (ii))) #assert(nn == len(params[ii]) ) nn = len(params[0]) newbad = range(nn) # Compute average projection params and pixel errors avgtrans = [None] * nn pixer = [0.0] * nn for j in xrange(nn): nas = [0.0, 0.0, 0.0] if (sym == 1): #pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in xrange(ll): n1, n2, n3 = getfvec(params[i][j][0], params[i][j][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1, m2, m3 = getfvec(params[0][j][0], params[0][j][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 for i in xrange(1, ll): qnom = -1.e10 for j in xrange(-1, 2, 1): t1, t2, t3 = getfvec(params[i][j][0] + j * qsym, params[i][j][1]) nom = t1 * m1 + t2 * m2 + t3 * m3 if (nom > qnom): qnom = nom n1 = t1 n2 = t2 n3 = t3 #if(k == 2): # print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 print(qnom, n1, n2, n3, nas) # To get the correct pixer phi angle has to be taken from the above!! nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if (nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2] / nom)) % 360.0 if (sym > 1 and ntheta > 90.0): nphi = (degrees(atan2(nas[1], nas[0])) - 180.0) % qsym + 180.0 else: nphi = degrees(atan2(nas[1], nas[0])) % qsym #print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform( [params[ii][j][2:] for ii in xrange(ll)]) avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]] write_text_row(avgtrans, os.path.join(outdir, "avgtrans.txt")) perr = errors_per_image(params, avgtrans, thresherr, radius) rescued = [] rejects = [] for j in xrange(nn): #print chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)] if (perr[j][0] <= thresherr): rescued.append([newbad[j], j]) else: rejects.append([newbad[j], j]) if (len(rescued) == 0): write_text_row([-1, -1], os.path.join(outdir, "rescued%01d.txt" % jj)) else: write_text_row(rescued, os.path.join(outdir, "rescued%01d.txt" % jj)) # We also have to write params. if (len(rescued) != 0): for ii in xrange(ll): write_text_row( [ params[ii][rescued[k][1]] for k in xrange(len(rescued)) ], os.path.join(outdir, "params-rescued%01d%01d.txt" % (jj, ii))) if (len(rejects) == 0): write_text_row([-1, -1], os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) else: write_text_row( rejects, os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) if (len(rejects) != 0): for ii in xrange(ll): write_text_row( [ params[ii][rejects[k][1]] for k in xrange(len(rejects)) ], os.path.join(outdir, "params-rejects%01d%01d.txt" % (jj, ii))) hi = hist_list([perr[j][0] for j in xrange(nn)], 16) print("Pixel errors for BAD GROUP ", chr(65 + jj)) for ii in xrange(len(hi[0])): print("%4d %12.3f %12.0f " % (ii, hi[0][ii], hi[1][ii])) else: print("Usage: ") print(""" Phase 1: sxconsistency.py --phase=1 bdb:data outdir output files are: in directory outdir: lili0.txt to lili5.txt contain indices of images in resampled six groups bdb:dataX0 to bdb:dataX5 are metafiles containing six resampled groups of images derived from bdb:data Phase 2 sxconsistency.py --phase=2 outdir --ndigits=1 --params=paramsb howmanythesame.txt outdir - directory containing files lili0.txt to lili5.txt produced in phase 1 --params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be master/main/params0.txt to master/main/params5.txt output files: howmanythesame.txt - contains one number, a ratio of number of images that did not change orientations to the total number of images Phase 3: sxconsistency.py --phase=3 outdir --ou=133 --thresherr=3.0 --params=paramsa outgrouparms input files: outdir - directory containing files lili0.txt to lili5.txt produced in phase 1 --params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be master/main/params0.txt to master/main/params5.txt output files: outgrouparms*.txt - Root of of six parameters files with average 3D orientation parameters computed from six runs, can be imported into bdb:data The next two files contain the original image numbers refering to the top bdb. outdir/newgood.txt - Text file with indices of images whose orientation parameters arrors are below specified thresherr (to be kept) outdir/bad.txt - Text file with indices of images whose orientation parameters arrors are above specified thresherr (to be rejected) Phase 4: sxconsistency.py --phase=4 outdir --ou=133 --thresherr=3.0 --params=master/main001/newbad input files: outdir - directory containing files badparamsij.txt i= 0,3, j=0,2, which resulted from three alignments of four badchunk images --params= - Root of four files with original indices of bad images, they will be read and a subset corresponding to rescued ones will be outputed output files: rescued*.txt - four files with indices of accepted images from badchunk. rejects*.txt - four files with indices of rejected images from badchunk. There are two columns: [number in original bdb:chunk, number in bdb:newbad] """) print("Please run '" + progname + " -h' for detailed options") global_def.BATCH = False
def ali3d_MPI(stack, ref_vol, outdir, maskfile = None, ir = 1, ou = -1, rs = 1, xr = "4 2 2 1", yr = "-1", ts = "1 1 0.5 0.25", delta = "10 6 4 4", an = "-1", center = 0, maxit = 5, term = 95, CTF = False, fourvar = False, snr = 1.0, ref_a = "S", sym = "c1", sort=True, cutoff=999.99, pix_cutoff="0", two_tail=False, model_jump="1 1 1 1 1", restart=False, save_half=False, protos=None, oplane=None, lmask=-1, ilmask=-1, findseam=False, vertstep=None, hpars="-1", hsearch="73.0 170.0", full_output = False, compare_repro = False, compare_ref_free = "-1", ref_free_cutoff= "-1 -1 -1 -1", wcmask = None, debug = False, recon_pad = 4): from alignment import Numrinit, prepare_refrings from utilities import model_circle, get_image, drop_image, get_input_from_string from utilities import bcast_list_to_all, bcast_number_to_all, reduce_EMData_to_root, bcast_EMData_to_all from utilities import send_attr_dict from utilities import get_params_proj, file_type from fundamentals import rot_avg_image import os import types from utilities import print_begin_msg, print_end_msg, print_msg from mpi import mpi_bcast, mpi_comm_size, mpi_comm_rank, MPI_FLOAT, MPI_COMM_WORLD, mpi_barrier, mpi_reduce from mpi import mpi_reduce, MPI_INT, MPI_SUM, mpi_finalize from filter import filt_ctf from projection import prep_vol, prgs from statistics import hist_list, varf3d_MPI, fsc_mask from numpy import array, bincount, array2string, ones number_of_proc = mpi_comm_size(MPI_COMM_WORLD) myid = mpi_comm_rank(MPI_COMM_WORLD) main_node = 0 if myid == main_node: if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "ali3d_MPI", 1) os.mkdir(outdir) mpi_barrier(MPI_COMM_WORLD) if debug: from time import sleep while not os.path.exists(outdir): print "Node ",myid," waiting..." sleep(5) info_file = os.path.join(outdir, "progress%04d"%myid) finfo = open(info_file, 'w') else: finfo = None mjump = get_input_from_string(model_jump) xrng = get_input_from_string(xr) if yr == "-1": yrng = xrng else : yrng = get_input_from_string(yr) step = get_input_from_string(ts) delta = get_input_from_string(delta) ref_free_cutoff = get_input_from_string(ref_free_cutoff) pix_cutoff = get_input_from_string(pix_cutoff) lstp = min(len(xrng), len(yrng), len(step), len(delta)) if an == "-1": an = [-1] * lstp else: an = get_input_from_string(an) # make sure pix_cutoff is set for all iterations if len(pix_cutoff)<lstp: for i in xrange(len(pix_cutoff),lstp): pix_cutoff.append(pix_cutoff[-1]) # don't waste time on sub-pixel alignment for low-resolution ang incr for i in range(len(step)): if (delta[i] > 4 or delta[i] == -1) and step[i] < 1: step[i] = 1 first_ring = int(ir) rstep = int(rs) last_ring = int(ou) max_iter = int(maxit) center = int(center) nrefs = EMUtil.get_image_count( ref_vol ) nmasks = 0 if maskfile: # read number of masks within each maskfile (mc) nmasks = EMUtil.get_image_count( maskfile ) # open masks within maskfile (mc) maskF = EMData.read_images(maskfile, xrange(nmasks)) vol = EMData.read_images(ref_vol, xrange(nrefs)) nx = vol[0].get_xsize() ## make sure box sizes are the same if myid == main_node: im=EMData.read_images(stack,[0]) bx = im[0].get_xsize() if bx!=nx: print_msg("Error: Stack box size (%i) differs from initial model (%i)\n"%(bx,nx)) sys.exit() del im,bx # for helical processing: helicalrecon = False if protos is not None or hpars != "-1" or findseam is True: helicalrecon = True # if no out-of-plane param set, use 5 degrees if oplane is None: oplane=5.0 if protos is not None: proto = get_input_from_string(protos) if len(proto) != nrefs: print_msg("Error: insufficient protofilament numbers supplied") sys.exit() if hpars != "-1": hpars = get_input_from_string(hpars) if len(hpars) != 2*nrefs: print_msg("Error: insufficient helical parameters supplied") sys.exit() ## create helical parameter file for helical reconstruction if helicalrecon is True and myid == main_node: from hfunctions import createHpar # create initial helical parameter files dp=[0]*nrefs dphi=[0]*nrefs vdp=[0]*nrefs vdphi=[0]*nrefs for iref in xrange(nrefs): hpar = os.path.join(outdir,"hpar%02d.spi"%(iref)) params = False if hpars != "-1": # if helical parameters explicitly given, set twist & rise params = [float(hpars[iref*2]),float(hpars[(iref*2)+1])] dp[iref],dphi[iref],vdp[iref],vdphi[iref] = createHpar(hpar,proto[iref],params,vertstep) # get values for helical search parameters hsearch = get_input_from_string(hsearch) if len(hsearch) != 2: print_msg("Error: specify outer and inner radii for helical search") sys.exit() if last_ring < 0 or last_ring > int(nx/2)-2 : last_ring = int(nx/2) - 2 if myid == main_node: # import user_functions # user_func = user_functions.factory[user_func_name] print_begin_msg("ali3d_MPI") print_msg("Input stack : %s\n"%(stack)) print_msg("Reference volume : %s\n"%(ref_vol)) print_msg("Output directory : %s\n"%(outdir)) if nmasks > 0: print_msg("Maskfile (number of masks) : %s (%i)\n"%(maskfile,nmasks)) print_msg("Inner radius : %i\n"%(first_ring)) print_msg("Outer radius : %i\n"%(last_ring)) print_msg("Ring step : %i\n"%(rstep)) print_msg("X search range : %s\n"%(xrng)) print_msg("Y search range : %s\n"%(yrng)) print_msg("Translational step : %s\n"%(step)) print_msg("Angular step : %s\n"%(delta)) print_msg("Angular search range : %s\n"%(an)) print_msg("Maximum iteration : %i\n"%(max_iter)) print_msg("Center type : %i\n"%(center)) print_msg("CTF correction : %s\n"%(CTF)) print_msg("Signal-to-Noise Ratio : %f\n"%(snr)) print_msg("Reference projection method : %s\n"%(ref_a)) print_msg("Symmetry group : %s\n"%(sym)) print_msg("Fourier padding for 3D : %i\n"%(recon_pad)) print_msg("Number of reference models : %i\n"%(nrefs)) print_msg("Sort images between models : %s\n"%(sort)) print_msg("Allow images to jump : %s\n"%(mjump)) print_msg("CC cutoff standard dev : %f\n"%(cutoff)) print_msg("Two tail cutoff : %s\n"%(two_tail)) print_msg("Termination pix error : %f\n"%(term)) print_msg("Pixel error cutoff : %s\n"%(pix_cutoff)) print_msg("Restart : %s\n"%(restart)) print_msg("Full output : %s\n"%(full_output)) print_msg("Compare reprojections : %s\n"%(compare_repro)) print_msg("Compare ref free class avgs : %s\n"%(compare_ref_free)) print_msg("Use cutoff from ref free : %s\n"%(ref_free_cutoff)) if protos: print_msg("Protofilament numbers : %s\n"%(proto)) print_msg("Using helical search range : %s\n"%hsearch) if findseam is True: print_msg("Using seam-based reconstruction\n") if hpars != "-1": print_msg("Using hpars : %s\n"%hpars) if vertstep != None: print_msg("Using vertical step : %.2f\n"%vertstep) if save_half is True: print_msg("Saving even/odd halves\n") for i in xrange(100) : print_msg("*") print_msg("\n\n") if maskfile: if type(maskfile) is types.StringType: mask3D = get_image(maskfile) else: mask3D = maskfile else: mask3D = model_circle(last_ring, nx, nx, nx) numr = Numrinit(first_ring, last_ring, rstep, "F") mask2D = model_circle(last_ring,nx,nx) - model_circle(first_ring,nx,nx) fscmask = model_circle(last_ring,nx,nx,nx) if CTF: from filter import filt_ctf from reconstruction_rjh import rec3D_MPI_noCTF if myid == main_node: active = EMUtil.get_all_attributes(stack, 'active') list_of_particles = [] for im in xrange(len(active)): if active[im]: list_of_particles.append(im) del active nima = len(list_of_particles) else: nima = 0 total_nima = bcast_number_to_all(nima, source_node = main_node) if myid != main_node: list_of_particles = [-1]*total_nima list_of_particles = bcast_list_to_all(list_of_particles, source_node = main_node) image_start, image_end = MPI_start_end(total_nima, number_of_proc, myid) # create a list of images for each node list_of_particles = list_of_particles[image_start: image_end] nima = len(list_of_particles) if debug: finfo.write("image_start, image_end: %d %d\n" %(image_start, image_end)) finfo.flush() data = EMData.read_images(stack, list_of_particles) t_zero = Transform({"type":"spider","phi":0,"theta":0,"psi":0,"tx":0,"ty":0}) transmulti = [[t_zero for i in xrange(nrefs)] for j in xrange(nima)] for iref,im in ((iref,im) for iref in xrange(nrefs) for im in xrange(nima)): if nrefs == 1: transmulti[im][iref] = data[im].get_attr("xform.projection") else: # if multi models, keep track of eulers for all models try: transmulti[im][iref] = data[im].get_attr("eulers_txty.%i"%iref) except: data[im].set_attr("eulers_txty.%i"%iref,t_zero) scoremulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)] pixelmulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)] ref_res = [0.0 for x in xrange(nrefs)] apix = data[0].get_attr('apix_x') # for oplane parameter, create cylindrical mask if oplane is not None and myid == main_node: from hfunctions import createCylMask cmaskf=os.path.join(outdir, "mask3D_cyl.mrc") mask3D = createCylMask(data,ou,lmask,ilmask,cmaskf) # if finding seam of helix, create wedge masks if findseam is True: wedgemask=[] for pf in xrange(nrefs): wedgemask.append(EMData()) # wedgemask option if wcmask is not None: wcmask = get_input_from_string(wcmask) if len(wcmask) != 3: print_msg("Error: wcmask option requires 3 values: x y radius") sys.exit() # determine if particles have helix info: try: data[0].get_attr('h_angle') original_data = [] boxmask = True from hfunctions import createBoxMask except: boxmask = False # prepare particles for im in xrange(nima): data[im].set_attr('ID', list_of_particles[im]) data[im].set_attr('pix_score', int(0)) if CTF: # only phaseflip particles, not full CTF correction ctf_params = data[im].get_attr("ctf") st = Util.infomask(data[im], mask2D, False) data[im] -= st[0] data[im] = filt_ctf(data[im], ctf_params, sign = -1, binary=1) data[im].set_attr('ctf_applied', 1) # for window mask: if boxmask is True: h_angle = data[im].get_attr("h_angle") original_data.append(data[im].copy()) bmask = createBoxMask(nx,apix,ou,lmask,h_angle) data[im]*=bmask del bmask if debug: finfo.write( '%d loaded \n' % nima ) finfo.flush() if myid == main_node: # initialize data for the reference preparation function ref_data = [ mask3D, max(center,0), None, None, None, None ] # for method -1, switch off centering in user function from time import time # this is needed for gathering of pixel errors disps = [] recvcount = [] disps_score = [] recvcount_score = [] for im in xrange(number_of_proc): if( im == main_node ): disps.append(0) disps_score.append(0) else: disps.append(disps[im-1] + recvcount[im-1]) disps_score.append(disps_score[im-1] + recvcount_score[im-1]) ib, ie = MPI_start_end(total_nima, number_of_proc, im) recvcount.append( ie - ib ) recvcount_score.append((ie-ib)*nrefs) pixer = [0.0]*nima cs = [0.0]*3 total_iter = 0 volodd = EMData.read_images(ref_vol, xrange(nrefs)) voleve = EMData.read_images(ref_vol, xrange(nrefs)) if restart: # recreate initial volumes from alignments stored in header itout = "000_00" for iref in xrange(nrefs): if(nrefs == 1): modout = "" else: modout = "_model_%02d"%(iref) if(sort): group = iref for im in xrange(nima): imgroup = data[im].get_attr('group') if imgroup == iref: data[im].set_attr('xform.projection',transmulti[im][iref]) else: group = int(999) for im in xrange(nima): data[im].set_attr('xform.projection',transmulti[im][iref]) fscfile = os.path.join(outdir, "fsc_%s%s"%(itout,modout)) vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF(data, sym, fscmask, fscfile, myid, main_node, index = group, npad = recon_pad) if myid == main_node: if helicalrecon: from hfunctions import processHelicalVol vstep=None if vertstep is not None: vstep=(vdp[iref],vdphi[iref]) print_msg("Old rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref])) hvals=processHelicalVol(vol[iref],voleve[iref],volodd[iref],iref,outdir,itout, dp[iref],dphi[iref],apix,hsearch,findseam,vstep,wcmask) (vol[iref],voleve[iref],volodd[iref],dp[iref],dphi[iref],vdp[iref],vdphi[iref])=hvals print_msg("New rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref])) # get new FSC from symmetrized half volumes fscc = fsc_mask( volodd[iref], voleve[iref], mask3D, rstep, fscfile) else: vol[iref].write_image(os.path.join(outdir, "vol_%s.hdf"%itout),-1) if save_half is True: volodd[iref].write_image(os.path.join(outdir, "volodd_%s.hdf"%itout),-1) voleve[iref].write_image(os.path.join(outdir, "voleve_%s.hdf"%itout),-1) if nmasks > 1: # Read mask for multiplying ref_data[0] = maskF[iref] ref_data[2] = vol[iref] ref_data[3] = fscc # call user-supplied function to prepare reference image, i.e., center and filter it vol[iref], cs,fl = ref_ali3d(ref_data) vol[iref].write_image(os.path.join(outdir, "volf_%s.hdf"%(itout)),-1) if (apix == 1): res_msg = "Models filtered at spatial frequency of:\t" res = fl else: res_msg = "Models filtered at resolution of: \t" res = apix / fl ares = array2string(array(res), precision = 2) print_msg("%s%s\n\n"%(res_msg,ares)) bcast_EMData_to_all(vol[iref], myid, main_node) # write out headers, under MPI writing has to be done sequentially mpi_barrier(MPI_COMM_WORLD) # projection matching for N_step in xrange(lstp): terminate = 0 Iter = -1 while(Iter < max_iter-1 and terminate == 0): Iter += 1 total_iter += 1 itout = "%03g_%02d" %(delta[N_step], Iter) if myid == main_node: print_msg("ITERATION #%3d, inner iteration #%3d\nDelta = %4.1f, an = %5.2f, xrange = %5.2f, yrange = %5.2f, step = %5.2f\n\n"%(N_step, Iter, delta[N_step], an[N_step], xrng[N_step],yrng[N_step],step[N_step])) for iref in xrange(nrefs): if myid == main_node: start_time = time() volft,kb = prep_vol( vol[iref] ) ## constrain projections to out of plane parameter theta1 = None theta2 = None if oplane is not None: theta1 = 90-oplane theta2 = 90+oplane refrings = prepare_refrings( volft, kb, nx, delta[N_step], ref_a, sym, numr, MPI=True, phiEqpsi = "Minus", initial_theta=theta1, delta_theta=theta2) del volft,kb if myid== main_node: print_msg( "Time to prepare projections for model %i: %s\n" % (iref, legibleTime(time()-start_time)) ) start_time = time() for im in xrange( nima ): data[im].set_attr("xform.projection", transmulti[im][iref]) if an[N_step] == -1: t1, peak, pixer[im] = proj_ali_incore(data[im],refrings,numr,xrng[N_step],yrng[N_step],step[N_step],finfo) else: t1, peak, pixer[im] = proj_ali_incore_local(data[im],refrings,numr,xrng[N_step],yrng[N_step],step[N_step],an[N_step],finfo) #data[im].set_attr("xform.projection"%iref, t1) if nrefs > 1: data[im].set_attr("eulers_txty.%i"%iref,t1) scoremulti[im][iref] = peak from pixel_error import max_3D_pixel_error # t1 is the current param, t2 is old t2 = transmulti[im][iref] pixelmulti[im][iref] = max_3D_pixel_error(t1,t2,numr[-3]) transmulti[im][iref] = t1 if myid == main_node: print_msg("Time of alignment for model %i: %s\n"%(iref, legibleTime(time()-start_time))) start_time = time() # gather scoring data from all processors from mpi import mpi_gatherv scoremultisend = sum(scoremulti,[]) pixelmultisend = sum(pixelmulti,[]) tmp = mpi_gatherv(scoremultisend,len(scoremultisend),MPI_FLOAT, recvcount_score, disps_score, MPI_FLOAT, main_node,MPI_COMM_WORLD) tmp1 = mpi_gatherv(pixelmultisend,len(pixelmultisend),MPI_FLOAT, recvcount_score, disps_score, MPI_FLOAT, main_node,MPI_COMM_WORLD) tmp = mpi_bcast(tmp,(total_nima * nrefs), MPI_FLOAT,0, MPI_COMM_WORLD) tmp1 = mpi_bcast(tmp1,(total_nima * nrefs), MPI_FLOAT,0, MPI_COMM_WORLD) tmp = map(float,tmp) tmp1 = map(float,tmp1) score = array(tmp).reshape(-1,nrefs) pixelerror = array(tmp1).reshape(-1,nrefs) score_local = array(scoremulti) mean_score = score.mean(axis=0) std_score = score.std(axis=0) cut = mean_score - (cutoff * std_score) cut2 = mean_score + (cutoff * std_score) res_max = score_local.argmax(axis=1) minus_cc = [0.0 for x in xrange(nrefs)] minus_pix = [0.0 for x in xrange(nrefs)] minus_ref = [0.0 for x in xrange(nrefs)] #output pixel errors if(myid == main_node): from statistics import hist_list lhist = 20 pixmin = pixelerror.min(axis=1) region, histo = hist_list(pixmin, lhist) if(region[0] < 0.0): region[0] = 0.0 print_msg("Histogram of pixel errors\n ERROR number of particles\n") for lhx in xrange(lhist): print_msg(" %10.3f %7d\n"%(region[lhx], histo[lhx])) # Terminate if 95% within 1 pixel error im = 0 for lhx in xrange(lhist): if(region[lhx] > 1.0): break im += histo[lhx] print_msg( "Percent of particles with pixel error < 1: %f\n\n"% (im/float(total_nima)*100)) term_cond = float(term)/100 if(im/float(total_nima) > term_cond): terminate = 1 print_msg("Terminating internal loop\n") del region, histo terminate = mpi_bcast(terminate, 1, MPI_INT, 0, MPI_COMM_WORLD) terminate = int(terminate[0]) for im in xrange(nima): if(sort==False): data[im].set_attr('group',999) elif (mjump[N_step]==1): data[im].set_attr('group',int(res_max[im])) pix_run = data[im].get_attr('pix_score') if (pix_cutoff[N_step]==1 and (terminate==1 or Iter == max_iter-1)): if (pixelmulti[im][int(res_max[im])] > 1): data[im].set_attr('pix_score',int(777)) if (score_local[im][int(res_max[im])]<cut[int(res_max[im])]) or (two_tail and score_local[im][int(res_max[im])]>cut2[int(res_max[im])]): data[im].set_attr('group',int(888)) minus_cc[int(res_max[im])] = minus_cc[int(res_max[im])] + 1 if(pix_run == 777): data[im].set_attr('group',int(777)) minus_pix[int(res_max[im])] = minus_pix[int(res_max[im])] + 1 if (compare_ref_free != "-1") and (ref_free_cutoff[N_step] != -1) and (total_iter > 1): id = data[im].get_attr('ID') if id in rejects: data[im].set_attr('group',int(666)) minus_ref[int(res_max[im])] = minus_ref[int(res_max[im])] + 1 minus_cc_tot = mpi_reduce(minus_cc,nrefs,MPI_FLOAT,MPI_SUM,0,MPI_COMM_WORLD) minus_pix_tot = mpi_reduce(minus_pix,nrefs,MPI_FLOAT,MPI_SUM,0,MPI_COMM_WORLD) minus_ref_tot = mpi_reduce(minus_ref,nrefs,MPI_FLOAT,MPI_SUM,0,MPI_COMM_WORLD) if (myid == main_node): if(sort): tot_max = score.argmax(axis=1) res = bincount(tot_max) else: res = ones(nrefs) * total_nima print_msg("Particle distribution: \t\t%s\n"%(res*1.0)) afcut1 = res - minus_cc_tot afcut2 = afcut1 - minus_pix_tot afcut3 = afcut2 - minus_ref_tot print_msg("Particle distribution after cc cutoff:\t\t%s\n"%(afcut1)) print_msg("Particle distribution after pix cutoff:\t\t%s\n"%(afcut2)) print_msg("Particle distribution after ref cutoff:\t\t%s\n\n"%(afcut3)) res = [0.0 for i in xrange(nrefs)] for iref in xrange(nrefs): if(center == -1): from utilities import estimate_3D_center_MPI, rotate_3D_shift dummy=EMData() cs[0], cs[1], cs[2], dummy, dummy = estimate_3D_center_MPI(data, total_nima, myid, number_of_proc, main_node) cs = mpi_bcast(cs, 3, MPI_FLOAT, main_node, MPI_COMM_WORLD) cs = [-float(cs[0]), -float(cs[1]), -float(cs[2])] rotate_3D_shift(data, cs) if(sort): group = iref for im in xrange(nima): imgroup = data[im].get_attr('group') if imgroup == iref: data[im].set_attr('xform.projection',transmulti[im][iref]) else: group = int(999) for im in xrange(nima): data[im].set_attr('xform.projection',transmulti[im][iref]) if(nrefs == 1): modout = "" else: modout = "_model_%02d"%(iref) fscfile = os.path.join(outdir, "fsc_%s%s"%(itout,modout)) vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF(data, sym, fscmask, fscfile, myid, main_node, index=group, npad=recon_pad) if myid == main_node: print_msg("3D reconstruction time for model %i: %s\n"%(iref, legibleTime(time()-start_time))) start_time = time() # Compute Fourier variance if fourvar: outvar = os.path.join(outdir, "volVar_%s.hdf"%(itout)) ssnr_file = os.path.join(outdir, "ssnr_%s"%(itout)) varf = varf3d_MPI(data, ssnr_text_file=ssnr_file, mask2D=None, reference_structure=vol[iref], ou=last_ring, rw=1.0, npad=1, CTF=None, sign=1, sym=sym, myid=myid) if myid == main_node: print_msg("Time to calculate 3D Fourier variance for model %i: %s\n"%(iref, legibleTime(time()-start_time))) start_time = time() varf = 1.0/varf varf.write_image(outvar,-1) else: varf = None if myid == main_node: if helicalrecon: from hfunctions import processHelicalVol vstep=None if vertstep is not None: vstep=(vdp[iref],vdphi[iref]) print_msg("Old rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref])) hvals=processHelicalVol(vol[iref],voleve[iref],volodd[iref],iref,outdir,itout, dp[iref],dphi[iref],apix,hsearch,findseam,vstep,wcmask) (vol[iref],voleve[iref],volodd[iref],dp[iref],dphi[iref],vdp[iref],vdphi[iref])=hvals print_msg("New rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref])) # get new FSC from symmetrized half volumes fscc = fsc_mask( volodd[iref], voleve[iref], mask3D, rstep, fscfile) print_msg("Time to search and apply helical symmetry for model %i: %s\n\n"%(iref, legibleTime(time()-start_time))) start_time = time() else: vol[iref].write_image(os.path.join(outdir, "vol_%s.hdf"%(itout)),-1) if save_half is True: volodd[iref].write_image(os.path.join(outdir, "volodd_%s.hdf"%(itout)),-1) voleve[iref].write_image(os.path.join(outdir, "voleve_%s.hdf"%(itout)),-1) if nmasks > 1: # Read mask for multiplying ref_data[0] = maskF[iref] ref_data[2] = vol[iref] ref_data[3] = fscc ref_data[4] = varf # call user-supplied function to prepare reference image, i.e., center and filter it vol[iref], cs,fl = ref_ali3d(ref_data) vol[iref].write_image(os.path.join(outdir, "volf_%s.hdf"%(itout)),-1) if (apix == 1): res_msg = "Models filtered at spatial frequency of:\t" res[iref] = fl else: res_msg = "Models filtered at resolution of: \t" res[iref] = apix / fl del varf bcast_EMData_to_all(vol[iref], myid, main_node) if compare_ref_free != "-1": compare_repro = True if compare_repro: outfile_repro = comp_rep(refrings, data, itout, modout, vol[iref], group, nima, nx, myid, main_node, outdir) mpi_barrier(MPI_COMM_WORLD) if compare_ref_free != "-1": ref_free_output = os.path.join(outdir,"ref_free_%s%s"%(itout,modout)) rejects = compare(compare_ref_free, outfile_repro,ref_free_output,yrng[N_step], xrng[N_step], rstep,nx,apix,ref_free_cutoff[N_step], number_of_proc, myid, main_node) # retrieve alignment params from all processors par_str = ['xform.projection','ID','group'] if nrefs > 1: for iref in xrange(nrefs): par_str.append('eulers_txty.%i'%iref) if myid == main_node: from utilities import recv_attr_dict recv_attr_dict(main_node, stack, data, par_str, image_start, image_end, number_of_proc) else: send_attr_dict(main_node, data, par_str, image_start, image_end) if myid == main_node: ares = array2string(array(res), precision = 2) print_msg("%s%s\n\n"%(res_msg,ares)) dummy = EMData() if full_output: nimat = EMUtil.get_image_count(stack) output_file = os.path.join(outdir, "paramout_%s"%itout) foutput = open(output_file, 'w') for im in xrange(nimat): # save the parameters for each of the models outstring = "" dummy.read_image(stack,im,True) param3d = dummy.get_attr('xform.projection') g = dummy.get_attr("group") # retrieve alignments in EMAN-format pE = param3d.get_params('eman') outstring += "%f\t%f\t%f\t%f\t%f\t%i\n" %(pE["az"], pE["alt"], pE["phi"], pE["tx"], pE["ty"],g) foutput.write(outstring) foutput.close() del dummy mpi_barrier(MPI_COMM_WORLD) # mpi_finalize() if myid == main_node: print_end_msg("ali3d_MPI")
def ali3d_MPI(stack, ref_vol, outdir, maskfile=None, ir=1, ou=-1, rs=1, xr="4 2 2 1", yr="-1", ts="1 1 0.5 0.25", delta="10 6 4 4", an="-1", center=0, maxit=5, term=95, CTF=False, fourvar=False, snr=1.0, ref_a="S", sym="c1", sort=True, cutoff=999.99, pix_cutoff="0", two_tail=False, model_jump="1 1 1 1 1", restart=False, save_half=False, protos=None, oplane=None, lmask=-1, ilmask=-1, findseam=False, vertstep=None, hpars="-1", hsearch="0.0 50.0", full_output=False, compare_repro=False, compare_ref_free="-1", ref_free_cutoff="-1 -1 -1 -1", wcmask=None, debug=False, recon_pad=4, olmask=75): from alignment import Numrinit, prepare_refrings from utilities import model_circle, get_image, drop_image, get_input_from_string from utilities import bcast_list_to_all, bcast_number_to_all, reduce_EMData_to_root, bcast_EMData_to_all from utilities import send_attr_dict from utilities import get_params_proj, file_type from fundamentals import rot_avg_image import os import types from utilities import print_begin_msg, print_end_msg, print_msg from mpi import mpi_bcast, mpi_comm_size, mpi_comm_rank, MPI_FLOAT, MPI_COMM_WORLD, mpi_barrier, mpi_reduce from mpi import mpi_reduce, MPI_INT, MPI_SUM, mpi_finalize from filter import filt_ctf from projection import prep_vol, prgs from statistics import hist_list, varf3d_MPI, fsc_mask from numpy import array, bincount, array2string, ones number_of_proc = mpi_comm_size(MPI_COMM_WORLD) myid = mpi_comm_rank(MPI_COMM_WORLD) main_node = 0 if myid == main_node: if os.path.exists(outdir): ERROR( 'Output directory exists, please change the name and restart the program', "ali3d_MPI", 1) os.mkdir(outdir) mpi_barrier(MPI_COMM_WORLD) if debug: from time import sleep while not os.path.exists(outdir): print "Node ", myid, " waiting..." sleep(5) info_file = os.path.join(outdir, "progress%04d" % myid) finfo = open(info_file, 'w') else: finfo = None mjump = get_input_from_string(model_jump) xrng = get_input_from_string(xr) if yr == "-1": yrng = xrng else: yrng = get_input_from_string(yr) step = get_input_from_string(ts) delta = get_input_from_string(delta) ref_free_cutoff = get_input_from_string(ref_free_cutoff) pix_cutoff = get_input_from_string(pix_cutoff) lstp = min(len(xrng), len(yrng), len(step), len(delta)) if an == "-1": an = [-1] * lstp else: an = get_input_from_string(an) # make sure pix_cutoff is set for all iterations if len(pix_cutoff) < lstp: for i in xrange(len(pix_cutoff), lstp): pix_cutoff.append(pix_cutoff[-1]) # don't waste time on sub-pixel alignment for low-resolution ang incr for i in range(len(step)): if (delta[i] > 4 or delta[i] == -1) and step[i] < 1: step[i] = 1 first_ring = int(ir) rstep = int(rs) last_ring = int(ou) max_iter = int(maxit) center = int(center) nrefs = EMUtil.get_image_count(ref_vol) nmasks = 0 if maskfile: # read number of masks within each maskfile (mc) nmasks = EMUtil.get_image_count(maskfile) # open masks within maskfile (mc) maskF = EMData.read_images(maskfile, xrange(nmasks)) vol = EMData.read_images(ref_vol, xrange(nrefs)) nx = vol[0].get_xsize() ## make sure box sizes are the same if myid == main_node: im = EMData.read_images(stack, [0]) bx = im[0].get_xsize() if bx != nx: print_msg( "Error: Stack box size (%i) differs from initial model (%i)\n" % (bx, nx)) sys.exit() del im, bx # for helical processing: helicalrecon = False if protos is not None or hpars != "-1" or findseam is True: helicalrecon = True # if no out-of-plane param set, use 5 degrees if oplane is None: oplane = 5.0 if protos is not None: proto = get_input_from_string(protos) if len(proto) != nrefs: print_msg("Error: insufficient protofilament numbers supplied") sys.exit() if hpars != "-1": hpars = get_input_from_string(hpars) if len(hpars) != 2 * nrefs: print_msg("Error: insufficient helical parameters supplied") sys.exit() ## create helical parameter file for helical reconstruction if helicalrecon is True and myid == main_node: from hfunctions import createHpar # create initial helical parameter files dp = [0] * nrefs dphi = [0] * nrefs vdp = [0] * nrefs vdphi = [0] * nrefs for iref in xrange(nrefs): hpar = os.path.join(outdir, "hpar%02d.spi" % (iref)) params = False if hpars != "-1": # if helical parameters explicitly given, set twist & rise params = [float(hpars[iref * 2]), float(hpars[(iref * 2) + 1])] dp[iref], dphi[iref], vdp[iref], vdphi[iref] = createHpar( hpar, proto[iref], params, vertstep) # get values for helical search parameters hsearch = get_input_from_string(hsearch) if len(hsearch) != 2: print_msg("Error: specify outer and inner radii for helical search") sys.exit() if last_ring < 0 or last_ring > int(nx / 2) - 2: last_ring = int(nx / 2) - 2 if myid == main_node: # import user_functions # user_func = user_functions.factory[user_func_name] print_begin_msg("ali3d_MPI") print_msg("Input stack : %s\n" % (stack)) print_msg("Reference volume : %s\n" % (ref_vol)) print_msg("Output directory : %s\n" % (outdir)) if nmasks > 0: print_msg("Maskfile (number of masks) : %s (%i)\n" % (maskfile, nmasks)) print_msg("Inner radius : %i\n" % (first_ring)) print_msg("Outer radius : %i\n" % (last_ring)) print_msg("Ring step : %i\n" % (rstep)) print_msg("X search range : %s\n" % (xrng)) print_msg("Y search range : %s\n" % (yrng)) print_msg("Translational step : %s\n" % (step)) print_msg("Angular step : %s\n" % (delta)) print_msg("Angular search range : %s\n" % (an)) print_msg("Maximum iteration : %i\n" % (max_iter)) print_msg("Center type : %i\n" % (center)) print_msg("CTF correction : %s\n" % (CTF)) print_msg("Signal-to-Noise Ratio : %f\n" % (snr)) print_msg("Reference projection method : %s\n" % (ref_a)) print_msg("Symmetry group : %s\n" % (sym)) print_msg("Fourier padding for 3D : %i\n" % (recon_pad)) print_msg("Number of reference models : %i\n" % (nrefs)) print_msg("Sort images between models : %s\n" % (sort)) print_msg("Allow images to jump : %s\n" % (mjump)) print_msg("CC cutoff standard dev : %f\n" % (cutoff)) print_msg("Two tail cutoff : %s\n" % (two_tail)) print_msg("Termination pix error : %f\n" % (term)) print_msg("Pixel error cutoff : %s\n" % (pix_cutoff)) print_msg("Restart : %s\n" % (restart)) print_msg("Full output : %s\n" % (full_output)) print_msg("Compare reprojections : %s\n" % (compare_repro)) print_msg("Compare ref free class avgs : %s\n" % (compare_ref_free)) print_msg("Use cutoff from ref free : %s\n" % (ref_free_cutoff)) if protos: print_msg("Protofilament numbers : %s\n" % (proto)) print_msg("Using helical search range : %s\n" % hsearch) if findseam is True: print_msg("Using seam-based reconstruction\n") if hpars != "-1": print_msg("Using hpars : %s\n" % hpars) if vertstep != None: print_msg("Using vertical step : %.2f\n" % vertstep) if save_half is True: print_msg("Saving even/odd halves\n") for i in xrange(100): print_msg("*") print_msg("\n\n") if maskfile: if type(maskfile) is types.StringType: mask3D = get_image(maskfile) else: mask3D = maskfile else: mask3D = model_circle(last_ring, nx, nx, nx) numr = Numrinit(first_ring, last_ring, rstep, "F") mask2D = model_circle(last_ring, nx, nx) - model_circle(first_ring, nx, nx) fscmask = model_circle(last_ring, nx, nx, nx) if CTF: from filter import filt_ctf from reconstruction_rjh import rec3D_MPI_noCTF if myid == main_node: active = EMUtil.get_all_attributes(stack, 'active') list_of_particles = [] for im in xrange(len(active)): if active[im]: list_of_particles.append(im) del active nima = len(list_of_particles) else: nima = 0 total_nima = bcast_number_to_all(nima, source_node=main_node) if myid != main_node: list_of_particles = [-1] * total_nima list_of_particles = bcast_list_to_all(list_of_particles, source_node=main_node) image_start, image_end = MPI_start_end(total_nima, number_of_proc, myid) # create a list of images for each node list_of_particles = list_of_particles[image_start:image_end] nima = len(list_of_particles) if debug: finfo.write("image_start, image_end: %d %d\n" % (image_start, image_end)) finfo.flush() data = EMData.read_images(stack, list_of_particles) t_zero = Transform({ "type": "spider", "phi": 0, "theta": 0, "psi": 0, "tx": 0, "ty": 0 }) transmulti = [[t_zero for i in xrange(nrefs)] for j in xrange(nima)] for iref, im in ((iref, im) for iref in xrange(nrefs) for im in xrange(nima)): if nrefs == 1: transmulti[im][iref] = data[im].get_attr("xform.projection") else: # if multi models, keep track of eulers for all models try: transmulti[im][iref] = data[im].get_attr("eulers_txty.%i" % iref) except: data[im].set_attr("eulers_txty.%i" % iref, t_zero) scoremulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)] pixelmulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)] ref_res = [0.0 for x in xrange(nrefs)] apix = data[0].get_attr('apix_x') # for oplane parameter, create cylindrical mask if oplane is not None and myid == main_node: from hfunctions import createCylMask cmaskf = os.path.join(outdir, "mask3D_cyl.mrc") mask3D = createCylMask(data, olmask, lmask, ilmask, cmaskf) # if finding seam of helix, create wedge masks if findseam is True: wedgemask = [] for pf in xrange(nrefs): wedgemask.append(EMData()) # wedgemask option if wcmask is not None: wcmask = get_input_from_string(wcmask) if len(wcmask) != 3: print_msg( "Error: wcmask option requires 3 values: x y radius") sys.exit() # determine if particles have helix info: try: data[0].get_attr('h_angle') original_data = [] boxmask = True from hfunctions import createBoxMask except: boxmask = False # prepare particles for im in xrange(nima): data[im].set_attr('ID', list_of_particles[im]) data[im].set_attr('pix_score', int(0)) if CTF: # only phaseflip particles, not full CTF correction ctf_params = data[im].get_attr("ctf") st = Util.infomask(data[im], mask2D, False) data[im] -= st[0] data[im] = filt_ctf(data[im], ctf_params, sign=-1, binary=1) data[im].set_attr('ctf_applied', 1) # for window mask: if boxmask is True: h_angle = data[im].get_attr("h_angle") original_data.append(data[im].copy()) bmask = createBoxMask(nx, apix, ou, lmask, h_angle) data[im] *= bmask del bmask if debug: finfo.write('%d loaded \n' % nima) finfo.flush() if myid == main_node: # initialize data for the reference preparation function ref_data = [mask3D, max(center, 0), None, None, None, None] # for method -1, switch off centering in user function from time import time # this is needed for gathering of pixel errors disps = [] recvcount = [] disps_score = [] recvcount_score = [] for im in xrange(number_of_proc): if (im == main_node): disps.append(0) disps_score.append(0) else: disps.append(disps[im - 1] + recvcount[im - 1]) disps_score.append(disps_score[im - 1] + recvcount_score[im - 1]) ib, ie = MPI_start_end(total_nima, number_of_proc, im) recvcount.append(ie - ib) recvcount_score.append((ie - ib) * nrefs) pixer = [0.0] * nima cs = [0.0] * 3 total_iter = 0 volodd = EMData.read_images(ref_vol, xrange(nrefs)) voleve = EMData.read_images(ref_vol, xrange(nrefs)) if restart: # recreate initial volumes from alignments stored in header itout = "000_00" for iref in xrange(nrefs): if (nrefs == 1): modout = "" else: modout = "_model_%02d" % (iref) if (sort): group = iref for im in xrange(nima): imgroup = data[im].get_attr('group') if imgroup == iref: data[im].set_attr('xform.projection', transmulti[im][iref]) else: group = int(999) for im in xrange(nima): data[im].set_attr('xform.projection', transmulti[im][iref]) fscfile = os.path.join(outdir, "fsc_%s%s" % (itout, modout)) vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF( data, sym, fscmask, fscfile, myid, main_node, index=group, npad=recon_pad) if myid == main_node: if helicalrecon: from hfunctions import processHelicalVol vstep = None if vertstep is not None: vstep = (vdp[iref], vdphi[iref]) print_msg( "Old rise and twist for model %i : %8.3f, %8.3f\n" % (iref, dp[iref], dphi[iref])) hvals = processHelicalVol(vol[iref], voleve[iref], volodd[iref], iref, outdir, itout, dp[iref], dphi[iref], apix, hsearch, findseam, vstep, wcmask) (vol[iref], voleve[iref], volodd[iref], dp[iref], dphi[iref], vdp[iref], vdphi[iref]) = hvals print_msg( "New rise and twist for model %i : %8.3f, %8.3f\n" % (iref, dp[iref], dphi[iref])) # get new FSC from symmetrized half volumes fscc = fsc_mask(volodd[iref], voleve[iref], mask3D, rstep, fscfile) else: vol[iref].write_image( os.path.join(outdir, "vol_%s.hdf" % itout), -1) if save_half is True: volodd[iref].write_image( os.path.join(outdir, "volodd_%s.hdf" % itout), -1) voleve[iref].write_image( os.path.join(outdir, "voleve_%s.hdf" % itout), -1) if nmasks > 1: # Read mask for multiplying ref_data[0] = maskF[iref] ref_data[2] = vol[iref] ref_data[3] = fscc # call user-supplied function to prepare reference image, i.e., center and filter it vol[iref], cs, fl = ref_ali3d(ref_data) vol[iref].write_image( os.path.join(outdir, "volf_%s.hdf" % (itout)), -1) if (apix == 1): res_msg = "Models filtered at spatial frequency of:\t" res = fl else: res_msg = "Models filtered at resolution of: \t" res = apix / fl ares = array2string(array(res), precision=2) print_msg("%s%s\n\n" % (res_msg, ares)) bcast_EMData_to_all(vol[iref], myid, main_node) # write out headers, under MPI writing has to be done sequentially mpi_barrier(MPI_COMM_WORLD) # projection matching for N_step in xrange(lstp): terminate = 0 Iter = -1 while (Iter < max_iter - 1 and terminate == 0): Iter += 1 total_iter += 1 itout = "%03g_%02d" % (delta[N_step], Iter) if myid == main_node: print_msg( "ITERATION #%3d, inner iteration #%3d\nDelta = %4.1f, an = %5.2f, xrange = %5.2f, yrange = %5.2f, step = %5.2f\n\n" % (N_step, Iter, delta[N_step], an[N_step], xrng[N_step], yrng[N_step], step[N_step])) for iref in xrange(nrefs): if myid == main_node: start_time = time() volft, kb = prep_vol(vol[iref]) ## constrain projections to out of plane parameter theta1 = None theta2 = None if oplane is not None: theta1 = 90 - oplane theta2 = 90 + oplane refrings = prepare_refrings(volft, kb, nx, delta[N_step], ref_a, sym, numr, MPI=True, phiEqpsi="Minus", initial_theta=theta1, delta_theta=theta2) del volft, kb if myid == main_node: print_msg( "Time to prepare projections for model %i: %s\n" % (iref, legibleTime(time() - start_time))) start_time = time() for im in xrange(nima): data[im].set_attr("xform.projection", transmulti[im][iref]) if an[N_step] == -1: t1, peak, pixer[im] = proj_ali_incore( data[im], refrings, numr, xrng[N_step], yrng[N_step], step[N_step], finfo) else: t1, peak, pixer[im] = proj_ali_incore_local( data[im], refrings, numr, xrng[N_step], yrng[N_step], step[N_step], an[N_step], finfo) #data[im].set_attr("xform.projection"%iref, t1) if nrefs > 1: data[im].set_attr("eulers_txty.%i" % iref, t1) scoremulti[im][iref] = peak from pixel_error import max_3D_pixel_error # t1 is the current param, t2 is old t2 = transmulti[im][iref] pixelmulti[im][iref] = max_3D_pixel_error(t1, t2, numr[-3]) transmulti[im][iref] = t1 if myid == main_node: print_msg("Time of alignment for model %i: %s\n" % (iref, legibleTime(time() - start_time))) start_time = time() # gather scoring data from all processors from mpi import mpi_gatherv scoremultisend = sum(scoremulti, []) pixelmultisend = sum(pixelmulti, []) tmp = mpi_gatherv(scoremultisend, len(scoremultisend), MPI_FLOAT, recvcount_score, disps_score, MPI_FLOAT, main_node, MPI_COMM_WORLD) tmp1 = mpi_gatherv(pixelmultisend, len(pixelmultisend), MPI_FLOAT, recvcount_score, disps_score, MPI_FLOAT, main_node, MPI_COMM_WORLD) tmp = mpi_bcast(tmp, (total_nima * nrefs), MPI_FLOAT, 0, MPI_COMM_WORLD) tmp1 = mpi_bcast(tmp1, (total_nima * nrefs), MPI_FLOAT, 0, MPI_COMM_WORLD) tmp = map(float, tmp) tmp1 = map(float, tmp1) score = array(tmp).reshape(-1, nrefs) pixelerror = array(tmp1).reshape(-1, nrefs) score_local = array(scoremulti) mean_score = score.mean(axis=0) std_score = score.std(axis=0) cut = mean_score - (cutoff * std_score) cut2 = mean_score + (cutoff * std_score) res_max = score_local.argmax(axis=1) minus_cc = [0.0 for x in xrange(nrefs)] minus_pix = [0.0 for x in xrange(nrefs)] minus_ref = [0.0 for x in xrange(nrefs)] #output pixel errors if (myid == main_node): from statistics import hist_list lhist = 20 pixmin = pixelerror.min(axis=1) region, histo = hist_list(pixmin, lhist) if (region[0] < 0.0): region[0] = 0.0 print_msg( "Histogram of pixel errors\n ERROR number of particles\n" ) for lhx in xrange(lhist): print_msg(" %10.3f %7d\n" % (region[lhx], histo[lhx])) # Terminate if 95% within 1 pixel error im = 0 for lhx in xrange(lhist): if (region[lhx] > 1.0): break im += histo[lhx] print_msg("Percent of particles with pixel error < 1: %f\n\n" % (im / float(total_nima) * 100)) term_cond = float(term) / 100 if (im / float(total_nima) > term_cond): terminate = 1 print_msg("Terminating internal loop\n") del region, histo terminate = mpi_bcast(terminate, 1, MPI_INT, 0, MPI_COMM_WORLD) terminate = int(terminate[0]) for im in xrange(nima): if (sort == False): data[im].set_attr('group', 999) elif (mjump[N_step] == 1): data[im].set_attr('group', int(res_max[im])) pix_run = data[im].get_attr('pix_score') if (pix_cutoff[N_step] == 1 and (terminate == 1 or Iter == max_iter - 1)): if (pixelmulti[im][int(res_max[im])] > 1): data[im].set_attr('pix_score', int(777)) if (score_local[im][int(res_max[im])] < cut[int( res_max[im])]) or (two_tail and score_local[im][int( res_max[im])] > cut2[int(res_max[im])]): data[im].set_attr('group', int(888)) minus_cc[int(res_max[im])] = minus_cc[int(res_max[im])] + 1 if (pix_run == 777): data[im].set_attr('group', int(777)) minus_pix[int( res_max[im])] = minus_pix[int(res_max[im])] + 1 if (compare_ref_free != "-1") and (ref_free_cutoff[N_step] != -1) and (total_iter > 1): id = data[im].get_attr('ID') if id in rejects: data[im].set_attr('group', int(666)) minus_ref[int( res_max[im])] = minus_ref[int(res_max[im])] + 1 minus_cc_tot = mpi_reduce(minus_cc, nrefs, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD) minus_pix_tot = mpi_reduce(minus_pix, nrefs, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD) minus_ref_tot = mpi_reduce(minus_ref, nrefs, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD) if (myid == main_node): if (sort): tot_max = score.argmax(axis=1) res = bincount(tot_max) else: res = ones(nrefs) * total_nima print_msg("Particle distribution: \t\t%s\n" % (res * 1.0)) afcut1 = res - minus_cc_tot afcut2 = afcut1 - minus_pix_tot afcut3 = afcut2 - minus_ref_tot print_msg("Particle distribution after cc cutoff:\t\t%s\n" % (afcut1)) print_msg("Particle distribution after pix cutoff:\t\t%s\n" % (afcut2)) print_msg("Particle distribution after ref cutoff:\t\t%s\n\n" % (afcut3)) res = [0.0 for i in xrange(nrefs)] for iref in xrange(nrefs): if (center == -1): from utilities import estimate_3D_center_MPI, rotate_3D_shift dummy = EMData() cs[0], cs[1], cs[2], dummy, dummy = estimate_3D_center_MPI( data, total_nima, myid, number_of_proc, main_node) cs = mpi_bcast(cs, 3, MPI_FLOAT, main_node, MPI_COMM_WORLD) cs = [-float(cs[0]), -float(cs[1]), -float(cs[2])] rotate_3D_shift(data, cs) if (sort): group = iref for im in xrange(nima): imgroup = data[im].get_attr('group') if imgroup == iref: data[im].set_attr('xform.projection', transmulti[im][iref]) else: group = int(999) for im in xrange(nima): data[im].set_attr('xform.projection', transmulti[im][iref]) if (nrefs == 1): modout = "" else: modout = "_model_%02d" % (iref) fscfile = os.path.join(outdir, "fsc_%s%s" % (itout, modout)) vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF( data, sym, fscmask, fscfile, myid, main_node, index=group, npad=recon_pad) if myid == main_node: print_msg("3D reconstruction time for model %i: %s\n" % (iref, legibleTime(time() - start_time))) start_time = time() # Compute Fourier variance if fourvar: outvar = os.path.join(outdir, "volVar_%s.hdf" % (itout)) ssnr_file = os.path.join(outdir, "ssnr_%s" % (itout)) varf = varf3d_MPI(data, ssnr_text_file=ssnr_file, mask2D=None, reference_structure=vol[iref], ou=last_ring, rw=1.0, npad=1, CTF=None, sign=1, sym=sym, myid=myid) if myid == main_node: print_msg( "Time to calculate 3D Fourier variance for model %i: %s\n" % (iref, legibleTime(time() - start_time))) start_time = time() varf = 1.0 / varf varf.write_image(outvar, -1) else: varf = None if myid == main_node: if helicalrecon: from hfunctions import processHelicalVol vstep = None if vertstep is not None: vstep = (vdp[iref], vdphi[iref]) print_msg( "Old rise and twist for model %i : %8.3f, %8.3f\n" % (iref, dp[iref], dphi[iref])) hvals = processHelicalVol(vol[iref], voleve[iref], volodd[iref], iref, outdir, itout, dp[iref], dphi[iref], apix, hsearch, findseam, vstep, wcmask) (vol[iref], voleve[iref], volodd[iref], dp[iref], dphi[iref], vdp[iref], vdphi[iref]) = hvals print_msg( "New rise and twist for model %i : %8.3f, %8.3f\n" % (iref, dp[iref], dphi[iref])) # get new FSC from symmetrized half volumes fscc = fsc_mask(volodd[iref], voleve[iref], mask3D, rstep, fscfile) print_msg( "Time to search and apply helical symmetry for model %i: %s\n\n" % (iref, legibleTime(time() - start_time))) start_time = time() else: vol[iref].write_image( os.path.join(outdir, "vol_%s.hdf" % (itout)), -1) if save_half is True: volodd[iref].write_image( os.path.join(outdir, "volodd_%s.hdf" % (itout)), -1) voleve[iref].write_image( os.path.join(outdir, "voleve_%s.hdf" % (itout)), -1) if nmasks > 1: # Read mask for multiplying ref_data[0] = maskF[iref] ref_data[2] = vol[iref] ref_data[3] = fscc ref_data[4] = varf # call user-supplied function to prepare reference image, i.e., center and filter it vol[iref], cs, fl = ref_ali3d(ref_data) vol[iref].write_image( os.path.join(outdir, "volf_%s.hdf" % (itout)), -1) if (apix == 1): res_msg = "Models filtered at spatial frequency of:\t" res[iref] = fl else: res_msg = "Models filtered at resolution of: \t" res[iref] = apix / fl del varf bcast_EMData_to_all(vol[iref], myid, main_node) if compare_ref_free != "-1": compare_repro = True if compare_repro: outfile_repro = comp_rep(refrings, data, itout, modout, vol[iref], group, nima, nx, myid, main_node, outdir) mpi_barrier(MPI_COMM_WORLD) if compare_ref_free != "-1": ref_free_output = os.path.join( outdir, "ref_free_%s%s" % (itout, modout)) rejects = compare(compare_ref_free, outfile_repro, ref_free_output, yrng[N_step], xrng[N_step], rstep, nx, apix, ref_free_cutoff[N_step], number_of_proc, myid, main_node) # retrieve alignment params from all processors par_str = ['xform.projection', 'ID', 'group'] if nrefs > 1: for iref in xrange(nrefs): par_str.append('eulers_txty.%i' % iref) if myid == main_node: from utilities import recv_attr_dict recv_attr_dict(main_node, stack, data, par_str, image_start, image_end, number_of_proc) else: send_attr_dict(main_node, data, par_str, image_start, image_end) if myid == main_node: ares = array2string(array(res), precision=2) print_msg("%s%s\n\n" % (res_msg, ares)) dummy = EMData() if full_output: nimat = EMUtil.get_image_count(stack) output_file = os.path.join(outdir, "paramout_%s" % itout) foutput = open(output_file, 'w') for im in xrange(nimat): # save the parameters for each of the models outstring = "" dummy.read_image(stack, im, True) param3d = dummy.get_attr('xform.projection') g = dummy.get_attr("group") # retrieve alignments in EMAN-format pE = param3d.get_params('eman') outstring += "%f\t%f\t%f\t%f\t%f\t%i\n" % ( pE["az"], pE["alt"], pE["phi"], pE["tx"], pE["ty"], g) foutput.write(outstring) foutput.close() del dummy mpi_barrier(MPI_COMM_WORLD) # mpi_finalize() if myid == main_node: print_end_msg("ali3d_MPI")
def proj_ali_incore_local(data, refrings, numr, xrng, yrng, step, an, finfo=None): from utilities import compose_transform2 #from utilities import set_params_proj, get_params_proj from math import cos, sin, pi ID = data.get_attr("ID") mode = "F" nx = data.get_xsize() ny = data.get_ysize() # center is in SPIDER convention cnx = nx // 2 + 1 cny = ny // 2 + 1 ant = cos(an * pi / 180.0) #phi, theta, psi, sxo, syo = get_params_proj(data) t1 = data.get_attr("xform.projection") dp = t1.get_params("spider") # get translations from data tx = dp["tx"] ty = dp["ty"] if finfo: finfo.write("Image id: %6d\n" % (ID)) finfo.write("Old parameters: %9.4f %9.4f %9.4f %9.4f %9.4f\n" % (dp["phi"], dp["theta"], dp["psi"], -tx, -ty)) finfo.flush() [ang, sxs, sys, mirror, iref, peak] = Util.multiref_polar_ali_2d_local(data, refrings, xrng, yrng, step, ant, mode, numr, cnx + tx, cny + ty) iref = int(iref) data.set_attr("assign", iref) if iref > -1: # The ormqip returns parameters such that the transformation is applied first, the mirror operation second. # What that means is that one has to change the the Eulerian angles so they point into mirrored direction: phi+180, 180-theta, 180-psi angb, sxb, syb, ct = compose_transform2(0.0, sxs, sys, 1, -ang, 0.0, 0.0, 1) if mirror: phi = (refrings[iref].get_attr("phi") + 540.0) % 360.0 theta = 180.0 - refrings[iref].get_attr("theta") psi = (540.0 - refrings[iref].get_attr("psi") + angb) % 360.0 s2x = sxb - tx s2y = syb - ty else: phi = refrings[iref].get_attr("phi") theta = refrings[iref].get_attr("theta") psi = (refrings[iref].get_attr("psi") + angb + 360.0) % 360.0 s2x = sxb - tx s2y = syb - ty t2 = Transform({ "type": "spider", "phi": phi, "theta": theta, "psi": psi }) t2.set_trans(Vec2f(-s2x, -s2y)) from pixel_error import max_3D_pixel_error pixel_error = max_3D_pixel_error(t1, t2, numr[-3]) if finfo: finfo.write( "New parameters: %9.4f %9.4f %9.4f %9.4f %9.4f %10.5f %11.3e\n\n" % (phi, theta, psi, s2x, s2y, peak, pixel_error)) finfo.flush() return t2, peak, pixel_error else: return -1.0e23, 0.0
def main(): arglist = [] for arg in sys.argv: arglist.append( arg ) progname = os.path.basename(arglist[0]) usage = progname + " stack outdir --phase=1 --ou=outer_radius|sxconsistency.py --phase=3 newlocal/main000 --ou=133 --thresherr=3.0 --params=paramsa outgrouparms" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--phase", type= "int", default= 1, help="Phase =1 prepares resampled stacks, =2 analyzes consistency of orientation parameters") parser.add_option("--ou", type= "int", default= -1, help="outer radius for calculation of pixel error") parser.add_option("--sym", type="string", default= "c1", help="symmetry of the refined structure") parser.add_option("--thresherr", type="float", default=1.0, help="Threshold for accpetable orientation errors (in pixels)") parser.add_option("--ndigits", type="int", default=1, help="Accuracy for checking whether parameters are identical") parser.add_option("--chunk", type="string", default="", help="Root of of four chunk files with indeces") parser.add_option("--params", type="string", default="", help="Root of of six parameter file names with refinement results") (options, args) = parser.parse_args(arglist[1:]) global_def.BATCH = True if options.phase == 1 and len(args) == 2: inputbdb = args[0] outdir = args[1] nn = EMUtil.get_image_count(inputbdb) t = range(nn) shuffle(t) chunks = [] for i in xrange(4): # I use the MPI function here just to easily get the balanced load j,k = MPI_start_end(nn, 4, i) chunks.append(t[j:k]) chunks[i].sort() write_text_file(chunks[i],os.path.join(outdir,'chunk%01d.txt'%i)) del t write_text_file([ len(chunks[i]) for i in xrange(4) ],os.path.join(outdir,'chunklengths.txt')) """ pt = [[None]]*6 ll=0 for i in xrange(3): for j in xrange(i+1,4): pt[ll] = chunks[i]+chunks[j] ll+=1 for i in xrange(6): listfile = os.path.join(outdir,'lili%01d.txt'%i) write_text_file(pt[i],listfile) outbdb = "bdb:"+ os.path.join(outdir,"X%01d"%i) cmd = '{} {} {} {}'.format('e2bdb.py', inputbdb, '--makevstack='+outbdb, '--list='+listfile) subprocess.call(cmd, shell=True) """ # Run 6 programs elif options.phase == 2 and len(args) == 2: outdir = args[0] howmanythesame = args[1] ndigits = options.ndigits #for chc5 1, for ribo 4#4.0 prms = [] for i in xrange(6): prms.append( read_text_row(os.path.join(outdir,options.params+"%01d.txt"%i)) ) for j in xrange(len(prms[-1])): for k in xrange(5): prms[-1][j][k] = round(prms[-1][j][k], ndigits) nn = 2*len(prms[0]) n4 = nn//4 qt=[[[-1.0]]*nn for i in xrange(6)] ll=0 for i in xrange(3): for j in xrange(i+1,4): qt[ll][i*n4:(i+1)*n4]=prms[ll][:n4] qt[ll][j*n4:(j+1)*n4]=prms[ll][n4:] ll+=1 thesame = 0 for ll in xrange(len(qt[0])): rw = [] for j in xrange(6): if(len(qt[j][ll]) > 1): rw.append(qt[j][ll]) isame = True for j in xrange(3): if(rw[0][j] != rw[1][j]): isame = False #print ll,rw[0][j], rw[1][j] break if(rw[0][j] != rw[2][j]): isame = False #print ll,rw[0][j], rw[2][j] break if isame: thesame += 1 qt = float(thesame)/nn print "Proportion of the same orientations ",qt write_text_file([qt], os.path.join(outdir,howmanythesame) ) ######### PHASE 3 elif options.phase == 3 and len(args) == 2: outdir = args[0] outgrouparms= args[1] radius = options.ou thresherr = options.thresherr #for chc5 1, for ribo 4#4.0 sym = int(options.sym[1:]) qsym = 360.0/sym blocks = ['A','B','C','D'] params = {} ll = 0 for i in xrange(3): for j in xrange(i+1,4): params[chr(65+i)+chr(48+ll)]=[] params[chr(65+j)+chr(48+ll)]=[] ll+=1 chunks = {} chunklengths = {} for i in xrange(4): chunks[chr(65+i)] = map(int,read_text_file(os.path.join(outdir,options.chunk+"%01d.txt"%i))) chunklengths[chr(65+i)] = len(chunks[chr(65+i)]) for i in xrange(6): prms = read_text_row(os.path.join(outdir,options.params+"%01d.txt"%i)) for q in blocks: if params.has_key(q+chr(48+i)): params[q+chr(48+i)] = prms[:chunklengths[q]] del prms[:chunklengths[q]] pairs = [["A0","A1"],["B3","B4"],["C1","C5"],["D2","D4"]] lefts = ["A2","B0","C3","D5"] # Compute average projection params and pixel errors avgtrans = {} pixer = {} for i,q in enumerate(pairs): avgtrans[q[0][0]] = [0.0]*chunklengths[q[0][0]] pixer[q[0][0]] = [0.0]*chunklengths[q[0][0]] for j in xrange(chunklengths[q[0][0]]): fifi = [ params[q[0]][j], params[q[1]][j] ] nas = [0.0,0.0,0.0] if( sym == 1): pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in xrange(2): n1,n2,n3 = getfvec(fifi[i][0],fifi[i][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1,m2,m3 = getfvec(fifi[0][0],fifi[0][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 #if(k == 2): # print "XXXX" # print fifi[0],nas for i in xrange(1,2): qnom = -1.e10 for j in xrange(-1,2,1): t1,t2,t3 = getfvec(fifi[i][0]+j*qsym,fifi[i][1]) nom = t1*m1 + t2*m2 + t3*m3 if(nom > qnom): qnom = nom n1=t1 n2=t2 n3=t3 #if(k == 2): # print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 print qnom, n1,n2,n3,nas # To get the correct pixer phi angle has to be taken from the above!! pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if(nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2]/nom))%360.0 if(sym>1 and ntheta>90.0): nphi = (degrees(atan2( nas[1], nas[0] ))-180.0)%qsym + 180.0 else: nphi = degrees(atan2( nas[1], nas[0] ))%qsym #print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform([fifi[0][2:],fifi[1][2:]]) avgtrans[q[0][0]][j] = [nphi, ntheta, twod[0], twod[1], twod[2]] perr = {} tgood = 0 for q in blocks: perr[q] = [True]*chunklengths[q] for k in xrange(chunklengths[q]): if(pixer[q][k] > thresherr): perr[q][k] = False if perr[q][k]: tgood += 1 if(tgood < 4): print " No good images within the pixel error threshold specified" exit() print " tgood ", tgood hi = hist_list([pixer[q][k] for q in blocks for k in xrange(chunklengths[q]) ],16) for i in xrange(len(hi[0])): print "%4d %12.3f %12.0f "%(i,hi[0][i],hi[1][i]) # Finished, store average orientation params and table of good images # store lists of good images for each group # blocks is indexed by first letter good = [[] for i in xrange(4)] bad = [[] for i in xrange(4)] for i,q in enumerate(blocks): for k in xrange(chunklengths[q]): #deprt = max_3D_pixel_error(params[lefts[i]][k],avgtrans[q][k],r=radius) if perr[q][k]: good[i].append(chunks[q][k]) #[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k] ]) else: bad[i].append(chunks[q][k]) #[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k]]) write_text_file( good[i], os.path.join(outdir,"newgood%01d.txt"%i) ) write_text_file( bad[i], os.path.join(outdir,"newbad%01d.txt"%i) ) # write out parameters, for those in pairs write out average, for leftouts leave them as they were # These parameters refer to the original X files. ll = 0 for m in xrange(6): if params.has_key(q+chr(48+m)): prmsgood = [] prmsbad = [] try: j = lefts.index(q+chr(48+m)) for k in xrange(chunklengths[q]): if perr[q][k]: prmsgood.append(params[q+chr(48+m)][k]) else: prmsbad.append(params[q+chr(48+m)][k]) except: for k in xrange(chunklengths[q]): if perr[q][k]: prmsgood.append(avgtrans[q][k]) else: prmsbad.append(avgtrans[q][k]) write_text_row(prmsgood, os.path.join(outdir,"params-newgood%01d%01d.txt"%(i,ll))) write_text_row(prmsbad, os.path.join(outdir, "params-newbad%01d%01d.txt"%(i,ll))) ll += 1 # Generate newlili files from newgood, these contain original numbering of the total single file ll = 0 for i in xrange(3): for j in xrange(i+1,4): write_text_file( good[i] + good[j], os.path.join(outdir,"newlili%01d.txt"%ll) ) ll += 1 # write out parameters, for those in pairs write out average, for leftouts leave them as they were # These parameters refer to the original X files. for i in xrange(6): prms = [] for q in blocks: if params.has_key(q+chr(48+i)): try: j = lefts.index(q+chr(48+i)) prms += params[q+chr(48+i)] except: prms += avgtrans[q] write_text_row(prms, os.path.join(outdir,outgrouparms+"%01d.txt"%i)) # Write chunklengths chunklengths = [len(good[i]) for i in xrange(4)] write_text_file(chunklengths, os.path.join(outdir,"chunklengths.txt") ) """ # We do not use consecutive numbering anymore # Generate newx files from newgood, these contain consecutive (with gaps) numbering that allows to generate truncated X files from the previous X files ll = 0 for i in xrange(3): firstblock = [] l = len(perr[blocks[i]]) for k in xrange(l): if perr[blocks[i]][k]: firstblock.append(k) for j in xrange(i+1,4): secondblock = [] for k in xrange(len(perr[blocks[j]])): if perr[blocks[j]][k]: secondblock.append(k+l) write_text_file( firstblock + secondblock, os.path.join(outdir,"goodX%01d.txt"%ll) ) ll += 1 del good,bad,firstblock,secondblock,perr """ """ 0 1 2 3 4 5 A A A = = = B = = B B = = C = C = C = = D = D D """ elif options.phase == 4 and len(args) == 1: outdir = args[0] bp = 'badparams' #outgrouparms= args[1] radius = options.ou thresherr = options.thresherr sym = int(options.sym[1:]) qsym = 360.0/sym #params = [[None for i in xrange(3)] for j in xrange(4)] ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general. for jj in xrange(4): params = [None for ii in xrange(ll)] newbad = map(int, read_text_file(options.params+"%01d.txt"%jj) ) nn = len(newbad) for ii in xrange(ll): params[ii] = read_text_row(os.path.join(outdir,bp+"%01d%01d.txt"%(jj,ii))) assert(nn == len(params[ii]) ) # Compute average projection params and pixel errors avgtrans = [None]*nn pixer = [0.0]*nn for j in xrange(nn): nas = [0.0,0.0,0.0] if( sym == 1): #pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in xrange(ll): n1,n2,n3 = getfvec(params[i][j][0],params[i][j][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1,m2,m3 = getfvec(params[0][j][0],params[0][j][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 for i in xrange(1,ll): qnom = -1.e10 for j in xrange(-1,2,1): t1,t2,t3 = getfvec(params[i][j][0]+j*qsym,params[i][j][1]) nom = t1*m1 + t2*m2 + t3*m3 if(nom > qnom): qnom = nom n1=t1 n2=t2 n3=t3 #if(k == 2): # print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 print qnom, n1,n2,n3,nas # To get the correct pixer phi angle has to be taken from the above!! nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if(nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2]/nom))%360.0 if(sym>1 and ntheta>90.0): nphi = (degrees(atan2( nas[1], nas[0] ))-180.0)%qsym + 180.0 else: nphi = degrees(atan2( nas[1], nas[0] ))%qsym #print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform([params[ii][j][2:] for ii in xrange(ll)]) avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]] perr = errors_per_image(params, avgtrans, thresherr, radius ) rescued = [] rejects = [] for j in xrange(nn): #print chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)] if( perr[j][0] <= thresherr ): rescued.append([newbad[j],j]) else: rejects.append([newbad[j],j]) if( len(rescued) == 0 ): write_text_row([-1,-1], os.path.join(outdir,"rescued%01d.txt"%jj)) else: write_text_row(rescued, os.path.join(outdir,"rescued%01d.txt"%jj)) # We also have to write params. if( len(rescued) != 0 ): for ii in xrange(ll): write_text_row([params[ii][rescued[k][1]] for k in xrange(len(rescued))] , os.path.join(outdir,"params-rescued%01d%01d.txt"%(jj,ii))) if( len(rejects) == 0 ): write_text_row([-1,-1], os.path.join(outdir,'rejects'+"%01d.txt"%jj)) else: write_text_row(rejects, os.path.join(outdir,'rejects'+"%01d.txt"%jj)) if( len(rejects) != 0 ): for ii in xrange(ll): write_text_row([params[ii][rejects[k][1]] for k in xrange(len(rejects))] , os.path.join(outdir,"params-rejects%01d%01d.txt"%(jj,ii))) hi = hist_list([perr[j][0] for j in xrange(nn) ], 16) print "Pixel errors for BAD GROUP ",chr(65+jj) for ii in xrange(len(hi[0])): print "%4d %12.3f %12.0f "%(ii,hi[0][ii],hi[1][ii]) elif options.phase == 5 and len(args) == 1: # This version is for meridien refinement. There are simply three full sets of params. outdir = args[0] bp = 'badparams' #outgrouparms= args[1] radius = options.ou thresherr = options.thresherr sym = int(options.sym[1:]) qsym = 360.0/sym #params = [[None for i in xrange(3)] for j in xrange(4)] ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general. for jj in xrange(1): params = [None for ii in xrange(ll)] #newbad = map(int, read_text_file(options.params+"%01d.txt"%jj) ) #nn = len(newbad) for ii in xrange(ll): params[ii] = read_text_row(os.path.join(outdir,"params%01d.txt"%(ii))) #assert(nn == len(params[ii]) ) nn = len(params[0]) newbad = range(nn) # Compute average projection params and pixel errors avgtrans = [None]*nn pixer = [0.0]*nn for j in xrange(nn): nas = [0.0,0.0,0.0] if( sym == 1): #pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in xrange(ll): n1,n2,n3 = getfvec(params[i][j][0],params[i][j][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1,m2,m3 = getfvec(params[0][j][0],params[0][j][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 for i in xrange(1,ll): qnom = -1.e10 for j in xrange(-1,2,1): t1,t2,t3 = getfvec(params[i][j][0]+j*qsym,params[i][j][1]) nom = t1*m1 + t2*m2 + t3*m3 if(nom > qnom): qnom = nom n1=t1 n2=t2 n3=t3 #if(k == 2): # print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 print qnom, n1,n2,n3,nas # To get the correct pixer phi angle has to be taken from the above!! nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if(nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2]/nom))%360.0 if(sym>1 and ntheta>90.0): nphi = (degrees(atan2( nas[1], nas[0] ))-180.0)%qsym + 180.0 else: nphi = degrees(atan2( nas[1], nas[0] ))%qsym #print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform([params[ii][j][2:] for ii in xrange(ll)]) avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]] write_text_row(avgtrans, os.path.join(outdir,"avgtrans.txt")) perr = errors_per_image(params, avgtrans, thresherr, radius ) rescued = [] rejects = [] for j in xrange(nn): #print chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)] if( perr[j][0] <= thresherr ): rescued.append([newbad[j],j]) else: rejects.append([newbad[j],j]) if( len(rescued) == 0 ): write_text_row([-1,-1], os.path.join(outdir,"rescued%01d.txt"%jj)) else: write_text_row(rescued, os.path.join(outdir,"rescued%01d.txt"%jj)) # We also have to write params. if( len(rescued) != 0 ): for ii in xrange(ll): write_text_row([params[ii][rescued[k][1]] for k in xrange(len(rescued))] , os.path.join(outdir,"params-rescued%01d%01d.txt"%(jj,ii))) if( len(rejects) == 0 ): write_text_row([-1,-1], os.path.join(outdir,'rejects'+"%01d.txt"%jj)) else: write_text_row(rejects, os.path.join(outdir,'rejects'+"%01d.txt"%jj)) if( len(rejects) != 0 ): for ii in xrange(ll): write_text_row([params[ii][rejects[k][1]] for k in xrange(len(rejects))] , os.path.join(outdir,"params-rejects%01d%01d.txt"%(jj,ii))) hi = hist_list([perr[j][0] for j in xrange(nn) ], 16) print "Pixel errors for BAD GROUP ",chr(65+jj) for ii in xrange(len(hi[0])): print "%4d %12.3f %12.0f "%(ii,hi[0][ii],hi[1][ii]) else: print "Usage: " print """ Phase 1: sxconsistency.py --phase=1 bdb:data outdir output files are: in directory outdir: lili0.txt to lili5.txt contain indices of images in resampled six groups bdb:dataX0 to bdb:dataX5 are metafiles containing six resampled groups of images derived from bdb:data Phase 2 sxconsistency.py --phase=2 outdir --ndigits=1 --params=paramsb howmanythesame.txt outdir - directory containing files lili0.txt to lili5.txt produced in phase 1 --params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be master/main/params0.txt to master/main/params5.txt output files: howmanythesame.txt - contains one number, a ratio of number of images that did not change orientations to the total number of images Phase 3: sxconsistency.py --phase=3 outdir --ou=133 --thresherr=3.0 --params=paramsa outgrouparms input files: outdir - directory containing files lili0.txt to lili5.txt produced in phase 1 --params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be master/main/params0.txt to master/main/params5.txt output files: outgrouparms*.txt - Root of of six parameters files with average 3D orientation parameters computed from six runs, can be imported into bdb:data The next two files contain the original image numbers refering to the top bdb. outdir/newgood.txt - Text file with indices of images whose orientation parameters arrors are below specified thresherr (to be kept) outdir/bad.txt - Text file with indices of images whose orientation parameters arrors are above specified thresherr (to be rejected) Phase 4: sxconsistency.py --phase=4 outdir --ou=133 --thresherr=3.0 --params=master/main001/newbad input files: outdir - directory containing files badparamsij.txt i= 0,3, j=0,2, which resulted from three alignments of four badchunk images --params= - Root of four files with original indices of bad images, they will be read and a subset corresponding to rescued ones will be outputed output files: rescued*.txt - four files with indices of accepted images from badchunk. rejects*.txt - four files with indices of rejected images from badchunk. There are two columns: [number in original bdb:chunk, number in bdb:newbad] """ print "Please run '" + progname + " -h' for detailed options" global_def.BATCH = False