def __new__(cls, file_name): if not file_exists(file_name): raise splt = file_name.split(".") if len(file_name) > 4 and file_name[:4] == "bdb:": pass elif EMUtil.get_image_ext_type(splt[-1]) == EMUtil.ImageType.IMAGE_UNKNOWN: raise if splt[-1] in ["img","hed"]: return EMImagicTmpFileHandle(file_name) elif file_name[:4] == "bdb:": return EMDBTmpFileHandle(file_name) else: return EMGeneralTmpFileHandle(file_name)
def main(): progname = os.path.basename(sys.argv[0]) usage = """%prog [options] <image1> <image2> <image3> <image4> .... WARNING: Experimental program. Contact [email protected] for more info. Currently only supports bootstrapping an initial probe doing all versus all alignment of the input images """ parser = EMArgumentParser(usage=usage,version=EMANVERSION) parser.add_argument("--align",type=str,help="The aligner and its parameters. e.g. --align=rt.3d.grid:ralt=180:dalt=10:dphi=10:rphi=180:search=5", default="rt.3d.grid") parser.add_argument("--aligncmp",type=str,help="The comparator used for determing the best initial alignment", default="dot.tomo:threshold=0") parser.add_argument("--cmp",type=str,help="The comparator used to obtain the final similarity", default="dot.tomo:threshold=0") parser.add_argument("--ralign",type=str,help="This is the second stage aligner used to refine the first alignment. This is usually the \'refine\' aligner.", default=None) parser.add_argument("--raligncmp",type=str,help="The comparator used for determing the refined alignment", default="dot.tomo:threshold=0") parser.add_argument("--bootstrap",action="store_true",default=True,help="Boot strap alignment") parser.add_argument("--output",type=str,default="e2tomoave.hdf",help="The output image which will store the results matrix") parser.add_argument("--parallel",type=str,default=None,help="Use parallelism") parser.add_argument("--path", default=None, type=str,help="The name of a directory where results are placed. If unspecified will generate one automatically of type tomo_ave_??.") parser.add_argument("--nsoln", default=1, type=int,help="If supplied and greater than 1, the nsoln-best alignments will be written to a text file. This is useful for debug but may be left unspecified") parser.add_argument("--dbls",type=str,help="data base list storage, used by the workflow. You can ignore this argument.",default=None) parser.add_argument("--shrink",type=int,help="Shrink the data as part of the alignment - for speed purposes but at the potential loss of accuracy",default=None) parser.add_argument("--filter",type=str,help="The name and parameters of an EMAN2 processor. Will be applied prior to shrinking.",default=None) parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1) parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness") if EMUtil.cuda_available(): parser.add_argument("--cuda",action="store_true",help="GPU acceleration using CUDA. Experimental", default=False) (options, args) = parser.parse_args() print options.shrink error_messages = check_options(options,args) if len(error_messages) != 0: msg = "\n" for error in error_messages: msg += error +"\n" parser.error(msg) exit(1) logger=E2init(sys.argv,options.ppid) if options.bootstrap: module = EMBootStrappedAverages(args,options,logger) module.execute() else: print "boot strap only supported technique" E2end(logger)
def __init__(self,files,options,logger): ''' @param options - the options returned by the call to (options, args) = parser.parse_args() @param args - a list of image names - args is that which is returned by the call to (options, args) = parser.parse_args() ''' self.options = options self.files = files self.logger = logger self.images = None self.using_cuda = EMUtil.cuda_available() and options.cuda self.current_files = None self.jobs_completed = 0 self.total_jobs = 0 # you can never know how many jobs will need to be completed self.current_progress = 0.0
def __init__(self, files, options, logger): ''' @param options - the options returned by the call to (options, args) = parser.parse_args() @param args - a list of image names - args is that which is returned by the call to (options, args) = parser.parse_args() ''' self.options = options self.files = files self.logger = logger self.images = None self.using_cuda = EMUtil.cuda_available() and options.cuda self.current_files = None self.jobs_completed = 0 self.total_jobs = 0 # you can never know how many jobs will need to be completed self.current_progress = 0.0
def stackToHDF(infile,outfile,apix,pinfo=None): """ convert stack to an hdf stack pinfo contains CTF information from apFrealign.getPerParticleCTF pinfo may also contain helical information """ from utilities import generate_ctf a = EMAN2.EMData() imn = EMUtil.get_image_count(infile) if pinfo is not None: if len(pinfo) != imn: apDisplay.printError("insufficient particle info for stack") # output must end with hdf outf,ext = os.path.splitext(outfile) if ext != '.hdf': outstack = outf+".hdf" apDisplay.printMsg("Generating '%s' with %i particles"%(outstack,imn)) for i in xrange(imn): a.read_image(infile,i) a.set_attr_dict({'active':1}) t2 = EMAN2.Transform({"type":"spider","phi":0,"theta":0,"psi":0}) a.set_attr("xform.projection", t2) a.set_attr("apix_x",apix) if pinfo is not None: pdata = pinfo[i+1] df1 = pdata['df1'] df2 = pdata['df2'] astig = pdata['angastig'] kv = pdata['kev'] cs = pdata['cs'] ampc = pdata['ampc']*100 # save CTF dict (bfactor is hard coded to 0.0) df=(float(df1)+float(df2))/2 ctfgen = generate_ctf([df,cs,kv,apix,0.0,ampc]) a.set_attr("ctf",ctfgen) # if helical info is present if pdata['hnum'] is not None: a.set_attr("h_angle",pdata['hangle']) a.write_image(outstack,i) return outstack
def __init__(self,file_name): ''' @param file_name the name of the file that which is being overwritten @exception raised if the file_name does not exist on the file system @exception raised if the file_name has a unrecognized (or no) type ''' if not os.path.exists(file_name): raise splt = file_name.split(".") if len(splt) < 2: raise if EMUtil.get_image_ext_type(splt[-1]) == EMUtil.ImageType.IMAGE_UNKNOWN: raise self.__file_ext = "."+splt[-1] EMTmpFileHandleBase.__init__(self) self.__file_name = get_file_tag(file_name) self.__full_file_name = file_name self.__tmp_file_name = None self.__establish_tmp_file_name()
def __init__(self,file_name): ''' @param file_name the name of the file that which is being overwritten @exception raised if the file_name does not exist on the file system @exception raised if the file_name has a unrecognized (or no) type ''' if not os.path.exists(file_name): raise splt = file_name.split(".") if len(splt) < 2: raise if EMUtil.get_image_ext_type(splt[-1]) == EMUtil.ImageType.IMAGE_UNKNOWN: raise self.__file_ext = "."+splt[-1] EMTmpFileHandleBase.__init__(self) self.__file_name = base_name(file_name) self.__full_file_name = file_name self.__tmp_file_name = None self.__establish_tmp_file_name()
def main(args): if os.path.exists(args.output): os.remove(args.output) for fn in args.input: if fn.endswith(".star"): star = parse_star(fn, keep_index=False) for p in star["rlnImageName"]: stack = p.split("@")[1] idx = int(p.split("@")[0]) - 1 try: img = EMData(stack, idx) img.append_image(args.output) except Exception: print("Error at %s" % p) else: n = EMUtil.get_image_count(fn) for i in range(n): img = EMData(fn, i) img.append_image(args.output) return 0
def main(): arglist = [] for arg in sys.argv: arglist.append( arg ) progname = os.path.basename( arglist[0] ) usage = progname + " prj_stack volume [begin end step] --CTF --npad=ntimes_padding --list=file --group=ID --snr=SNR --sym=symmetry --verbose=(0|1) --xysize --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--CTF", action="store_true", default=False, help="apply CTF correction") parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio" ) parser.add_option("--sym", type="string", default="c1", help="symmetry" ) parser.add_option("--list", type="string", help="file with list of images to be used in the first column" ) parser.add_option("--group", type="int", default=-1, help="perform reconstruction using images for a given group number (group is attribute in the header)" ) parser.add_option("--MPI", action="store_true", default=False, help="use MPI version ") parser.add_option("--npad", type="int", default=2, help="number of times padding (default 2)" ) parser.add_option("--verbose", type="int", default=0, help="verbose level: 0 no verbose, 1 verbose" ) parser.add_option("--xysize", type="int", default=-1, help="user expected size at xy direction" ) parser.add_option("--zsize", type="int", default=-1, help="user expected size at z direction" ) parser.add_option("--smearstep", type="float", default=0.0, help="Rotational smear step (default 0.0, no smear)" ) (options,args) = parser.parse_args(arglist[1:]) if options.MPI: from mpi import mpi_init sys.argv = mpi_init(len(sys.argv), sys.argv) if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() if len(args) == 2: prj_stack = args[0] vol_stack = args[1] nimage = EMUtil.get_image_count( prj_stack ) pid_list = range(0, nimage) elif len(args) == 5: prj_stack = args[0] vol_stack = args[1] begin = atoi( args[2] ) end = atoi( args[3] ) step = atoi( args[4] ) pid_list = range(begin, end, step) else: ERROR("incomplete list of arguments","recon3d_n",1) exit() if(options.list and options.group > -1): ERROR("options group and list cannot be used together","recon3d_n",1) sys.exit() from applications import recons3d_n global_def.BATCH = True recons3d_n(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\ options.sym, options.list, options.group, options.verbose, options.MPI,options.xysize, options.zsize, options.smearstep) global_def.BATCH = False if options.MPI: from mpi import mpi_finalize mpi_finalize()
def main(): def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror): # the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D if mirror: m = 1 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0) else: m = 0 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0) return alpha, sx, sy, m progname = os.path.basename(sys.argv[0]) usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl= --aa= --sym=symmetry --CTF" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--output_dir", type="string" , default="./", help="Output directory") parser.add_option("--ave2D", type="string" , default=False, help="Write to the disk a stack of 2D averages") parser.add_option("--var2D", type="string" , default=False, help="Write to the disk a stack of 2D variances") parser.add_option("--ave3D", type="string" , default=False, help="Write to the disk reconstructed 3D average") parser.add_option("--var3D", type="string" , default=False, help="Compute 3D variability (time consuming!)") parser.add_option("--img_per_grp", type="int" , default=100, help="Number of neighbouring projections.(Default is 100)") parser.add_option("--no_norm", action="store_true", default=False, help="Do not use normalization.(Default is to apply normalization)") #parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" ) parser.add_option("--npad", type="int" , default=2 , help="Number of time to pad the original images.(Default is 2 times padding)") parser.add_option("--sym" , type="string" , default="c1", help="Symmetry. (Default is no symmetry)") parser.add_option("--fl", type="float" , default=0.0, help="Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)") parser.add_option("--aa", type="float" , default=0.02 , help="Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)") parser.add_option("--CTF", action="store_true", default=False, help="Use CFT correction.(Default is no CTF correction)") #parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version") #parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" ) #parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" ) #parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" ) #parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" ) parser.add_option("--VAR" , action="store_true", default=False, help="Stack of input consists of 2D variances (Default False)") parser.add_option("--decimate", type ="float", default=0.25, help="Image decimate rate, a number less than 1. (Default is 0.25)") parser.add_option("--window", type ="int", default=0, help="Target image size relative to original image size. (Default value is zero.)") #parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)") #parser.add_option("--nvec", type="int" , default=0 , help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)") parser.add_option("--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)") parser.add_option("--overhead", type ="float", default=0.5, help="python overhead per CPU.") (options,args) = parser.parse_args() ##### from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX #from mpi import * from applications import MPI_start_end from reconstruction import recons3d_em, recons3d_em_MPI from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI from utilities import print_begin_msg, print_end_msg, print_msg from utilities import read_text_row, get_image, get_im, wrap_mpi_send, wrap_mpi_recv from utilities import bcast_EMData_to_all, bcast_number_to_all from utilities import get_symt # This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015 from EMAN2db import db_open_dict # Set up global variables related to bdb cache if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() # Set up global variables related to ERROR function global_def.BATCH = True # detect if program is running under MPI RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ if RUNNING_UNDER_MPI: global_def.MPI = True if options.output_dir =="./": current_output_dir = os.path.abspath(options.output_dir) else: current_output_dir = options.output_dir if options.symmetrize : if RUNNING_UNDER_MPI: try: sys.argv = mpi_init(len(sys.argv), sys.argv) try: number_of_proc = mpi_comm_size(MPI_COMM_WORLD) if( number_of_proc > 1 ): ERROR("Cannot use more than one CPU for symmetry preparation","sx3dvariability",1) except: pass except: pass if not os.path.exists(current_output_dir): os.mkdir(current_output_dir) # Input #instack = "Clean_NORM_CTF_start_wparams.hdf" #instack = "bdb:data" from logger import Logger,BaseLogger_Files if os.path.exists(os.path.join(current_output_dir, "log.txt")): os.remove(os.path.join(current_output_dir, "log.txt")) log_main=Logger(BaseLogger_Files()) log_main.prefix = os.path.join(current_output_dir, "./") instack = args[0] sym = options.sym.lower() if( sym == "c1" ): ERROR("There is no need to symmetrize stack for C1 symmetry","sx3dvariability",1) line ="" for a in sys.argv: line +=" "+a log_main.add(line) if(instack[:4] !="bdb:"): #if output_dir =="./": stack = "bdb:data" stack = "bdb:"+current_output_dir+"/data" delete_bdb(stack) junk = cmdexecute("sxcpy.py "+instack+" "+stack) else: stack = instack qt = EMUtil.get_all_attributes(stack,'xform.projection') na = len(qt) ts = get_symt(sym) ks = len(ts) angsa = [None]*na for k in range(ks): #Qfile = "Q%1d"%k #if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k) Qfile = os.path.join(current_output_dir, "Q%1d"%k) #delete_bdb("bdb:Q%1d"%k) delete_bdb("bdb:"+Qfile) #junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k) junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:"+Qfile) #DB = db_open_dict("bdb:Q%1d"%k) DB = db_open_dict("bdb:"+Qfile) for i in range(na): ut = qt[i]*ts[k] DB.set_attr(i, "xform.projection", ut) #bt = ut.get_params("spider") #angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]] #write_text_row(angsa, 'ptsma%1d.txt'%k) #junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k) #junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k)) DB.close() #if options.output_dir =="./": delete_bdb("bdb:sdata") delete_bdb("bdb:" + current_output_dir + "/"+"sdata") #junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q") sdata = "bdb:"+current_output_dir+"/"+"sdata" print(sdata) junk = cmdexecute("e2bdb.py " + current_output_dir +" --makevstack="+sdata +" --filt=Q") #junk = cmdexecute("ls EMAN2DB/sdata*") #a = get_im("bdb:sdata") a = get_im(sdata) a.set_attr("variabilitysymmetry",sym) #a.write_image("bdb:sdata") a.write_image(sdata) else: from fundamentals import window2d sys.argv = mpi_init(len(sys.argv), sys.argv) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) main_node = 0 shared_comm = mpi_comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL) myid_on_node = mpi_comm_rank(shared_comm) no_of_processes_per_group = mpi_comm_size(shared_comm) masters_from_groups_vs_everything_else_comm = mpi_comm_split(MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node) color, no_of_groups, balanced_processor_load_on_nodes = get_colors_and_subsets(main_node, MPI_COMM_WORLD, myid, \ shared_comm, myid_on_node, masters_from_groups_vs_everything_else_comm) overhead_loading = options.overhead*number_of_proc #memory_per_node = options.memory_per_node #if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group keepgoing = 1 current_window = options.window current_decimate = options.decimate if len(args) == 1: stack = args[0] else: print(( "usage: " + usage)) print(( "Please run '" + progname + " -h' for detailed options")) return 1 t0 = time() # obsolete flags options.MPI = True #options.nvec = 0 options.radiuspca = -1 options.iter = 40 options.abs = 0.0 options.squ = 0.0 if options.fl > 0.0 and options.aa == 0.0: ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid) #if options.VAR and options.SND: # ERROR("Only one of var and SND can be set!", "sx3dvariability", myid) if options.VAR and (options.ave2D or options.ave3D or options.var2D): ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid) #if options.SND and (options.ave2D or options.ave3D): # ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid) #if options.nvec > 0 : # ERROR("PCA option not implemented", "sx3dvariability", 1, myid) #if options.nvec > 0 and options.ave3D == None: # ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", 1, myid) if current_decimate>1.0 or current_decimate<0.0: ERROR("Decimate rate should be a value between 0.0 and 1.0", "sx3dvariability", 1, myid) if current_window < 0.0: ERROR("Target window size should be always larger than zero", "sx3dvariability", 1, myid) if myid == main_node: img = get_image(stack, 0) nx = img.get_xsize() ny = img.get_ysize() if(min(nx, ny) < current_window): keepgoing = 0 keepgoing = bcast_number_to_all(keepgoing, main_node, MPI_COMM_WORLD) if keepgoing == 0: ERROR("The target window size cannot be larger than the size of decimated image", "sx3dvariability", 1, myid) import string options.sym = options.sym.lower() # if global_def.CACHE_DISABLE: # from utilities import disable_bdb_cache # disable_bdb_cache() # global_def.BATCH = True if myid == main_node: if not os.path.exists(current_output_dir): os.mkdir(current_output_dir)# Never delete output_dir in the program! img_per_grp = options.img_per_grp #nvec = options.nvec radiuspca = options.radiuspca from logger import Logger,BaseLogger_Files #if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt")) log_main=Logger(BaseLogger_Files()) log_main.prefix = os.path.join(current_output_dir, "./") if myid == main_node: line = "" for a in sys.argv: line +=" "+a log_main.add(line) log_main.add("-------->>>Settings given by all options<<<-------") log_main.add("Symmetry : %s"%options.sym) log_main.add("Input stack : %s"%stack) log_main.add("Output_dir : %s"%current_output_dir) if options.ave3D: log_main.add("Ave3d : %s"%options.ave3D) if options.var3D: log_main.add("Var3d : %s"%options.var3D) if options.ave2D: log_main.add("Ave2D : %s"%options.ave2D) if options.var2D: log_main.add("Var2D : %s"%options.var2D) if options.VAR: log_main.add("VAR : True") else: log_main.add("VAR : False") if options.CTF: log_main.add("CTF correction : True ") else: log_main.add("CTF correction : False ") log_main.add("Image per group : %5d"%options.img_per_grp) log_main.add("Image decimate rate : %4.3f"%current_decimate) log_main.add("Low pass filter : %4.3f"%options.fl) current_fl = options.fl if current_fl == 0.0: current_fl = 0.5 log_main.add("Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"%round((current_fl*current_decimate),3)) log_main.add("Window size : %5d "%current_window) log_main.add("sx3dvariability begins") symbaselen = 0 if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() nnxo = nx nnyo = ny if options.sym != "c1" : imgdata = get_im(stack) try: i = imgdata.get_attr("variabilitysymmetry").lower() if(i != options.sym): ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", 1, myid) except: ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", 1, myid) from utilities import get_symt i = len(get_symt(options.sym)) if((nima/i)*i != nima): ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", 1, myid) symbaselen = nima/i else: symbaselen = nima else: nima = 0 nx = 0 ny = 0 nnxo = 0 nnyo = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) nnxo = bcast_number_to_all(nnxo) nnyo = bcast_number_to_all(nnyo) if current_window > max(nx, ny): ERROR("Window size is larger than the original image size", "sx3dvariability", 1) if current_decimate == 1.: if current_window !=0: nx = current_window ny = current_window else: if current_window == 0: nx = int(nx*current_decimate+0.5) ny = int(ny*current_decimate+0.5) else: nx = int(current_window*current_decimate+0.5) ny = nx symbaselen = bcast_number_to_all(symbaselen) # check FFT prime number from fundamentals import smallprime is_fft_friendly = (nx == smallprime(nx)) if not is_fft_friendly: if myid == main_node: log_main.add("The target image size is not a product of small prime numbers") log_main.add("Program adjusts the input settings!") ### two cases if current_decimate == 1.: nx = smallprime(nx) ny = nx current_window = nx # update if myid == main_node: log_main.add("The window size is updated to %d."%current_window) else: if current_window == 0: nx = smallprime(int(nx*current_decimate+0.5)) current_decimate = float(nx)/nnxo ny = nx if (myid == main_node): log_main.add("The decimate rate is updated to %f."%current_decimate) else: nx = smallprime(int(current_window*current_decimate+0.5)) ny = nx current_window = int(nx/current_decimate+0.5) if (myid == main_node): log_main.add("The window size is updated to %d."%current_window) if myid == main_node: log_main.add("The target image size is %d"%nx) if radiuspca == -1: radiuspca = nx/2-2 if myid == main_node: log_main.add("%-70s: %d\n"%("Number of projection", nima)) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) """ if options.SND: from projection import prep_vol, prgs from statistics import im_diff from utilities import get_im, model_circle, get_params_proj, set_params_proj from utilities import get_ctf, generate_ctf from filter import filt_ctf imgdata = EMData.read_images(stack, range(img_begin, img_end)) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) bcast_EMData_to_all(vol, myid) volft, kb = prep_vol(vol) mask = model_circle(nx/2-2, nx, ny) varList = [] for i in xrange(img_begin, img_end): phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin]) ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y]) if options.CTF: ctf_params = get_ctf(imgdata[i-img_begin]) ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params)) diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask) diff2 = diff*diff set_params_proj(diff2, [phi, theta, psi, s2x, s2y]) varList.append(diff2) mpi_barrier(MPI_COMM_WORLD) """ if options.VAR: # 2D variance images have no shifts #varList = EMData.read_images(stack, range(img_begin, img_end)) from EMAN2 import Region for index_of_particle in range(img_begin,img_end): image = get_im(stack, index_of_proj) if current_window > 0: varList.append(fdecimate(window2d(image,current_window,current_window), nx,ny)) else: varList.append(fdecimate(image, nx,ny)) else: from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2 from utilities import model_blank, nearest_proj, model_circle, write_text_row, wrap_mpi_gatherv from applications import pca from statistics import avgvar, avgvar_ctf, ccc from filter import filt_tanl from morphology import threshold, square_root from projection import project, prep_vol, prgs from sets import Set from utilities import wrap_mpi_recv, wrap_mpi_bcast, wrap_mpi_send import numpy as np if myid == main_node: t1 = time() proj_angles = [] aveList = [] tab = EMUtil.get_all_attributes(stack, 'xform.projection') for i in range(nima): t = tab[i].get_params('spider') phi = t['phi'] theta = t['theta'] psi = t['psi'] x = theta if x > 90.0: x = 180.0 - x x = x*10000+psi proj_angles.append([x, t['phi'], t['theta'], t['psi'], i]) t2 = time() log_main.add( "%-70s: %d\n"%("Number of neighboring projections", img_per_grp)) log_main.add("...... Finding neighboring projections\n") log_main.add( "Number of images per group: %d"%img_per_grp) log_main.add( "Now grouping projections") proj_angles.sort() proj_angles_list = np.full((nima, 4), 0.0, dtype=np.float32) for i in range(nima): proj_angles_list[i][0] = proj_angles[i][1] proj_angles_list[i][1] = proj_angles[i][2] proj_angles_list[i][2] = proj_angles[i][3] proj_angles_list[i][3] = proj_angles[i][4] else: proj_angles_list = 0 proj_angles_list = wrap_mpi_bcast(proj_angles_list, main_node, MPI_COMM_WORLD) proj_angles = [] for i in range(nima): proj_angles.append([proj_angles_list[i][0], proj_angles_list[i][1], proj_angles_list[i][2], int(proj_angles_list[i][3])]) del proj_angles_list proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end)) all_proj = Set() for im in proj_list: for jm in im: all_proj.add(proj_angles[jm][3]) all_proj = list(all_proj) index = {} for i in range(len(all_proj)): index[all_proj[i]] = i mpi_barrier(MPI_COMM_WORLD) if myid == main_node: log_main.add("%-70s: %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2)) log_main.add("%-70s: %d\n"%("Number of groups processed on the main node", len(proj_list))) log_main.add("Grouping projections took: %12.1f [m]"%((time()-t2)/60.)) log_main.add("Number of groups on main node: ", len(proj_list)) mpi_barrier(MPI_COMM_WORLD) if myid == main_node: log_main.add("...... Calculating the stack of 2D variances \n") # Memory estimation. There are two memory consumption peaks # peak 1. Compute ave, var; # peak 2. Var volume reconstruction; # proj_params = [0.0]*(nima*5) aveList = [] varList = [] #if nvec > 0: eigList = [[] for i in range(nvec)] dnumber = len(all_proj)# all neighborhood set for assigned to myid pnumber = len(proj_list)*2. + img_per_grp # aveList and varList tnumber = dnumber+pnumber vol_size2 = nx**3*4.*8/1.e9 vol_size1 = 2.*nnxo**3*4.*8/1.e9 proj_size = nnxo*nnyo*len(proj_list)*4.*2./1.e9 # both aveList and varList orig_data_size = nnxo*nnyo*4.*tnumber/1.e9 reduced_data_size = nx*nx*4.*tnumber/1.e9 full_data = np.full((number_of_proc, 2), -1., dtype=np.float16) full_data[myid] = orig_data_size, reduced_data_size if myid != main_node: wrap_mpi_send(full_data, main_node, MPI_COMM_WORLD) if myid == main_node: for iproc in range(number_of_proc): if iproc != main_node: dummy = wrap_mpi_recv(iproc, MPI_COMM_WORLD) full_data[np.where(dummy>-1)] = dummy[np.where(dummy>-1)] del dummy mpi_barrier(MPI_COMM_WORLD) full_data = wrap_mpi_bcast(full_data, main_node, MPI_COMM_WORLD) # find the CPU with heaviest load minindx = np.argsort(full_data, 0) heavy_load_myid = minindx[-1][1] total_mem = sum(full_data) if myid == main_node: if current_window == 0: log_main.add("Nx: current image size = %d. Decimated by %f from %d"%(nx, current_decimate, nnxo)) else: log_main.add("Nx: current image size = %d. Windowed to %d, and decimated by %f from %d"%(nx, current_window, current_decimate, nnxo)) log_main.add("Nproj: number of particle images.") log_main.add("Navg: number of 2D average images.") log_main.add("Nvar: number of 2D variance images.") log_main.add("Img_per_grp: user defined image per group for averaging = %d"%img_per_grp) log_main.add("Overhead: total python overhead memory consumption = %f"%overhead_loading) log_main.add("Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"%\ (total_mem[1] + overhead_loading)) del full_data mpi_barrier(MPI_COMM_WORLD) if myid == heavy_load_myid: log_main.add("Begin reading and preprocessing images on processor. Wait... ") ttt = time() #imgdata = EMData.read_images(stack, all_proj) imgdata = [ None for im in range(len(all_proj))] for index_of_proj in range(len(all_proj)): #image = get_im(stack, all_proj[index_of_proj]) if( current_window > 0): imgdata[index_of_proj] = fdecimate(window2d(get_im(stack, all_proj[index_of_proj]),current_window,current_window), nx, ny) else: imgdata[index_of_proj] = fdecimate(get_im(stack, all_proj[index_of_proj]), nx, ny) if (current_decimate> 0.0 and options.CTF): ctf = imgdata[index_of_proj].get_attr("ctf") ctf.apix = ctf.apix/current_decimate imgdata[index_of_proj].set_attr("ctf", ctf) if myid == heavy_load_myid and index_of_proj%100 == 0: log_main.add(" ...... %6.2f%% "%(index_of_proj/float(len(all_proj))*100.)) mpi_barrier(MPI_COMM_WORLD) if myid == heavy_load_myid: log_main.add("All_proj preprocessing cost %7.2f m"%((time()-ttt)/60.)) log_main.add("Wait untill reading on all CPUs done...") ''' imgdata2 = EMData.read_images(stack, range(img_begin, img_end)) if options.fl > 0.0: for k in xrange(len(imgdata2)): imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) if myid == main_node: vol.write_image("vol_ctf.hdf") print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf")) del vol, imgdata2 mpi_barrier(MPI_COMM_WORLD) ''' from applications import prepare_2d_forPCA from utilities import model_blank from EMAN2 import Transform if not options.no_norm: mask = model_circle(nx/2-2, nx, nx) if options.CTF: from utilities import pad from filter import filt_ctf from filter import filt_tanl if myid == heavy_load_myid: log_main.add("Start computing 2D aveList and varList. Wait...") ttt = time() inner=nx//2-4 outer=inner+2 xform_proj_for_2D = [ None for i in range(len(proj_list))] for i in range(len(proj_list)): ki = proj_angles[proj_list[i][0]][3] if ki >= symbaselen: continue mi = index[ki] dpar = Util.get_transform_params(imgdata[mi], "xform.projection", "spider") phiM, thetaM, psiM, s2xM, s2yM = dpar["phi"],dpar["theta"],dpar["psi"],-dpar["tx"]*current_decimate,-dpar["ty"]*current_decimate grp_imgdata = [] for j in range(img_per_grp): mj = index[proj_angles[proj_list[i][j]][3]] cpar = Util.get_transform_params(imgdata[mj], "xform.projection", "spider") alpha, sx, sy, mirror = params_3D_2D_NEW(cpar["phi"], cpar["theta"],cpar["psi"], -cpar["tx"]*current_decimate, -cpar["ty"]*current_decimate, mirror_list[i][j]) if thetaM <= 90: if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM - cpar["phi"]), 0.0, 0.0, 1.0) else: if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM- cpar["phi"]), 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM - cpar["phi"])), 0.0, 0.0, 1.0) imgdata[mj].set_attr("xform.align2d", Transform({"type":"2D","alpha":alpha,"tx":sx,"ty":sy,"mirror":mirror,"scale":1.0})) grp_imgdata.append(imgdata[mj]) if not options.no_norm: for k in range(img_per_grp): ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False) grp_imgdata[k] -= ave grp_imgdata[k] /= std if options.fl > 0.0: for k in range(img_per_grp): grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) # Because of background issues, only linear option works. if options.CTF: ave, var = aves_wiener(grp_imgdata, SNR = 1.0e5, interpolation_method = "linear") else: ave, var = ave_var(grp_imgdata) # Switch to std dev # threshold is not really needed,it is just in case due to numerical accuracy something turns out negative. var = square_root(threshold(var)) set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0]) set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0]) aveList.append(ave) varList.append(var) xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0] ''' if nvec > 0: eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True) for k in range(nvec): set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0]) eigList[k].append(eig[k]) """ if myid == 0 and i == 0: for k in xrange(nvec): eig[k].write_image("eig.hdf", k) """ ''' if (myid == heavy_load_myid) and (i%100 == 0): log_main.add(" ......%6.2f%% "%(i/float(len(proj_list))*100.)) del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index if not options.no_norm: del mask if myid == main_node: del tab # At this point, all averages and variances are computed mpi_barrier(MPI_COMM_WORLD) if (myid == heavy_load_myid): log_main.add("Computing aveList and varList took %12.1f [m]"%((time()-ttt)/60.)) xform_proj_for_2D = wrap_mpi_gatherv(xform_proj_for_2D, main_node, MPI_COMM_WORLD) if (myid == main_node): write_text_row(xform_proj_for_2D, os.path.join(current_output_dir, "params.txt")) del xform_proj_for_2D mpi_barrier(MPI_COMM_WORLD) if options.ave2D: from fundamentals import fpol from applications import header if myid == main_node: log_main.add("Compute ave2D ... ") km = 0 for i in range(number_of_proc): if i == main_node : for im in range(len(aveList)): aveList[im].write_image(os.path.join(current_output_dir, options.ave2D), km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in range(nl): ave = recv_EMData(i, im+i+70000) """ nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('members', map(int, members)) members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('pix_err', map(float, members)) members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('refprojdir', map(float, members)) """ tmpvol=fpol(ave, nx, nx,1) tmpvol.write_image(os.path.join(current_output_dir, options.ave2D), km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in range(len(aveList)): send_EMData(aveList[im], main_node,im+myid+70000) """ members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) members = aveList[im].get_attr('pix_err') mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) except: mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) """ if myid == main_node: header(os.path.join(current_output_dir, options.ave2D), params='xform.projection', fimport = os.path.join(current_output_dir, "params.txt")) mpi_barrier(MPI_COMM_WORLD) if options.ave3D: from fundamentals import fpol t5 = time() if myid == main_node: log_main.add("Reconstruct ave3D ... ") ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(ave3D, myid) if myid == main_node: if current_decimate != 1.0: ave3D = resample(ave3D, 1./current_decimate) ave3D = fpol(ave3D, nnxo, nnxo, nnxo) # always to the orignal image size set_pixel_size(ave3D, 1.0) ave3D.write_image(os.path.join(current_output_dir, options.ave3D)) log_main.add("Ave3D reconstruction took %12.1f [m]"%((time()-t5)/60.0)) log_main.add("%-70s: %s\n"%("The reconstructed ave3D is saved as ", options.ave3D)) mpi_barrier(MPI_COMM_WORLD) del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList ''' if nvec > 0: for k in range(nvec): if myid == main_node:log_main.add("Reconstruction eigenvolumes", k) cont = True ITER = 0 mask2d = model_circle(radiuspca, nx, nx) while cont: #print "On node %d, iteration %d"%(myid, ITER) eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(eig3D, myid, main_node) if options.fl > 0.0: eig3D = filt_tanl(eig3D, options.fl, options.aa) if myid == main_node: eig3D.write_image(os.path.join(options.outpout_dir, "eig3d_%03d.hdf"%(k, ITER))) Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) ) eig3Df, kb = prep_vol(eig3D) del eig3D cont = False icont = 0 for l in range(len(eigList[k])): phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l]) proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y]) cl = ccc(proj, eigList[k][l], mask2d) if cl < 0.0: icont += 1 cont = True eigList[k][l] *= -1.0 u = int(cont) u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD) icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: u = int(u[0]) log_main.add(" Eigenvector: ",k," number changed ",int(icont[0])) else: u = 0 u = bcast_number_to_all(u, main_node) cont = bool(u) ITER += 1 del eig3Df, kb mpi_barrier(MPI_COMM_WORLD) del eigList, mask2d ''' if options.ave3D: del ave3D if options.var2D: from fundamentals import fpol from applications import header if myid == main_node: log_main.add("Compute var2D...") km = 0 for i in range(number_of_proc): if i == main_node : for im in range(len(varList)): tmpvol=fpol(varList[im], nx, nx,1) tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in range(nl): ave = recv_EMData(i, im+i+70000) tmpvol=fpol(ave, nx, nx,1) tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km) km += 1 else: mpi_send(len(varList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in range(len(varList)): send_EMData(varList[im], main_node, im+myid+70000)# What with the attributes?? mpi_barrier(MPI_COMM_WORLD) if myid == main_node: from applications import header header(os.path.join(current_output_dir, options.var2D), params = 'xform.projection',fimport = os.path.join(current_output_dir, "params.txt")) mpi_barrier(MPI_COMM_WORLD) if options.var3D: if myid == main_node: log_main.add("Reconstruct var3D ...") t6 = time() # radiusvar = options.radius # if( radiusvar < 0 ): radiusvar = nx//2 -3 res = recons3d_4nn_MPI(myid, varList, symmetry = options.sym, npad=options.npad) #res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ) if myid == main_node: from fundamentals import fpol if current_decimate != 1.0: res = resample(res, 1./current_decimate) res = fpol(res, nnxo, nnxo, nnxo) set_pixel_size(res, 1.0) res.write_image(os.path.join(current_output_dir, options.var3D)) log_main.add("%-70s: %s\n"%("The reconstructed var3D is saved as ", options.var3D)) log_main.add("Var3D reconstruction took %f12.1 [m]"%((time()-t6)/60.0)) log_main.add("Total computation time %f12.1 [m]"%((time()-t0)/60.0)) log_main.add("sx3dvariability finishes") from mpi import mpi_finalize mpi_finalize() if RUNNING_UNDER_MPI: global_def.MPI = False global_def.BATCH = False
def main(): global debug progname = os.path.basename(sys.argv[0]) usage = """prog [options] <path or db> ... Various utilities related to BDB databases. examples : e2bdb.py -c Is perhaps the most critical function, as it cleans up the database cache. See the Wiki for more. e2bdb.py <path> -s will list the contents of the database in a directory in bdb: notation e2bdb.py <path> -l Will give useful summary info about stacks in a directory e2bdb.py <database> --dump Gives a mechanism to dump all of the metadata in a database, even if the database contains no images """ parser = EMArgumentParser(usage=usage,version=EMANVERSION) parser.add_argument("--cleanup","-c",action="store_true",default=False,help="This option will clean up the database cache so files can safely be moved or accessed on another computer via NFS.") parser.add_argument("--force","-F",action="store_true",default=False,help="This will force an action that would normally fail due to failed checks.") parser.add_argument("--delete",action="store_true",default=False,help="This will delete (or at least empty) the named database(s)") parser.add_argument("--all","-a",action="store_true",help="List per-particle info",default=False) parser.add_argument("--long","-l",action="store_true",help="Long listing",default=False) parser.add_argument("--short","-s",action="store_true",help="Dense listing of names only",default=False) parser.add_argument("--filt",type=str,help="Only include dictionary names containing the specified string",default=None) parser.add_argument("--filtexclude",type=str,help="Exclude dictionary names containing the specified string",default=None) parser.add_argument("--match",type=str,help="Only include dictionaries matching the provided Python regular expression",default=None) parser.add_argument("--exclude",type=str,help="The name of a database containing a list of exclusion keys",default=None) parser.add_argument("--dump","-D",action="store_true",help="List contents of an entire database, eg 'e2bdb.py -D refine_01#register",default=False) parser.add_argument("--smalldump",action="store_true",help="Lists contents of an entire database, but only list 2 items per dictionary to better see headers",default=False) parser.add_argument("--extractplots",action="store_true",help="If a database contains sets of plots, such as bdb:refine_xx#convergence.results, this will extract the plots as text files.") parser.add_argument("--check",action="store_true",help="Check for self-consistency and errors in the structure of specified databases",default=False) parser.add_argument("--nocache",action="store_true",help="Don't use the database cache for this operation",default=False) parser.add_argument("--merge",action="store_true",help="This will merge the contents of BDB 2-N into BDB 1 (including BDB 1's contents)",default=False) parser.add_argument("--makevstack",type=str,help="Creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks",default=None) parser.add_argument("--appendvstack",type=str,help="Appends to/creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks",default=None) parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness") parser.add_argument("--list",type=str,help="Specify the name of a file with a list of images to use in creation of virtual stacks. Please see source for details.",default=None) parser.add_argument("--exlist",type=str,help="Specify the name of a file with a list of images to exclude in creation of virtual stacks. Please see source for details.",default=None) parser.add_argument("--restore",type=str,help="Write changes in the derived virtual stack back to the original stack",default=None) parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1) parser.add_argument("--checkctf",action="store_true",help="Verfies that all images in the file contain CTF information, and gives some basic statistics",default=False) parser.add_argument("--step",type=str,default="0,1",help="Specify <init>,<step>[,<max>]. Processes only a subset of the input data. For example, 0,2 would process only the even numbered particles") (options, args) = parser.parse_args() if options.nocache : EMAN2db.BDB_CACHE_DISABLE=True if options.cleanup : db_cleanup(options.force) sys.exit(0) try : options.step=int(options.step.split(",")[0]),int(options.step.split(",")[1]),int(options.step.split(",")[2]) # convert strings to tuple except: try: options.step=int(options.step.split(",")[0]),int(options.step.split(",")[1]) except: print "Invalid --step specification" sys.exit(1) if options.all : options.long=1 if len(args)==0 : args.append("bdb:.") logid=0 if options.makevstack : logid=E2init(sys.argv) vstack=db_open_dict(options.makevstack) vstackn=0 elif options.appendvstack : logid=E2init(sys.argv) vstack=db_open_dict(options.appendvstack) vstackn=len(vstack) else : vstack=None if options.merge : print "WARNING: Merge mode\nCombining contents of: ",", ".join(args[1:]) print "into ",args[0] if raw_input("Proceed (y/n) :").lower() != "y" : print "Aborting" sys.exit(1) for i,path in enumerate(args): if path.lower()[:4]=="bdb:" and not "#" in path : path="bdb:.#"+path[4:] if path.lower()[:4]!="bdb:" : path="bdb:"+path if i==0 : outdb=db_open_dict(path) continue indb=db_open_dict(path,True) for k in indb.keys(): outdb[k]=indb[k] print "Merging complete" sys.exit(0) for path in args: if path.lower()[:4]=="bdb:" and not "#" in path : uu = os.path.split(path) if(uu[0] == ''): path="bdb:.#"+path[4:] else: path=uu[0]+"#"+uu[1] if path.lower()[:4]!="bdb:" : path="bdb:"+path if '#' in path : if len(args)>1 : print "\n",path,":" path,dbs=path.rsplit("#",1) path+="#" dbs=[dbs] else: if not '#' in path and path[-1]!='/' : path+='#' if len(args)>1 : print "\n",path[:-1],":" dbs=db_list_dicts(path) dbs.sort() if options.filt: dbs=[db for db in dbs if options.filt in db] if options.filtexclude: dbs=[db for db in dbs if options.filtexclude not in db] if options.match!=None: dbs=[db for db in dbs if re.match(options.match,db)] if options.list : if options.makevstack==None and options.appendvstack==None : print "ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart" sys.exit(1) vdata=open(options.list,'r').readlines() n=len(vdata[0].split()) slist=[] for line in vdata: line=line.split() for i in xrange(n): val=int(line[i]) slist.append(val) del n,val,vdata if options.exlist : if options.makevstack==None: print "ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart" sys.exit(1) vdata=open(options.exlist,'r').readlines() n=len(vdata[0].split()) slist=[] for line in vdata: line=line.split() for i in xrange(n): val=int(line[i]) slist.append(val) n = EMUtil.get_image_count(args[0]) good = set(range(n)) - set(slist) slist = [i for i in good] slist.sort() del n,val,vdata,good if options.makevstack!=None or options.appendvstack!=None : vspath=os.path.realpath(vstack.path)+"/" if options.verbose>2 : print "vspath: ",vspath for db in dbs: dct,keys=db_open_dict(path+db,ro=True,with_keys=True) if dct==vstack : continue if len(options.step)==2 : if keys == None: vals = xrange(options.step[0],len(dct),options.step[1]) else: vals = keys[options.step[0]::options.step[1]] # we apply --step even if we have a list of keys else: if keys == None: vals = xrange(options.step[0],options.step[2],options.step[1]) else: vals = keys[options.step[0]:options.step[2]:options.step[1]] # we apply --step even if we have a list of keys if options.list !=None or options.exlist != None: vals=slist for n in vals: try: d=dct.get(n,nodata=1).get_attr_dict() except: traceback.print_exc() print "---\nerror reading ",db,n continue # This block converts an absolute path to the actual data to a relative path try: dpath=os.path.realpath(dct.get_data_path(n)) if options.verbose>2 : print "dpath: ",dpath if os.name == 'nt': vspath=vspath.replace("\\", '/') dpath=dpath.replace('\\', '/') rpath=makerelpath(vspath,dpath) if options.verbose>2 : print "rpath: ",rpath except: print "error with data_path ",db,n continue d["data_path"]=rpath d["data_n"]=n d["data_source"]= path+db if d["data_path"]==None : print "error with data_path ",db,n continue vstack[vstackn]=d vstackn+=1 if vstackn%100==0: try: print "\r ",vstackn," ", sys.stdout.flush() except: pass print "\r ",vstackn," " dct.close() try: maxname=max([len(s) for s in dbs]) except: print "Error reading ",path if options.restore : nima = EMUtil.get_image_count(options.restore) IB = db_open_dict(options.restore) source_old = None if len(options.step)==3 : nima=min(options.step[2],nima) for i in xrange(options.step[0],nima,options.step[1]): source = IB.get_header(i) source_path = source["source_path"] ID = source["source_n"] if( source_old != source_path): if( source_old != None): DB.close() DB = db_open_dict(source_path,ro=True) source_old = source_path target = DB.get_header( ID ) try: source["data_path"] = target["data_path"] source["data_n"] = target["data_n"] source["source_path"] = target["source_path"] source["source_n"] = target["source_n"] except: # top level does not have data_path del source['data_path'] del source['data_n'] source["source_path"] = target["source_path"] source["source_n"] = target["source_n"] DB.set_header(ID, source) DB.close() if options.extractplots : for db in dbs: print "#### Extracting plots from ",db dct=db_open_dict(path+db,ro=True) #### Dump keys=dct.keys() keys.sort() for k in keys: v=dct[k] try : ns=[len(i) for i in v] fsp=db+"-"+k+".txt" print "%s (%d columns)"%(fsp,len(ns)) out=file(fsp,"w") for i in range(ns[0]): for j in range(len(ns)): out.write(str(v[j][i])) if j<len(ns)-1 : out.write("\t") out.write("\n") out.close() except: continue dct.close() if options.smalldump : for db in dbs: print "##### ",db dct=db_open_dict(path+db,ro=True) #### Dump keys=dct.keys() keys.sort() if len(options.step)==3 : keys=keys[:options.step[2]] for k in keys[options.step[0]::options.step[1]]: v=dct[k] print "%s : "%k, if isinstance (v,list) or isinstance(v,tuple) : for i in v: print "\n\t%s"%str(i), print "" elif isinstance(v,dict) : ks2=v.keys() ks2.sort() kc=0 for i in ks2: if kc>=2 : print "..." break print "\n\t%s : %s"%(i,v[i]), kc+=1 print "" else : print str(v) dct.close() if options.checkctf: for db in dbs: print "##### CTF -> ",db dct=db_open_dict(path+db,ro=True) keys=dct.keys() if len(options.step)==3 : keys=keys[:options.step[2]] defocus=set() for k in keys[options.step[0]::options.step[1]]: v=dct.get_header(k) try: ctf=v["ctf"] except: if k!="maxrec" : print "CTF missing on image %s"%k continue defocus.add(ctf.defocus) defocus=list(defocus) print "Defocuses found: ", for i in defocus: print "%1.3f, "%i, print "\n\nRange: %1.3f - %1.3f (%d unique values)"%(min(defocus),max(defocus),len(defocus)) if options.dump : for db in dbs: print "##### ",db dct=db_open_dict(path+db,ro=True) #### Dump keys=dct.keys() if len(options.step)==3 : keys=keys[:options.step[2]] keys.sort() for k in keys[options.step[0]::options.step[1]]: v=dct[k] print "%s : "%k, if isinstance (v,list) or isinstance(v,tuple) : for i in v: print "\n\t%s"%str(i), print "" elif isinstance(v,dict) : ks2=v.keys() ks2.sort() for i in ks2: print "\n\t%s : %s"%(i,v[i]), print "" else : print str(v) dct.close() # long listing, one db per line elif options.long : width=maxname+3 fmt="%%-%ds %%-07d %%14s %%s"%width fmt2="%%-%ds (not an image stack)"%width total=[0,0] for db in dbs: dct=db_open_dict(path+db,True) ### Info on all particles if options.all : mx=len(dct) if len(options.step)==3 : mx=min(mx,options.step[2]) for i in range(options.step[0],mx,options.step[1]): try: im=dct[i] if im==None : raise Exception except: continue print "%d. %d x %d x %d\tA/pix=%1.2f\tM=%1.4f\tS=%1.4f\tSk=%1.4f"%(i,im["nx"],im["ny"],im["nz"],im["apix_x"],im["mean"],im["sigma"],im["skewness"]), try: print "\t%s"%str(im["model_id"]) except: pass try: print "\tdf=%1.3f\tB=%1.1f"%(im["ctf"].defocus,im["ctf"].bfactor) except: print " " first=EMData() try: first.read_image(path+db,0,True) size=first.get_xsize()*first.get_ysize()*first.get_zsize()*len(dct)*4; total[0]+=len(dct) total[1]+=size print fmt%(db,len(dct),"%dx%dx%d apix: %1.2f"%(first.get_xsize(),first.get_ysize(),first.get_zsize(),first["apix_x"]),human_size(size)), except: print fmt2%db try: print "\tdf: %1.3f\tB: %1.0f"%(first["ctf"].defocus,first["ctf"].bfactor) except: print "" dct.close() print fmt%("TOTAL",total[0],"",human_size(total[1])) elif options.check : from cPickle import loads for db in dbs: dct=db_open_dict(path+db,ro=True) dct.realopen() keys=dct.bdb.keys() allkvp={} for k in keys: s1,s2=k.split("\x80",1) # start of a pickled string. s2=loads("\x80"+s2) # the pickled part if len(s1)>0 : # If anything unpickled, then it is an axbxc prefix identifying the location of a binary st=allkvp.setdefault(s1,set()) # set of all positions seen so far v=loads(dct.bdb.get(k)) # position in binary file if v in st : print "Error: value %d seen multiple times in %s (%s,%s)"%(v,db,s1,s2) st.add(v) print "%s : "%db, for i in allkvp.keys(): if options.verbose>0 : print "%s %d/%d\t"%(i,len(allkvp[i]),int(max(allkvp[i]))+1), if len(allkvp[i])!=int(max(allkvp[i])+1) : print "\nMismatch found in %s. Could be normal if file has been rewritten multiple times, but is unusual"%db if options.verbose>0 : print "" else : print " done" dct.close() elif options.short : for db in dbs: print path+db, print " "
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " proj_stack output_averages --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--img_per_group", type="int", default=100, help="number of images per group") parser.add_option("--radius", type="int", default=-1, help="radius for alignment") parser.add_option( "--xr", type="string", default="2 1", help="range for translation search in x direction, search is +/xr") parser.add_option( "--yr", type="string", default="-1", help= "range for translation search in y direction, search is +/yr (default = same as xr)" ) parser.add_option( "--ts", type="string", default="1 0.5", help= "step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional" ) parser.add_option( "--iter", type="int", default=30, help="number of iterations within alignment (default = 30)") parser.add_option( "--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)") parser.add_option("--thld_err", type="float", default=1.0, help="threshold of pixel error (default = 1.732)") parser.add_option( "--grouping", type="string", default="GRP", help= "do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size" ) parser.add_option( "--delta", type="float", default=-1.0, help="angular step for reference projections (required for GEV method)" ) parser.add_option( "--fl", type="float", default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter") parser.add_option( "--aa", type="float", default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter") parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version") (options, args) = parser.parse_args() myid = mpi.mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi.mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 2: stack = args[0] outdir = args[1] else: sp_global_def.ERROR("Incomplete list of arguments", "sxproj_stability.main", 1, myid=myid) return if not options.MPI: sp_global_def.ERROR("Non-MPI not supported!", "sxproj_stability.main", 1, myid=myid) return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() sp_global_def.BATCH = True img_per_grp = options.img_per_group radius = options.radius ite = options.iter num_ali = options.num_ali thld_err = options.thld_err xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else: yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) if radius == -1: radius = nx / 2 - 2 mask = model_circle(radius, nx, nx) st = time() if options.grouping == "GRP": if myid == main_node: sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") proj_params = [] for i in range(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) # Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta, # So I will briefly explain it here # proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers # except for the last one. Depending on the number of particles left, they will either form a # group or append themselves to the last group # angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi, # theta) is the projection angle of the center of the group, delta is the range of this group # mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates # whether it should take mirror position. # In this program angle_list and mirror list are not of interest. proj_list_all, angle_list, mirror_list = group_proj_by_phitheta( proj_params, img_per_grp=img_per_grp) del proj_params sxprint(" B number of groups ", myid, " ", len(proj_list_all), time() - st) mpi_barrier(MPI_COMM_WORLD) # Number of groups, actually there could be one or two more groups, since the size of the remaining group varies # we will simply assign them to main node. n_grp = nima / img_per_grp - 1 # Divide proj_list_all equally to all nodes, and becomes proj_list proj_list = [] for i in range(n_grp): proc_to_stay = i % number_of_proc if proc_to_stay == main_node: if myid == main_node: proj_list.append(proj_list_all[i]) elif myid == main_node: mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) elif myid == proc_to_stay: img_per_grp = mpi_recv(1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) img_per_grp = int(img_per_grp[0]) temp = mpi_recv(img_per_grp, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) proj_list.append(list(map(int, temp))) del temp mpi_barrier(MPI_COMM_WORLD) sxprint(" C ", myid, " ", time() - st) if myid == main_node: # Assign the remaining groups to main_node for i in range(n_grp, len(proj_list_all)): proj_list.append(proj_list_all[i]) del proj_list_all, angle_list, mirror_list # Compute stability per projection projection direction, equal number assigned, thus overlaps elif options.grouping == "GEV": if options.delta == -1.0: ERROR( "Angular step for reference projections is required for GEV method" ) return from sp_utilities import even_angles, nearestk_to_refdir, getvec refproj = even_angles(options.delta) img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid) # Now each processor keeps its own share of reference projections refprojdir = refproj[img_begin:img_end] del refproj ref_ang = [0.0] * (len(refprojdir) * 2) for i in range(len(refprojdir)): ref_ang[i * 2] = refprojdir[0][0] ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1 sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") # the solution below is very slow, do not use it unless there is a problem with the i/O """ for i in xrange(number_of_proc): if myid == i: proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") mpi_barrier(MPI_COMM_WORLD) """ sxprint(" B ", myid, " ", time() - st) proj_ang = [0.0] * (nima * 2) for i in range(nima): dp = proj_attr[i].get_params("spider") proj_ang[i * 2] = dp["phi"] proj_ang[i * 2 + 1] = dp["theta"] sxprint(" C ", myid, " ", time() - st) asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp) del proj_ang, ref_ang proj_list = [] for i in range(len(refprojdir)): proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp]) del asi sxprint(" D ", myid, " ", time() - st) #from sys import exit #exit() # Compute stability per projection elif options.grouping == "PPR": sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") sxprint(" B ", myid, " ", time() - st) proj_params = [] for i in range(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) sxprint(" C ", myid, " ", time() - st) from sp_utilities import nearest_proj proj_list, mirror_list = nearest_proj( proj_params, img_per_grp, list(range(img_begin, img_begin + 1))) #range(img_begin, img_end)) refprojdir = proj_params[img_begin:img_end] del proj_params, mirror_list sxprint(" D ", myid, " ", time() - st) else: ERROR("Incorrect projection grouping option") return ########################################################################################################### # Begin stability test from sp_utilities import get_params_proj, read_text_file #if myid == 0: # from utilities import read_text_file # proj_list[0] = map(int, read_text_file("lggrpp0.txt")) from sp_utilities import model_blank aveList = [model_blank(nx, ny)] * len(proj_list) if options.grouping == "GRP": refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list) for i in range(len(proj_list)): sxprint(" E ", myid, " ", time() - st) class_data = EMData.read_images(stack, proj_list[i]) #print " R ",myid," ",time()-st if options.CTF: from sp_filter import filt_ctf for im in range(len(class_data)): # MEM LEAK!! atemp = class_data[im].copy() btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1) class_data[im] = btemp #class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: try: t = im.get_attr( "xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) #print " F ",myid," ",time()-st # Here, we perform realignment num_ali times all_ali_params = [] for j in range(num_ali): if (xrng[0] == 0.0 and yrng[0] == 0.0): avet = ali2d_ras(class_data, randomize=True, ir=1, ou=radius, rs=1, step=1.0, dst=90.0, maxit=ite, check_mirror=True, FH=options.fl, FF=options.aa) else: avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa) ali_params = [] for im in range(len(class_data)): alpha, sx, sy, mirror, scale = get_params2D(class_data[im]) ali_params.extend([alpha, sx, sy, mirror]) all_ali_params.append(ali_params) #aveList[i] = avet #print " G ",myid," ",time()-st del ali_params # We determine the stability of this group here. # stable_set contains all particles deemed stable, it is a list of list # each list has two elements, the first is the pixel error, the second is the image number # stable_set is sorted based on pixel error #from utilities import write_text_file #write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid) stable_set, mir_stab_rate, average_pix_err = multi_align_stability( all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1) #print " H ",myid," ",time()-st if (len(stable_set) > 5): stable_set_id = [] members = [] pix_err = [] # First put the stable members into attr 'members' and 'pix_err' for s in stable_set: # s[1] - number in this subset stable_set_id.append(s[1]) # the original image number members.append(proj_list[i][s[1]]) pix_err.append(s[0]) # Then put the unstable members into attr 'members' and 'pix_err' from sp_fundamentals import rot_shift2D avet.to_zero() if options.grouping == "GRP": aphi = 0.0 atht = 0.0 vphi = 0.0 vtht = 0.0 l = -1 for j in range(len(proj_list[i])): # Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses if j in stable_set_id: l += 1 avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3]) if options.grouping == "GRP": phi, theta, psi, sxs, sy_s = get_params_proj( class_data[j]) if (theta > 90.0): phi = (phi + 540.0) % 360.0 theta = 180.0 - theta aphi += phi atht += theta vphi += phi * phi vtht += theta * theta else: members.append(proj_list[i][j]) pix_err.append(99999.99) aveList[i] = avet.copy() if l > 1: l += 1 aveList[i] /= l if options.grouping == "GRP": aphi /= l atht /= l vphi = (vphi - l * aphi * aphi) / l vtht = (vtht - l * atht * atht) / l from math import sqrt refprojdir[i] = [ aphi, atht, (sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0 ] # Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION aveList[i].set_attr('members', members) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', pix_err) else: sxprint(" empty group ", i, refprojdir[i]) aveList[i].set_attr('members', [-1]) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', [99999.]) del class_data if myid == main_node: km = 0 for i in range(number_of_proc): if i == main_node: for im in range(len(aveList)): aveList[im].write_image(args[1], km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in range(nl): ave = recv_EMData(i, im + i + 70000) nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('members', list(map(int, members))) members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('pixerr', list(map(float, members))) members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('refprojdir', list(map(float, members))) ave.write_image(args[1], km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in range(len(aveList)): send_EMData(aveList[im], main_node, im + myid + 70000) members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) members = aveList[im].get_attr('pixerr') mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) except: mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) sp_global_def.BATCH = False mpi_barrier(MPI_COMM_WORLD)
def __new__(cls,filename,application,force_plot=False,force_2d=False,old=None): file_type = Util.get_filename_ext(filename) em_file_type = EMUtil.get_image_ext_type(file_type) if not file_exists(filename): return None if force_plot and force_2d: # ok this sucks but it suffices for the time being print "Error, the force_plot and force_2d options are mutually exclusive" return None if force_plot: from emplot2d import EMPlot2DWidget if isinstance(old,EMPlot2DWidget): widget = old else: widget = EMPlot2DWidget(application=application) widget.set_data_from_file(filename) return widget if em_file_type != IMAGE_UNKNOWN or filename[:4] == "bdb:": n = EMUtil.get_image_count(filename) nx,ny,nz = gimme_image_dimensions3D(filename) if n > 1 and nz == 1: if force_2d: a = EMData() data=a.read_images(filename) else: data = None # This is like a flag - the ImageMXWidget only needs the file name elif nz == 1: data = [EMData(filename,0)] else: data = EMData() data.read_image(filename,0,not force_2d) # This should be 3-D. We read the header-only here data = [data] if data != None and len(data) == 1: data = data[0] if force_2d or isinstance(data,EMData) and data.get_zsize()==1: if isinstance(data,list) or data.get_ysize() != 1: from emimage2d import EMImage2DWidget if isinstance(old,EMImage2DWidget): widget = old else: widget= EMImage2DWidget(application=application) else: from emplot2d import EMPlot2DWidget if isinstance(old,EMPlot2DWidget): widget = old else: widget = EMPlot2DWidget(application=application) widget.set_data_from_file(filename) return widget elif isinstance(data,EMData): if isinstance(old,EMScene3D): widget = old else: widget = EMScene3D() # print n,data for ii in xrange(n): data=EMData(filename,ii) datai = EMDataItem3D(data, transform=Transform()) widget.insertNewNode(os.path.basename(filename), datai, parentnode=widget) isosurface = EMIsosurface(datai, transform=Transform()) widget.insertNewNode("Iso", isosurface, parentnode=datai) return widget elif data == None or isinstance(data,list): from emimagemx import EMImageMXWidget if isinstance(old,EMImageMXWidget): widget = old else: widget = EMImageMXWidget(application=application) data = filename else: print filename raise # weirdness, this should never happen widget.set_data(data,filename) return widget else: from emplot2d import EMPlot2DWidget if isinstance(old,EMPlot2DWidget): widget = old else: widget = EMPlot2DWidget(application=application) widget.set_data_from_file(filename) return widget
def main(): def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror): if mirror: m = 1 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0 - psi, 0, 0, 1.0) else: m = 0 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0 - psi, 0, 0, 1.0) return alpha, sx, sy, m progname = os.path.basename(sys.argv[0]) usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=15. --aa=0.01 --sym=symmetry --CTF" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--output_dir", type="string", default="./", help="output directory") parser.add_option("--ave2D", type="string", default=False, help="write to the disk a stack of 2D averages") parser.add_option("--var2D", type="string", default=False, help="write to the disk a stack of 2D variances") parser.add_option("--ave3D", type="string", default=False, help="write to the disk reconstructed 3D average") parser.add_option("--var3D", type="string", default=False, help="compute 3D variability (time consuming!)") parser.add_option("--img_per_grp", type="int", default=10, help="number of neighbouring projections") parser.add_option("--no_norm", action="store_true", default=False, help="do not use normalization") #parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" ) parser.add_option("--npad", type="int", default=2, help="number of time to pad the original images") parser.add_option("--sym", type="string", default="c1", help="symmetry") parser.add_option( "--fl", type="float", default=0.0, help= "cutoff freqency in absolute frequency (0.0-0.5). (Default - no filtration)" ) parser.add_option( "--aa", type="float", default=0.0, help= "fall off of the filter. Put 0.01 if user has no clue about falloff (Default - no filtration)" ) parser.add_option("--CTF", action="store_true", default=False, help="use CFT correction") parser.add_option("--VERBOSE", action="store_true", default=False, help="Long output for debugging") #parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version") #parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" ) #parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" ) #parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" ) #parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" ) parser.add_option( "--VAR", action="store_true", default=False, help="stack on input consists of 2D variances (Default False)") parser.add_option( "--decimate", type="float", default=1.0, help= "image decimate rate, a number larger (expand image) or less (shrink image) than 1. default is 1" ) parser.add_option( "--window", type="int", default=0, help= "reduce images to a small image size without changing pixel_size. Default value is zero." ) #parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)") parser.add_option( "--nvec", type="int", default=0, help="number of eigenvectors, default = 0 meaning no PCA calculated") parser.add_option( "--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)") (options, args) = parser.parse_args() ##### from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX from applications import MPI_start_end from reconstruction import recons3d_em, recons3d_em_MPI from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI from utilities import print_begin_msg, print_end_msg, print_msg from utilities import read_text_row, get_image, get_im from utilities import bcast_EMData_to_all, bcast_number_to_all from utilities import get_symt # This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015 from EMAN2db import db_open_dict # Set up global variables related to bdb cache if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() # Set up global variables related to ERROR function global_def.BATCH = True # detect if program is running under MPI RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ if RUNNING_UNDER_MPI: global_def.MPI = True if options.symmetrize: if RUNNING_UNDER_MPI: try: sys.argv = mpi_init(len(sys.argv), sys.argv) try: number_of_proc = mpi_comm_size(MPI_COMM_WORLD) if (number_of_proc > 1): ERROR( "Cannot use more than one CPU for symmetry prepration", "sx3dvariability", 1) except: pass except: pass if options.output_dir != "./" and not os.path.exists( options.output_dir): os.mkdir(options.output_dir) # Input #instack = "Clean_NORM_CTF_start_wparams.hdf" #instack = "bdb:data" from logger import Logger, BaseLogger_Files if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt")) log_main = Logger(BaseLogger_Files()) log_main.prefix = os.path.join(options.output_dir, "./") instack = args[0] sym = options.sym.lower() if (sym == "c1"): ERROR("There is no need to symmetrize stack for C1 symmetry", "sx3dvariability", 1) line = "" for a in sys.argv: line += " " + a log_main.add(line) if (instack[:4] != "bdb:"): if output_dir == "./": stack = "bdb:data" else: stack = "bdb:" + options.output_dir + "/data" delete_bdb(stack) junk = cmdexecute("sxcpy.py " + instack + " " + stack) else: stack = instack qt = EMUtil.get_all_attributes(stack, 'xform.projection') na = len(qt) ts = get_symt(sym) ks = len(ts) angsa = [None] * na for k in xrange(ks): #Qfile = "Q%1d"%k if options.output_dir != "./": Qfile = os.path.join(options.output_dir, "Q%1d" % k) else: Qfile = os.path.join(options.output_dir, "Q%1d" % k) #delete_bdb("bdb:Q%1d"%k) delete_bdb("bdb:" + Qfile) #junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k) junk = cmdexecute("e2bdb.py " + stack + " --makevstack=bdb:" + Qfile) #DB = db_open_dict("bdb:Q%1d"%k) DB = db_open_dict("bdb:" + Qfile) for i in xrange(na): ut = qt[i] * ts[k] DB.set_attr(i, "xform.projection", ut) #bt = ut.get_params("spider") #angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]] #write_text_row(angsa, 'ptsma%1d.txt'%k) #junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k) #junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k)) DB.close() if options.output_dir == "./": delete_bdb("bdb:sdata") else: delete_bdb("bdb:" + options.output_dir + "/" + "sdata") #junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q") sdata = "bdb:" + options.output_dir + "/" + "sdata" print(sdata) junk = cmdexecute("e2bdb.py " + options.output_dir + " --makevstack=" + sdata + " --filt=Q") #junk = cmdexecute("ls EMAN2DB/sdata*") #a = get_im("bdb:sdata") a = get_im(sdata) a.set_attr("variabilitysymmetry", sym) #a.write_image("bdb:sdata") a.write_image(sdata) else: sys.argv = mpi_init(len(sys.argv), sys.argv) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 1: stack = args[0] else: print(("usage: " + usage)) print(("Please run '" + progname + " -h' for detailed options")) return 1 t0 = time() # obsolete flags options.MPI = True options.nvec = 0 options.radiuspca = -1 options.iter = 40 options.abs = 0.0 options.squ = 0.0 if options.fl > 0.0 and options.aa == 0.0: ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid) if options.VAR and options.SND: ERROR("Only one of var and SND can be set!", "sx3dvariability", myid) exit() if options.VAR and (options.ave2D or options.ave3D or options.var2D): ERROR( "When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid) exit() #if options.SND and (options.ave2D or options.ave3D): # ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid) # exit() if options.nvec > 0: ERROR("PCA option not implemented", "sx3dvariability", 1, myid) exit() if options.nvec > 0 and options.ave3D == None: ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid) exit() import string options.sym = options.sym.lower() # if global_def.CACHE_DISABLE: # from utilities import disable_bdb_cache # disable_bdb_cache() # global_def.BATCH = True if myid == main_node: if options.output_dir != "./" and not os.path.exists( options.output_dir): os.mkdir(options.output_dir) img_per_grp = options.img_per_grp nvec = options.nvec radiuspca = options.radiuspca from logger import Logger, BaseLogger_Files #if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt")) log_main = Logger(BaseLogger_Files()) log_main.prefix = os.path.join(options.output_dir, "./") if myid == main_node: line = "" for a in sys.argv: line += " " + a log_main.add(line) log_main.add("-------->>>Settings given by all options<<<-------") log_main.add("instack :" + stack) log_main.add("output_dir :" + options.output_dir) log_main.add("var3d :" + options.var3D) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" #print_begin_msg("sx3dvariability") msg = "sx3dvariability" log_main.add(msg) print(line, msg) msg = ("%-70s: %s\n" % ("Input stack", stack)) log_main.add(msg) print(line, msg) symbaselen = 0 if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() if options.sym != "c1": imgdata = get_im(stack) try: i = imgdata.get_attr("variabilitysymmetry").lower() if (i != options.sym): ERROR( "The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", myid=myid) except: ERROR( "Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", myid=myid) from utilities import get_symt i = len(get_symt(options.sym)) if ((nima / i) * i != nima): ERROR( "The length of the input stack is incorrect for symmetry processing", "sx3dvariability", myid=myid) symbaselen = nima / i else: symbaselen = nima else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) Tracker = {} Tracker["total_stack"] = nima if options.decimate == 1.: if options.window != 0: nx = options.window ny = options.window else: if options.window == 0: nx = int(nx * options.decimate) ny = int(ny * options.decimate) else: nx = int(options.window * options.decimate) ny = nx Tracker["nx"] = nx Tracker["ny"] = ny Tracker["nz"] = nx symbaselen = bcast_number_to_all(symbaselen) if radiuspca == -1: radiuspca = nx / 2 - 2 if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = "%-70s: %d\n" % ("Number of projection", nima) log_main.add(msg) print(line, msg) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) """ if options.SND: from projection import prep_vol, prgs from statistics import im_diff from utilities import get_im, model_circle, get_params_proj, set_params_proj from utilities import get_ctf, generate_ctf from filter import filt_ctf imgdata = EMData.read_images(stack, range(img_begin, img_end)) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) bcast_EMData_to_all(vol, myid) volft, kb = prep_vol(vol) mask = model_circle(nx/2-2, nx, ny) varList = [] for i in xrange(img_begin, img_end): phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin]) ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y]) if options.CTF: ctf_params = get_ctf(imgdata[i-img_begin]) ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params)) diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask) diff2 = diff*diff set_params_proj(diff2, [phi, theta, psi, s2x, s2y]) varList.append(diff2) mpi_barrier(MPI_COMM_WORLD) """ if options.VAR: #varList = EMData.read_images(stack, range(img_begin, img_end)) varList = [] this_image = EMData() for index_of_particle in xrange(img_begin, img_end): this_image.read_image(stack, index_of_particle) varList.append( image_decimate_window_xform_ctf(this_image, options.decimate, options.window, options.CTF)) else: from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2 from utilities import model_blank, nearest_proj, model_circle from applications import pca from statistics import avgvar, avgvar_ctf, ccc from filter import filt_tanl from morphology import threshold, square_root from projection import project, prep_vol, prgs from sets import Set if myid == main_node: t1 = time() proj_angles = [] aveList = [] tab = EMUtil.get_all_attributes(stack, 'xform.projection') for i in xrange(nima): t = tab[i].get_params('spider') phi = t['phi'] theta = t['theta'] psi = t['psi'] x = theta if x > 90.0: x = 180.0 - x x = x * 10000 + psi proj_angles.append([x, t['phi'], t['theta'], t['psi'], i]) t2 = time() line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = "%-70s: %d\n" % ("Number of neighboring projections", img_per_grp) log_main.add(msg) print(line, msg) msg = "...... Finding neighboring projections\n" log_main.add(msg) print(line, msg) if options.VERBOSE: msg = "Number of images per group: %d" % img_per_grp log_main.add(msg) print(line, msg) msg = "Now grouping projections" log_main.add(msg) print(line, msg) proj_angles.sort() proj_angles_list = [0.0] * (nima * 4) if myid == main_node: for i in xrange(nima): proj_angles_list[i * 4] = proj_angles[i][1] proj_angles_list[i * 4 + 1] = proj_angles[i][2] proj_angles_list[i * 4 + 2] = proj_angles[i][3] proj_angles_list[i * 4 + 3] = proj_angles[i][4] proj_angles_list = bcast_list_to_all(proj_angles_list, myid, main_node) proj_angles = [] for i in xrange(nima): proj_angles.append([ proj_angles_list[i * 4], proj_angles_list[i * 4 + 1], proj_angles_list[i * 4 + 2], int(proj_angles_list[i * 4 + 3]) ]) del proj_angles_list proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end)) all_proj = Set() for im in proj_list: for jm in im: all_proj.add(proj_angles[jm][3]) all_proj = list(all_proj) if options.VERBOSE: print("On node %2d, number of images needed to be read = %5d" % (myid, len(all_proj))) index = {} for i in xrange(len(all_proj)): index[all_proj[i]] = i mpi_barrier(MPI_COMM_WORLD) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("%-70s: %.2f\n" % ("Finding neighboring projections lasted [s]", time() - t2)) log_main.add(msg) print(msg) msg = ("%-70s: %d\n" % ("Number of groups processed on the main node", len(proj_list))) log_main.add(msg) print(line, msg) if options.VERBOSE: print("Grouping projections took: ", (time() - t2) / 60, "[min]") print("Number of groups on main node: ", len(proj_list)) mpi_barrier(MPI_COMM_WORLD) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("...... calculating the stack of 2D variances \n") log_main.add(msg) print(line, msg) if options.VERBOSE: print("Now calculating the stack of 2D variances") proj_params = [0.0] * (nima * 5) aveList = [] varList = [] if nvec > 0: eigList = [[] for i in xrange(nvec)] if options.VERBOSE: print("Begin to read images on processor %d" % (myid)) ttt = time() #imgdata = EMData.read_images(stack, all_proj) imgdata = [] for index_of_proj in xrange(len(all_proj)): #img = EMData() #img.read_image(stack, all_proj[index_of_proj]) dmg = image_decimate_window_xform_ctf( get_im(stack, all_proj[index_of_proj]), options.decimate, options.window, options.CTF) #print dmg.get_xsize(), "init" imgdata.append(dmg) if options.VERBOSE: print("Reading images on processor %d done, time = %.2f" % (myid, time() - ttt)) print("On processor %d, we got %d images" % (myid, len(imgdata))) mpi_barrier(MPI_COMM_WORLD) ''' imgdata2 = EMData.read_images(stack, range(img_begin, img_end)) if options.fl > 0.0: for k in xrange(len(imgdata2)): imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) if myid == main_node: vol.write_image("vol_ctf.hdf") print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf")) del vol, imgdata2 mpi_barrier(MPI_COMM_WORLD) ''' from applications import prepare_2d_forPCA from utilities import model_blank for i in xrange(len(proj_list)): ki = proj_angles[proj_list[i][0]][3] if ki >= symbaselen: continue mi = index[ki] phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi]) grp_imgdata = [] for j in xrange(img_per_grp): mj = index[proj_angles[proj_list[i][j]][3]] phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj]) alpha, sx, sy, mirror = params_3D_2D_NEW( phi, theta, psi, s2x, s2y, mirror_list[i][j]) if thetaM <= 90: if mirror == 0: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, phiM - phi, 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, 180 - (phiM - phi), 0.0, 0.0, 1.0) else: if mirror == 0: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, -(phiM - phi), 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2( alpha, sx, sy, 1.0, -(180 - (phiM - phi)), 0.0, 0.0, 1.0) set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0]) grp_imgdata.append(imgdata[mj]) #print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize() if not options.no_norm: #print grp_imgdata[j].get_xsize() mask = model_circle(nx / 2 - 2, nx, nx) for k in xrange(img_per_grp): ave, std, minn, maxx = Util.infomask( grp_imgdata[k], mask, False) grp_imgdata[k] -= ave grp_imgdata[k] /= std del mask if options.fl > 0.0: from filter import filt_ctf, filt_table from fundamentals import fft, window2d nx2 = 2 * nx ny2 = 2 * ny if options.CTF: from utilities import pad for k in xrange(img_per_grp): grp_imgdata[k] = window2d( fft( filt_tanl( filt_ctf( fft( pad(grp_imgdata[k], nx2, ny2, 1, 0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa)), nx, ny) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) else: for k in xrange(img_per_grp): grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) else: from utilities import pad, read_text_file from filter import filt_ctf, filt_table from fundamentals import fft, window2d nx2 = 2 * nx ny2 = 2 * ny if options.CTF: from utilities import pad for k in xrange(img_per_grp): grp_imgdata[k] = window2d( fft( filt_ctf(fft( pad(grp_imgdata[k], nx2, ny2, 1, 0.0)), grp_imgdata[k].get_attr("ctf"), binary=1)), nx, ny) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) ''' if i < 10 and myid == main_node: for k in xrange(10): grp_imgdata[k].write_image("grp%03d.hdf"%i, k) ''' """ if myid == main_node and i==0: for pp in xrange(len(grp_imgdata)): grp_imgdata[pp].write_image("pp.hdf", pp) """ ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata) """ if myid == main_node and i==0: for pp in xrange(len(grp_imgdata)): grp_imgdata[pp].write_image("qq.hdf", pp) """ var = model_blank(nx, ny) for q in grp_imgdata: Util.add_img2(var, q) Util.mul_scalar(var, 1.0 / (len(grp_imgdata) - 1)) # Switch to std dev var = square_root(threshold(var)) #if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a") #else: ave, var = avgvar(grp_imgdata, mode="a") """ if myid == main_node: ave.write_image("avgv.hdf",i) var.write_image("varv.hdf",i) """ set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0]) set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0]) aveList.append(ave) varList.append(var) if options.VERBOSE: print("%5.2f%% done on processor %d" % (i * 100.0 / len(proj_list), myid)) if nvec > 0: eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True) for k in xrange(nvec): set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0]) eigList[k].append(eig[k]) """ if myid == 0 and i == 0: for k in xrange(nvec): eig[k].write_image("eig.hdf", k) """ del imgdata # To this point, all averages, variances, and eigenvectors are computed if options.ave2D: from fundamentals import fpol if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node: for im in xrange(len(aveList)): aveList[im].write_image( os.path.join(options.output_dir, options.ave2D), km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im + i + 70000) """ nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('members', map(int, members)) members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('pix_err', map(float, members)) members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('refprojdir', map(float, members)) """ tmpvol = fpol(ave, Tracker["nx"], Tracker["nx"], 1) tmpvol.write_image( os.path.join(options.output_dir, options.ave2D), km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in xrange(len(aveList)): send_EMData(aveList[im], main_node, im + myid + 70000) """ members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) members = aveList[im].get_attr('pix_err') mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) except: mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) """ if options.ave3D: from fundamentals import fpol if options.VERBOSE: print("Reconstructing 3D average volume") ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(ave3D, myid) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" ave3D = fpol(ave3D, Tracker["nx"], Tracker["nx"], Tracker["nx"]) ave3D.write_image( os.path.join(options.output_dir, options.ave3D)) msg = ("%-70s: %s\n" % ( "Writing to the disk volume reconstructed from averages as", options.ave3D)) log_main.add(msg) print(line, msg) del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList if nvec > 0: for k in xrange(nvec): if options.VERBOSE: print("Reconstruction eigenvolumes", k) cont = True ITER = 0 mask2d = model_circle(radiuspca, nx, nx) while cont: #print "On node %d, iteration %d"%(myid, ITER) eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(eig3D, myid, main_node) if options.fl > 0.0: eig3D = filt_tanl(eig3D, options.fl, options.aa) if myid == main_node: eig3D.write_image( os.path.join(options.outpout_dir, "eig3d_%03d.hdf" % (k, ITER))) Util.mul_img(eig3D, model_circle(radiuspca, nx, nx, nx)) eig3Df, kb = prep_vol(eig3D) del eig3D cont = False icont = 0 for l in xrange(len(eigList[k])): phi, theta, psi, s2x, s2y = get_params_proj( eigList[k][l]) proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y]) cl = ccc(proj, eigList[k][l], mask2d) if cl < 0.0: icont += 1 cont = True eigList[k][l] *= -1.0 u = int(cont) u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD) icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" u = int(u[0]) msg = (" Eigenvector: ", k, " number changed ", int(icont[0])) log_main.add(msg) print(line, msg) else: u = 0 u = bcast_number_to_all(u, main_node) cont = bool(u) ITER += 1 del eig3Df, kb mpi_barrier(MPI_COMM_WORLD) del eigList, mask2d if options.ave3D: del ave3D if options.var2D: from fundamentals import fpol if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node: for im in xrange(len(varList)): tmpvol = fpol(varList[im], Tracker["nx"], Tracker["nx"], 1) tmpvol.write_image( os.path.join(options.output_dir, options.var2D), km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im + i + 70000) tmpvol = fpol(ave, Tracker["nx"], Tracker["nx"], 1) tmpvol.write_image( os.path.join(options.output_dir, options.var2D, km)) km += 1 else: mpi_send(len(varList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in xrange(len(varList)): send_EMData(varList[im], main_node, im + myid + 70000) # What with the attributes?? mpi_barrier(MPI_COMM_WORLD) if options.var3D: if myid == main_node and options.VERBOSE: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("Reconstructing 3D variability volume") log_main.add(msg) print(line, msg) t6 = time() # radiusvar = options.radius # if( radiusvar < 0 ): radiusvar = nx//2 -3 res = recons3d_4nn_MPI(myid, varList, symmetry=options.sym, npad=options.npad) #res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ) if myid == main_node: from fundamentals import fpol res = fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"]) res.write_image(os.path.join(options.output_dir, options.var3D)) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("%-70s: %.2f\n" % ("Reconstructing 3D variability took [s]", time() - t6)) log_main.add(msg) print(line, msg) if options.VERBOSE: print("Reconstruction took: %.2f [min]" % ((time() - t6) / 60)) if myid == main_node: line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("%-70s: %.2f\n" % ("Total time for these computations [s]", time() - t0)) print(line, msg) log_main.add(msg) if options.VERBOSE: print("Total time for these computations: %.2f [min]" % ((time() - t0) / 60)) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" msg = ("sx3dvariability") print(line, msg) log_main.add(msg) from mpi import mpi_finalize mpi_finalize() if RUNNING_UNDER_MPI: global_def.MPI = False global_def.BATCH = False
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " prj_stack volume [begin end step] --CTF --npad=ntimes_padding --list=file --group=ID --snr=SNR --sym=symmetry --verbose=(0|1) --xysize --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--CTF", action="store_true", default=False, help="apply CTF correction") parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio") parser.add_option("--sym", type="string", default="c1", help="symmetry") parser.add_option( "--list", type="string", help="file with list of images to be used in the first column") parser.add_option( "--group", type="int", default=-1, help= "perform reconstruction using images for a given group number (group is attribute in the header)" ) parser.add_option("--MPI", action="store_true", default=False, help="use MPI version ") parser.add_option("--npad", type="int", default=2, help="number of times padding (default 2)") parser.add_option("--verbose", type="int", default=0, help="verbose level: 0 no verbose, 1 verbose") parser.add_option("--xysize", type="int", default=-1, help="user expected size at xy direction") parser.add_option("--zsize", type="int", default=-1, help="user expected size at z direction") parser.add_option("--smearstep", type="float", default=0.0, help="Rotational smear step (default 0.0, no smear)") (options, args) = parser.parse_args(arglist[1:]) if options.MPI: from mpi import mpi_init sys.argv = mpi_init(len(sys.argv), sys.argv) if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() if len(args) == 2: prj_stack = args[0] vol_stack = args[1] nimage = EMUtil.get_image_count(prj_stack) pid_list = range(0, nimage) elif len(args) == 5: prj_stack = args[0] vol_stack = args[1] begin = atoi(args[2]) end = atoi(args[3]) step = atoi(args[4]) pid_list = range(begin, end, step) else: ERROR("incomplete list of arguments", "recon3d_n", 1) exit() if (options.list and options.group > -1): ERROR("options group and list cannot be used together", "recon3d_n", 1) sys.exit() from applications import recons3d_n global_def.BATCH = True recons3d_n(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\ options.sym, options.list, options.group, options.verbose, options.MPI,options.xysize, options.zsize, options.smearstep) global_def.BATCH = False if options.MPI: from mpi import mpi_finalize mpi_finalize()
def main(): import global_def from optparse import OptionParser from EMAN2 import EMUtil import os import sys from time import time progname = os.path.basename(sys.argv[0]) usage = progname + " proj_stack output_averages --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--img_per_group",type="int" , default=100 , help="number of images per group" ) parser.add_option("--radius", type="int" , default=-1 , help="radius for alignment" ) parser.add_option("--xr", type="string" , default="2 1", help="range for translation search in x direction, search is +/xr") parser.add_option("--yr", type="string" , default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)") parser.add_option("--ts", type="string" , default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional") parser.add_option("--iter", type="int" , default=30, help="number of iterations within alignment (default = 30)" ) parser.add_option("--num_ali", type="int" , default=5, help="number of alignments performed for stability (default = 5)" ) parser.add_option("--thld_err", type="float" , default=1.0, help="threshold of pixel error (default = 1.732)" ) parser.add_option("--grouping" , type="string" , default="GRP", help="do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size") parser.add_option("--delta", type="float" , default=-1.0, help="angular step for reference projections (required for GEV method)") parser.add_option("--fl", type="float" , default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter") parser.add_option("--aa", type="float" , default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter") parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ") parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version") (options,args) = parser.parse_args() from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD, MPI_TAG_UB from mpi import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT from applications import MPI_start_end, within_group_refinement, ali2d_ras from pixel_error import multi_align_stability from utilities import send_EMData, recv_EMData from utilities import get_image, bcast_number_to_all, set_params2D, get_params2D from utilities import group_proj_by_phitheta, model_circle, get_input_from_string sys.argv = mpi_init(len(sys.argv), sys.argv) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 2: stack = args[0] outdir = args[1] else: ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid) exit() if not options.MPI: ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid) exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True #if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid) #mpi_barrier(MPI_COMM_WORLD) img_per_grp = options.img_per_group radius = options.radius ite = options.iter num_ali = options.num_ali thld_err = options.thld_err xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else : yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) if radius == -1: radius = nx/2-2 mask = model_circle(radius, nx, nx) st = time() if options.grouping == "GRP": if myid == main_node: print " A ",myid," ",time()-st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") proj_params = [] for i in xrange(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) # Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta, # So I will briefly explain it here # proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers # except for the last one. Depending on the number of particles left, they will either form a # group or append themselves to the last group # angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi, # theta) is the projection angle of the center of the group, delta is the range of this group # mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates # whether it should take mirror position. # In this program angle_list and mirror list are not of interest. proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(proj_params, img_per_grp=img_per_grp) del proj_params print " B number of groups ",myid," ",len(proj_list_all),time()-st mpi_barrier(MPI_COMM_WORLD) # Number of groups, actually there could be one or two more groups, since the size of the remaining group varies # we will simply assign them to main node. n_grp = nima/img_per_grp-1 # Divide proj_list_all equally to all nodes, and becomes proj_list proj_list = [] for i in xrange(n_grp): proc_to_stay = i%number_of_proc if proc_to_stay == main_node: if myid == main_node: proj_list.append(proj_list_all[i]) elif myid == main_node: mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD) mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD) elif myid == proc_to_stay: img_per_grp = mpi_recv(1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) img_per_grp = int(img_per_grp[0]) temp = mpi_recv(img_per_grp, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) proj_list.append(map(int, temp)) del temp mpi_barrier(MPI_COMM_WORLD) print " C ",myid," ",time()-st if myid == main_node: # Assign the remaining groups to main_node for i in xrange(n_grp, len(proj_list_all)): proj_list.append(proj_list_all[i]) del proj_list_all, angle_list, mirror_list # Compute stability per projection projection direction, equal number assigned, thus overlaps elif options.grouping == "GEV": if options.delta == -1.0: ERROR("Angular step for reference projections is required for GEV method","sxproj_stability",1) from utilities import even_angles, nearestk_to_refdir, getvec refproj = even_angles(options.delta) img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid) # Now each processor keeps its own share of reference projections refprojdir = refproj[img_begin: img_end] del refproj ref_ang = [0.0]*(len(refprojdir)*2) for i in xrange(len(refprojdir)): ref_ang[i*2] = refprojdir[0][0] ref_ang[i*2+1] = refprojdir[0][1]+i*0.1 print " A ",myid," ",time()-st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") # the solution below is very slow, do not use it unless there is a problem with the i/O """ for i in xrange(number_of_proc): if myid == i: proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") mpi_barrier(MPI_COMM_WORLD) """ print " B ",myid," ",time()-st proj_ang = [0.0]*(nima*2) for i in xrange(nima): dp = proj_attr[i].get_params("spider") proj_ang[i*2] = dp["phi"] proj_ang[i*2+1] = dp["theta"] print " C ",myid," ",time()-st asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp) del proj_ang, ref_ang proj_list = [] for i in xrange(len(refprojdir)): proj_list.append(asi[i*img_per_grp:(i+1)*img_per_grp]) del asi print " D ",myid," ",time()-st #from sys import exit #exit() # Compute stability per projection elif options.grouping == "PPR": print " A ",myid," ",time()-st proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") print " B ",myid," ",time()-st proj_params = [] for i in xrange(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) print " C ",myid," ",time()-st from utilities import nearest_proj proj_list, mirror_list = nearest_proj(proj_params, img_per_grp, range(img_begin, img_begin+1))#range(img_begin, img_end)) refprojdir = proj_params[img_begin: img_end] del proj_params, mirror_list print " D ",myid," ",time()-st else: ERROR("Incorrect projection grouping option","sxproj_stability",1) """ from utilities import write_text_file for i in xrange(len(proj_list)): write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid)) """ ########################################################################################################### # Begin stability test from utilities import get_params_proj, read_text_file #if myid == 0: # from utilities import read_text_file # proj_list[0] = map(int, read_text_file("lggrpp0.txt")) from utilities import model_blank aveList = [model_blank(nx,ny)]*len(proj_list) if options.grouping == "GRP": refprojdir = [[0.0,0.0,-1.0]]*len(proj_list) for i in xrange(len(proj_list)): print " E ",myid," ",time()-st class_data = EMData.read_images(stack, proj_list[i]) #print " R ",myid," ",time()-st if options.CTF : from filter import filt_ctf for im in xrange(len(class_data)): # MEM LEAK!! atemp = class_data[im].copy() btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1) class_data[im] = btemp #class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: try: t = im.get_attr("xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0,-d["tx"],-d["ty"],0,1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) #print " F ",myid," ",time()-st # Here, we perform realignment num_ali times all_ali_params = [] for j in xrange(num_ali): if( xrng[0] == 0.0 and yrng[0] == 0.0 ): avet = ali2d_ras(class_data, randomize = True, ir = 1, ou = radius, rs = 1, step = 1.0, dst = 90.0, maxit = ite, check_mirror = True, FH=options.fl, FF=options.aa) else: avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa) ali_params = [] for im in xrange(len(class_data)): alpha, sx, sy, mirror, scale = get_params2D(class_data[im]) ali_params.extend( [alpha, sx, sy, mirror] ) all_ali_params.append(ali_params) #aveList[i] = avet #print " G ",myid," ",time()-st del ali_params # We determine the stability of this group here. # stable_set contains all particles deemed stable, it is a list of list # each list has two elements, the first is the pixel error, the second is the image number # stable_set is sorted based on pixel error #from utilities import write_text_file #write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid) stable_set, mir_stab_rate, average_pix_err = multi_align_stability(all_ali_params, 0.0, 10000.0, thld_err, False, 2*radius+1) #print " H ",myid," ",time()-st if(len(stable_set) > 5): stable_set_id = [] members = [] pix_err = [] # First put the stable members into attr 'members' and 'pix_err' for s in stable_set: # s[1] - number in this subset stable_set_id.append(s[1]) # the original image number members.append(proj_list[i][s[1]]) pix_err.append(s[0]) # Then put the unstable members into attr 'members' and 'pix_err' from fundamentals import rot_shift2D avet.to_zero() if options.grouping == "GRP": aphi = 0.0 atht = 0.0 vphi = 0.0 vtht = 0.0 l = -1 for j in xrange(len(proj_list[i])): # Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses if j in stable_set_id: l += 1 avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] ) if options.grouping == "GRP": phi, theta, psi, sxs, sys = get_params_proj(class_data[j]) if( theta > 90.0): phi = (phi+540.0)%360.0 theta = 180.0 - theta aphi += phi atht += theta vphi += phi*phi vtht += theta*theta else: members.append(proj_list[i][j]) pix_err.append(99999.99) aveList[i] = avet.copy() if l>1 : l += 1 aveList[i] /= l if options.grouping == "GRP": aphi /= l atht /= l vphi = (vphi - l*aphi*aphi)/l vtht = (vtht - l*atht*atht)/l from math import sqrt refprojdir[i] = [aphi, atht, (sqrt(max(vphi,0.0))+sqrt(max(vtht,0.0)))/2.0] # Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION aveList[i].set_attr('members', members) aveList[i].set_attr('refprojdir',refprojdir[i]) aveList[i].set_attr('pixerr', pix_err) else: print " empty group ",i, refprojdir[i] aveList[i].set_attr('members',[-1]) aveList[i].set_attr('refprojdir',refprojdir[i]) aveList[i].set_attr('pixerr', [99999.]) del class_data if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node : for im in xrange(len(aveList)): aveList[im].write_image(args[1], km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im+i+70000) nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('members', map(int, members)) members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('pixerr', map(float, members)) members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('refprojdir', map(float, members)) ave.write_image(args[1], km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) for im in xrange(len(aveList)): send_EMData(aveList[im], main_node,im+myid+70000) members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) members = aveList[im].get_attr('pixerr') mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) except: mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) global_def.BATCH = False mpi_barrier(MPI_COMM_WORLD) from mpi import mpi_finalize mpi_finalize()
def main(): import os import sys from optparse import OptionParser from global_def import SPARXVERSION import global_def arglist = [] for arg in sys.argv: arglist.append( arg ) progname = os.path.basename(arglist[0]) usage2 = progname + """ inputfile outputfile [options] Functionalities: 1. Helicise input volume and save the result to output volume: sxhelicon_utils.py input_vol.hdf output_vol.hdf --helicise --dp=27.6 --dphi=166.5 --fract=0.65 --rmax=70 --rmin=1 --apix=1.84 --sym=D1 2. Helicise pdb file and save the result to a new pdb file: sxhelicon_utils.py input.pdb output.pdb --helicisepdb --dp=27.6 --dphi=166.5 --nrepeats --apix=1.84 3. Generate two lists of image indices used to split segment stack into halves for helical fsc calculation. sxhelicon_utils.py bdb:big_stack --hfsc='flst' --filament_attr=filament 4. Map of filament distribution in the stack sxhelicon_utils.py bdb:big_stack --filinfo=info.txt The output file will contain four columns: 1 2 3 4 first image number last image number number of images in the filament name 5. Predict segments' orientation parameters based on distances between segments and known helical symmetry sxhelicon_utils.py bdb:big_stack --predict_helical=helical_params.txt --dp=27.6 --dphi=166.5 --apix=1.84 6. Generate disks from filament based reconstructions: sxheader.py stk.hdf --params=xform.projection --import=params.txt # horatio active_refactoring Jy51i1EwmLD4tWZ9_00000_1 # sxheader.py stk.hdf --params=active --one mpirun -np 2 sxhelicon_utils.py stk.hdf --gendisk='bdb:disk' --ref_nx=100 --ref_ny=100 --ref_nz=200 --apix=1.84 --dp=27.6 --dphi=166.715 --fract=0.67 --rmin=0 --rmax=64 --function="[.,nofunc,helical3c]" --sym="c1" --MPI 7. Stack disks based on helical symmetry parameters sxhelicon_utils.py disk_to_stack.hdf --stackdisk=stacked_disks.hdf --dphi=166.5 --dp=27.6 --ref_nx=160 --ref_ny=160 --ref_nz=225 --apix=1.84 8. Helical symmetry search: mpirun -np 3 sxhelicon_utils.py volf0010.hdf outsymsearch --symsearch --dp=27.6 --dphi=166.715 --apix=1.84 --fract=0.65 --rmin=0 --rmax=92.0 --datasym=datasym.txt --dp_step=0.92 --ndp=3 --dphi_step=1.0 --ndphi=10 --MPI """ parser = OptionParser(usage2,version=SPARXVERSION) #parser.add_option("--ir", type="float", default= -1, help="inner radius for rotational correlation > 0 (set to 1) (Angstroms)") parser.add_option("--ou", type="float", default= -1, help="outer radius for rotational 2D correlation < int(nx/2)-1 (set to the radius of the particle) (Angstroms)") parser.add_option("--rs", type="int", default= 1, help="step between rings in rotational correlation >0 (set to 1)" ) parser.add_option("--xr", type="string", default= "4 2 1 1 1", help="range for translation search in x direction, search is +/-xr (Angstroms) ") parser.add_option("--txs", type="string", default= "1 1 1 0.5 0.25", help="step size of the translation search in x directions, search is -xr, -xr+ts, 0, xr-ts, xr (Angstroms)") parser.add_option("--delta", type="string", default= "10 6 4 3 2", help="angular step of reference projections") parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches") parser.add_option("--maxit", type="int", default= 30, help="maximum number of iterations performed for each angular step (set to 30) ") parser.add_option("--CTF", action="store_true", default=False, help="CTF correction") parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version") #parser.add_option("--fourvar", action="store_true", default=False, help="compute Fourier variance") parser.add_option("--apix", type="float", default= -1.0, help="pixel size in Angstroms") parser.add_option("--dp", type="float", default= -1.0, help="delta z - translation in Angstroms") parser.add_option("--dphi", type="float", default= -1.0, help="delta phi - rotation in degrees") parser.add_option("--rmin", type="float", default= 0.0, help="minimal radius for hsearch (Angstroms)") parser.add_option("--rmax", type="float", default= 80.0, help="maximal radius for hsearch (Angstroms)") parser.add_option("--fract", type="float", default= 0.7, help="fraction of the volume used for helical search") parser.add_option("--sym", type="string", default= "c1", help="symmetry of the structure") parser.add_option("--function", type="string", default="helical", help="name of the reference preparation function") parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction") parser.add_option("--debug", action="store_true", default=False, help="debug") parser.add_option("--volalixshift", action="store_true", default=False, help="Use volalixshift refinement") parser.add_option("--searchxshift", type="float", default= 0.0, help="search range for x-shift determination: +/- searchxshift (Angstroms)") parser.add_option("--nearby", type="float", default= 6.0, help="neighborhood within which to search for peaks in 1D ccf for x-shift search (Angstroms)") # filinfo parser.add_option( "--filinfo", type="string", default="", help="Store in an output text file infomration about distribution of filaments in the stack." ) # diskali parser.add_option("--diskali", action="store_true", default=False, help="volume alignment") parser.add_option("--zstep", type="float", default= 1, help="Step size for translational search along z (Angstroms)") # helicise parser.add_option("--helicise", action="store_true", default=False, help="helicise input volume and save results to output volume") parser.add_option("--hfsc", type="string", default="", help="Generate two lists of image indices used to split segment stack into halves for helical fsc calculation. The lists will be stored in two text files named using file_prefix with '_even' and '_odd' suffixes, respectively." ) parser.add_option("--filament_attr", type="string", default="filament", help="attribute under which filament identification is stored" ) parser.add_option("--predict_helical", type="string", default="", help="Generate projection parameters consistent with helical symmetry") # helicise pdb parser.add_option("--helicisepdb", action="store_true", default=False, help="Helicise pdb file and save the result to a new pdb file") parser.add_option("--nrepeats", type="int", default= 50, help="Number of time the helical symmetry will be applied to the input file") # input options for generating disks parser.add_option("--gendisk", type="string", default="", help="Name of file under which generated disks will be saved to") parser.add_option("--ref_nx", type="int", default= -1, help="nx=ny volume size" ) parser.add_option("--ref_nz", type="int", default= -1, help="nz volume size - computed disks will be nx x ny x rise/apix" ) parser.add_option("--new_pixel_size", type="float", default= -1, help="desired pixel size of the output disks. The default is -1, in which case there is no resampling (unless --match_pixel_rise flag is True).") parser.add_option("--maxerror", type="float", default= 0.1, help="proportional to the maximum amount of error to tolerate between (dp/new_pixel_size) and int(dp/new_pixel_size ), where new_pixel_size is the pixel size calculated when the option --match_pixel_rise flag is True.") parser.add_option("--match_pixel_rise", action="store_true", default=False, help="calculate new pixel size such that the rise is approximately integer number of pixels given the new pixel size. This will be the pixel size of the output disks.") # get consistency parser.add_option("--consistency", type="string", default="", help="Name of parameters to get consistency statistics for") parser.add_option("--phithr", type="float", default= 2.0, help="phi threshold for consistency check") parser.add_option("--ythr", type="float", default= 2.0, help="y threshold (in Angstroms) for consistency check") parser.add_option("--segthr", type="int", default= 3, help="minimum number of segments/filament for consistency check") # stack disks parser.add_option("--stackdisk", type="string", default="", help="Name of file under which output volume will be saved to.") parser.add_option("--ref_ny", type="int", default=-1, help="ny of output volume size. Default is ref_nx" ) # symmetry search parser.add_option("--symsearch", action="store_true", default=False, help="Do helical symmetry search." ) parser.add_option("--ndp", type="int", default= 12, help="In symmetrization search, number of delta z steps equals to 2*ndp+1") parser.add_option("--ndphi", type="int", default= 12, help="In symmetrization search, number of dphi steps equals to 2*ndphi+1") parser.add_option("--dp_step", type="float", default= 0.1, help="delta z step for symmetrization [Angstroms] (default 0.1)") parser.add_option("--dphi_step", type="float", default= 0.1, help="dphi step for symmetrization [degrees] (default 0.1)") parser.add_option("--datasym", type="string", default="datasym.txt", help="symdoc") parser.add_option("--symdoc", type="string", default="", help="text file containing helical symmetry parameters dp and dphi") # filament statistics in the stack (options, args) = parser.parse_args(arglist[1:]) if len(args) < 1 or len(args) > 5: print "Various helical reconstruction related functionalities: " + usage2 print "Please run '" + progname + " -h' for detailed options" else: if len(options.hfsc) > 0: if len(args) != 1: print "Incorrect number of parameters" sys.exit() from applications import imgstat_hfsc imgstat_hfsc( args[0], options.hfsc, options.filament_attr) sys.exit() elif len(options.filinfo) > 0: if len(args) != 1: print "Incorrect number of parameters" sys.exit() from EMAN2 import EMUtil filams = EMUtil.get_all_attributes(args[0], "filament") ibeg = 0 filcur = filams[0] n = len(filams) inf = [] i = 1 while( i <= n): if(i < n): fis = filams[i] else: fis = "" if( fis != filcur ): iend = i-1 inf.append([ibeg,iend,iend-ibeg+1,filcur]) ibeg = i filcur = fis i += 1 from utilities import write_text_row write_text_row(inf, options.filinfo) sys.exit() if len(options.stackdisk) > 0: if len(args) != 1: print "Incorrect number of parameters" sys.exit() dpp = (float(options.dp)/options.apix) rise = int(dpp) if(abs(float(rise) - dpp)>1.0e-3): print " dpp has to be integer multiplicity of the pixel size" sys.exit() from utilities import get_im v = get_im(args[0]) from applications import stack_disks ref_ny = options.ref_ny if ref_ny < 0: ref_ny = options.ref_nx sv = stack_disks(v, options.ref_nx, ref_ny, options.ref_nz, options.dphi, rise) sv.write_image(options.stackdisk) sys.exit() if len(options.consistency) > 0: if len(args) != 1: print "Incorrect number of parameters" sys.exit() from development import consistency_params consistency_params(args[0], options.consistency, options.dphi, options.dp, options.apix,phithr=options.phithr, ythr=options.ythr, THR=options.segthr) sys.exit() rminp = int((float(options.rmin)/options.apix) + 0.5) rmaxp = int((float(options.rmax)/options.apix) + 0.5) from utilities import get_input_from_string, get_im xr = get_input_from_string(options.xr) txs = get_input_from_string(options.txs) irp = 1 if options.ou < 0: oup = -1 else: oup = int( (options.ou/options.apix) + 0.5) xrp = '' txsp = '' for i in xrange(len(xr)): xrp += " "+str(float(xr[i])/options.apix) for i in xrange(len(txs)): txsp += " "+str(float(txs[i])/options.apix) searchxshiftp = int( (options.searchxshift/options.apix) + 0.5) nearbyp = int( (options.nearby/options.apix) + 0.5) zstepp = int( (options.zstep/options.apix) + 0.5) if options.MPI: from mpi import mpi_init, mpi_finalize sys.argv = mpi_init(len(sys.argv), sys.argv) if len(options.predict_helical) > 0: if len(args) != 1: print "Incorrect number of parameters" sys.exit() if options.dp < 0: print "Helical symmetry paramter rise --dp should not be negative" sys.exit() from applications import predict_helical_params predict_helical_params(args[0], options.dp, options.dphi, options.apix, options.predict_helical) sys.exit() if options.helicise: if len(args) != 2: print "Incorrect number of parameters" sys.exit() if options.dp < 0: print "Helical symmetry paramter rise --dp should not be negative" sys.exit() from utilities import get_im, sym_vol vol = get_im(args[0]) vol = sym_vol(vol, options.sym) hvol = vol.helicise(options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp) hvol = sym_vol(hvol, options.sym) hvol.write_image(args[1]) sys.exit() if options.helicisepdb: if len(args) != 2: print "Incorrect number of parameters" sys.exit() if options.dp < 0: print "Helical symmetry paramter rise --dp should not be negative" sys.exit() from math import cos, sin, radians from copy import deepcopy import numpy from numpy import zeros,dot,float32 dp = options.dp dphi = options.dphi nperiod = options.nrepeats infile =open(args[0],"r") pall = infile.readlines() infile.close() p = [] pos = [] lkl = -1 for i in xrange( len(pall) ): if( (pall[i])[:4] == 'ATOM'): if( lkl == -1 ): lkl = i p.append( pall[i] ) pos.append(i) n = len(p) X = zeros( (3,len(p) ), dtype=float32 ) X_new = zeros( (3,len(p) ), dtype=float32 ) for i in xrange( len(p) ): element = deepcopy( p[i] ) X[0,i]=float(element[30:38]) X[1,i]=float(element[38:46]) X[2,i]=float(element[46:54]) pnew = [] for j in xrange(-nperiod, nperiod+1): for i in xrange( n ): pnew.append( deepcopy(p[i]) ) dphi = radians(dphi) m = zeros( (3,3 ), dtype=float32 ) t = zeros( (3,1 ), dtype=float32 ) m[2][2] = 1.0 t[0,0] = 0.0 t[1,0] = 0.0 for j in xrange(-nperiod, nperiod+1): if j != 0: rd = j*dphi m[0][0] = cos(rd) m[0][1] = sin(rd) m[1][0] = -m[0][1] m[1][1] = m[0][0] t[2,0] = j*dp X_new = dot(m, X) + t for i in xrange( n ): pnew[j*n+i] = pnew[j*n+i][:30] + "%8.3f"%( float(X_new[0,i]) )+"%8.3f"%( float(X_new[1,i]) )+"%8.3f"%( float(X_new[2,i]) ) + pnew[j*n+i][54:] outfile=open(args[1],"w") outfile.writelines(pall[0:lkl]) outfile.writelines(pnew) outfile.writelines("END\n") outfile.close() sys.exit() if options.volalixshift: if options.maxit > 1: print "Inner iteration for x-shift determinatin is restricted to 1" sys.exit() if len(args) < 4: mask = None else: mask = args[3] from applications import volalixshift_MPI global_def.BATCH = True volalixshift_MPI(args[0], args[1], args[2], searchxshiftp, options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp, mask, options.maxit, options.CTF, options.snr, options.sym, options.function, options.npad, options.debug, nearbyp) global_def.BATCH = False if options.diskali: #if options.maxit > 1: # print "Inner iteration for disk alignment is restricted to 1" # sys.exit() if len(args) < 4: mask = None else: mask = args[3] global_def.BATCH = True if(options.sym[:1] == "d" or options.sym[:1] == "D" ): from development import diskaliD_MPI diskaliD_MPI(args[0], args[1], args[2], mask, options.dp, options.dphi, options.apix, options.function, zstepp, options.fract, rmaxp, rminp, options.CTF, options.maxit, options.sym) else: from applications import diskali_MPI diskali_MPI(args[0], args[1], args[2], mask, options.dp, options.dphi, options.apix, options.function, zstepp, options.fract, rmaxp, rminp, options.CTF, options.maxit, options.sym) global_def.BATCH = False if options.symsearch: if len(options.symdoc) < 1: if options.dp < 0 or options.dphi < 0: print "Enter helical symmetry parameters either using --symdoc or --dp and --dphi" sys.exit() if options.dp < 0 or options.dphi < 0: # read helical symmetry parameters from symdoc from utilities import read_text_row hparams=read_text_row(options.symdoc) dp = hparams[0][0] dphi = hparams[0][1] else: dp = options.dp dphi = options.dphi from applications import symsearch_MPI if len(args) < 3: mask = None else: mask= args[2] global_def.BATCH = True symsearch_MPI(args[0], args[1], mask, dp, options.ndp, options.dp_step, dphi, options.ndphi, options.dphi_step, rminp, rmaxp, options.fract, options.sym, options.function, options.datasym, options.apix, options.debug) global_def.BATCH = False elif len(options.gendisk)> 0: from applications import gendisks_MPI global_def.BATCH = True if len(args) == 1: mask3d = None else: mask3d = args[1] if options.dp < 0: print "Helical symmetry paramter rise --dp must be explictly set!" sys.exit() gendisks_MPI(args[0], mask3d, options.ref_nx, options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp, options.CTF, options.function, options.sym, options.gendisk, options.maxerror, options.new_pixel_size, options.match_pixel_rise) global_def.BATCH = False if options.MPI: from mpi import mpi_finalize mpi_finalize()
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Generic 2-D image processing programs. Functionality: 1. Phase flip a stack of images and write output to new file: sxprocess.py input_stack.hdf output_stack.hdf --phase_flip 2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size. The window size will change accordingly. sxprocess input.hdf output.hdf --changesize --ratio=0.5 3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes or a 3-D volume. sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024] 4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs: sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64 5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3): sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid 6. Retrieve original image numbers of images listed in ISAC output stack of averages: sxprocess.py select1.hdf ohk.txt 7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file. Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack. sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw 8. Generate a 1D rotationally averaged power spectrum of an image. sxprocess.py vol.hdf --rotwp=rotpw.txt # Output will contain three columns: (1) rotationally averaged power spectrum (2) logarithm of the rotationally averaged power spectrum (3) integer line number (from zero to approximately to half the image size) 9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data. sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt The output file is then imported and 3D transformed volume computed: sxheader.py bdb:p --params=xform.projection --import=output.txt mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI The reconstructed volume is in the position of the volume computed using the input.txt parameters and then transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz) 10. Import ctf parameters from the output of sxcter into windowed particle headers. There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph. In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'. Normally this is done by e2boxer.py during windowing. Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created. sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 # Output will be a vritual stack bdb:Gdata sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 To concatenate output files: cd directory e2bdb.py . --makevstack=bdb:allparticles --filt=G IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them. 11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor sxprocess.py orientationparams.txt scaledparams.txt scale=0.5 12. Generate 3D mask from a given 3-D volume automatically or using threshold provided by user. 13. Postprocess 3-D or 2-D images: for 3-D volumes: calculate FSC with provided mask; weight summed volume with FSC; estimate B-factor from FSC weighted summed two volumes; apply negative B-factor to the weighted volume. for 2-D images: calculate B-factor and apply negative B-factor to 2-D images. 14. Winow stack file -reduce size of images without changing the pixel size. """ parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--order", action="store_true", help="Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False) parser.add_option("--order_lookup", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_metropolis", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_pca", action="store_true", help="Test/Debug.", default=False) parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)") parser.add_option("--circular", action="store_true", help="Select circular ordering (fisr image has to be similar to the last", default=False) parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") parser.add_option("--changesize", action="store_true", help="resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False) parser.add_option("--ratio", type="float", default=1.0, help="The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round") parser.add_option("--pw", action="store_true", help="compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False) parser.add_option("--wn", type="int", default=-1, help="Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))") parser.add_option("--phase_flip", action="store_true", help="Phase flip the input stack", default=False) parser.add_option("--makedb", metavar="param1=value1:param2=value2", type="string", action="append", help="One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0") parser.add_option("--generate_projections", metavar="param1=value1:param2=value2", type="string", action="append", help="Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64.") parser.add_option("--isacgroup", type="int", help="Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1) parser.add_option("--isacselect", action="store_true", help="Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False) parser.add_option("--params", type="string", default=None, help="Name of header of parameter, which one depends on specific option") parser.add_option("--adjpw", action="store_true", help="Adjust rotationally averaged power spectrum of an image", default=False) parser.add_option("--rotpw", type="string", default=None, help="Name of the text file to contain rotationally averaged power spectrum of the input image.") parser.add_option("--transformparams", type="string", default=None, help="Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters.") # import ctf estimates done using cter parser.add_option("--input", type="string", default= None, help="Input particles.") parser.add_option("--importctf", type="string", default= None, help="Name of the file containing CTF parameters produced by sxcter.") parser.add_option("--defocuserror", type="float", default=1000000.0, help="Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%") parser.add_option("--astigmatismerror", type="float", default=360.0, help="Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees.") # import ctf estimates done using cter parser.add_option("--scale", type="float", default=-1.0, help="Divide shifts in the input 3D orientation parameters text file by the scale factor.") # generate adaptive mask from an given 3-D volume parser.add_option("--adaptive_mask", action="store_true", help="create adavptive 3-D mask from a given volume", default=False) parser.add_option("--nsigma", type="float", default= 1., help="number of times of sigma of the input volume to obtain the the large density cluster") parser.add_option("--ndilation", type="int", default= 3, help="number of times of dilation applied to the largest cluster of density") parser.add_option("--kernel_size", type="int", default= 11, help="convolution kernel for smoothing the edge of the mask") parser.add_option("--gauss_standard_dev", type="int", default= 9, help="stanadard deviation value to generate Gaussian edge") parser.add_option("--threshold", type="float", default= 9999., help="threshold provided by user to binarize input volume") parser.add_option("--ne", type="int", default= 0, help="number of times to erode the binarized input image") parser.add_option("--nd", type="int", default= 0, help="number of times to dilate the binarized input image") parser.add_option("--postprocess", action="store_true", help="postprocess unfiltered odd, even 3-D volumes",default=False) parser.add_option("--fsc_weighted", action="store_true", help="postprocess unfiltered odd, even 3-D volumes") parser.add_option("--low_pass_filter", action="store_true", default=False, help="postprocess unfiltered odd, even 3-D volumes") parser.add_option("--ff", type="float", default=.25, help="low pass filter stop band frequency in absolute unit") parser.add_option("--aa", type="float", default=.1, help="low pass filter falloff" ) parser.add_option("--mask", type="string", help="input mask file", default=None) parser.add_option("--output", type="string", help="output file name", default="postprocessed.hdf") parser.add_option("--pixel_size", type="float", help="pixel size of the data", default=1.0) parser.add_option("--B_start", type="float", help="starting frequency in Angstrom for B-factor estimation", default=10.) parser.add_option("--FSC_cutoff", type="float", help="stop frequency in Angstrom for B-factor estimation", default=0.143) parser.add_option("--2d", action="store_true", help="postprocess isac 2-D averaged images",default=False) parser.add_option("--window_stack", action="store_true", help="window stack images using a smaller window size", default=False) parser.add_option("--box", type="int", default= 0, help="the new window size ") (options, args) = parser.parse_args() global_def.BATCH = True if options.phase_flip: nargs = len(args) if nargs != 2: print "must provide name of input and output file!" return from EMAN2 import Processor instack = args[0] outstack = args[1] nima = EMUtil.get_image_count(instack) from filter import filt_ctf for i in xrange(nima): img = EMData() img.read_image(instack, i) try: ctf = img.get_attr('ctf') except: print "no ctf information in input stack! Exiting..." return dopad = True sign = 1 binary = 1 # phase flip assert img.get_ysize() > 1 dict = ctf.to_dict() dz = dict["defocus"] cs = dict["cs"] voltage = dict["voltage"] pixel_size = dict["apix"] b_factor = dict["bfactor"] ampcont = dict["ampcont"] dza = dict["dfdiff"] azz = dict["dfang"] if dopad and not img.is_complex(): ip = 1 else: ip = 0 params = {"filter_type": Processor.fourier_filter_types.CTF_, "defocus" : dz, "Cs": cs, "voltage": voltage, "Pixel_size": pixel_size, "B_factor": b_factor, "amp_contrast": ampcont, "dopad": ip, "binary": binary, "sign": sign, "dza": dza, "azz":azz} tmp = Processor.EMFourierFilter(img, params) tmp.set_attr_dict({"ctf": ctf}) tmp.write_image(outstack, i) elif options.changesize: nargs = len(args) if nargs != 2: ERROR("must provide name of input and output file!", "change size", 1) return from utilities import get_im instack = args[0] outstack = args[1] sub_rate = float(options.ratio) nima = EMUtil.get_image_count(instack) from fundamentals import resample for i in xrange(nima): resample(get_im(instack, i), sub_rate).write_image(outstack, i) elif options.isacgroup>-1: nargs = len(args) if nargs != 3: ERROR("Three files needed on input!", "isacgroup", 1) return from utilities import get_im instack = args[0] m=get_im(args[1],int(options.isacgroup)).get_attr("members") l = [] for k in m: l.append(int(get_im(args[0],k).get_attr(options.params))) from utilities import write_text_file write_text_file(l, args[2]) elif options.isacselect: nargs = len(args) if nargs != 2: ERROR("Two files needed on input!", "isacgroup", 1) return from utilities import get_im nima = EMUtil.get_image_count(args[0]) m = [] for k in xrange(nima): m += get_im(args[0],k).get_attr("members") m.sort() from utilities import write_text_file write_text_file(m, args[1]) elif options.pw: nargs = len(args) if nargs < 2: ERROR("must provide name of input and output file!", "pw", 1) return from utilities import get_im, write_text_file from fundamentals import rops_table d = get_im(args[0]) ndim = d.get_ndim() if ndim ==3: pw = rops_table(d) write_text_file(pw, args[1]) else: nx = d.get_xsize() ny = d.get_ysize() if nargs ==3: mask = get_im(args[2]) wn = int(options.wn) if wn == -1: wn = max(nx, ny) else: if( (wn<nx) or (wn<ny) ): ERROR("window size cannot be smaller than the image size","pw",1) n = EMUtil.get_image_count(args[0]) from utilities import model_blank, model_circle, pad from EMAN2 import periodogram p = model_blank(wn,wn) for i in xrange(n): d = get_im(args[0], i) if nargs==3: d *=mask st = Util.infomask(d, None, True) d -= st[0] p += periodogram(pad(d, wn, wn, 1, 0.)) p /= n p.write_image(args[1]) elif options.adjpw: if len(args) < 3: ERROR("filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)","adjpw",1) return img_stack = args[0] from math import sqrt from fundamentals import rops_table, fft from utilities import read_text_file, get_im from filter import filt_tanl, filt_table if( args[1][-3:] == 'txt'): rops_dst = read_text_file( args[1] ) else: rops_dst = rops_table(get_im( args[1] )) out_stack = args[2] if(len(args) >4): fl = float(args[3]) aa = float(args[4]) else: fl = -1.0 aa = 0.0 nimage = EMUtil.get_image_count( img_stack ) for i in xrange(nimage): img = fft(get_im(img_stack, i) ) rops_src = rops_table(img) assert len(rops_dst) == len(rops_src) table = [0.0]*len(rops_dst) for j in xrange( len(rops_dst) ): table[j] = sqrt( rops_dst[j]/rops_src[j] ) if( fl > 0.0): img = filt_tanl(img, fl, aa) img = fft(filt_table(img, table)) img.write_image(out_stack, i) elif options.rotpw != None: if len(args) != 1: ERROR("Only one input permitted","rotpw",1) return from utilities import write_text_file, get_im from fundamentals import rops_table from math import log10 t = rops_table(get_im(args[0])) x = range(len(t)) r = [0.0]*len(x) for i in x: r[i] = log10(t[i]) write_text_file([t,r,x],options.rotpw) elif options.transformparams != None: if len(args) != 2: ERROR("Please provide names of input and output files with orientation parameters","transformparams",1) return from utilities import read_text_row, write_text_row transf = [0.0]*6 spl=options.transformparams.split(',') for i in xrange(len(spl)): transf[i] = float(spl[i]) write_text_row( rotate_shift_params(read_text_row(args[0]), transf) , args[1]) elif options.makedb != None: nargs = len(args) if nargs != 1: print "must provide exactly one argument denoting database key under which the input params will be stored" return dbkey = args[0] print "database key under which params will be stored: ", dbkey gbdb = js_open_dict("e2boxercache/gauss_box_DB.json") parmstr = 'dummy:'+options.makedb[0] (processorname, param_dict) = parsemodopt(parmstr) dbdict = {} for pkey in param_dict: if (pkey == 'invert_contrast') or (pkey == 'use_variance'): if param_dict[pkey] == 'True': dbdict[pkey] = True else: dbdict[pkey] = False else: dbdict[pkey] = param_dict[pkey] gbdb[dbkey] = dbdict elif options.generate_projections: nargs = len(args) if nargs != 3: ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\ "sxprocess - generate projections",1) return inpstr = args[0] outstk = args[1] micpref = args[2] parmstr = 'dummy:'+options.generate_projections[0] (processorname, param_dict) = parsemodopt(parmstr) parm_CTF = False parm_format = 'bdb' parm_apix = 2.5 if 'CTF' in param_dict: if param_dict['CTF'] == 'True': parm_CTF = True if 'format' in param_dict: parm_format = param_dict['format'] if 'apix' in param_dict: parm_apix = float(param_dict['apix']) boxsize = 64 if 'boxsize' in param_dict: boxsize = int(param_dict['boxsize']) print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize scale_mult = 2500 sigma_add = 1.5 sigma_proj = 30.0 sigma2_proj = 17.5 sigma_gauss = 0.3 sigma_mic = 30.0 sigma2_mic = 17.5 sigma_gauss_mic = 0.3 if 'scale_mult' in param_dict: scale_mult = float(param_dict['scale_mult']) if 'sigma_add' in param_dict: sigma_add = float(param_dict['sigma_add']) if 'sigma_proj' in param_dict: sigma_proj = float(param_dict['sigma_proj']) if 'sigma2_proj' in param_dict: sigma2_proj = float(param_dict['sigma2_proj']) if 'sigma_gauss' in param_dict: sigma_gauss = float(param_dict['sigma_gauss']) if 'sigma_mic' in param_dict: sigma_mic = float(param_dict['sigma_mic']) if 'sigma2_mic' in param_dict: sigma2_mic = float(param_dict['sigma2_mic']) if 'sigma_gauss_mic' in param_dict: sigma_gauss_mic = float(param_dict['sigma_gauss_mic']) from filter import filt_gaussl, filt_ctf from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank,pad,model_gauss_noise,set_params2D, set_params_proj from projection import prep_vol,prgs seed(14567) delta = 29 angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S") nangle = len(angles) modelvol = [] nvlms = EMUtil.get_image_count(inpstr) from utilities import get_im for k in xrange(nvlms): modelvol.append(get_im(inpstr,k)) nx = modelvol[0].get_xsize() if nx != boxsize: ERROR("Requested box dimension does not match dimension of the input model.", \ "sxprocess - generate projections",1) nvol = 10 volfts = [[] for k in xrange(nvlms)] for k in xrange(nvlms): for i in xrange(nvol): sigma = sigma_add + random() # 1.5-2.5 addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40 ) scale = scale_mult * (0.5+random()) vf, kb = prep_vol(modelvol[k] + scale*addon) volfts[k].append(vf) del vf, modelvol if parm_format == "bdb": stack_data = "bdb:"+outstk delete_bdb(stack_data) else: stack_data = outstk + ".hdf" Cs = 2.0 pixel = parm_apix voltage = 120.0 ampcont = 10.0 ibd = 4096/2-boxsize iprj = 0 width = 240 xstart = 8 + boxsize/2 ystart = 8 + boxsize/2 rowlen = 17 from random import randint params = [] for idef in xrange(3, 8): irow = 0 icol = 0 mic = model_blank(4096, 4096) defocus = idef * 0.5#0.2 if parm_CTF: astampl=defocus*0.15 astangl=50.0 ctf = generate_ctf([defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl]) for i in xrange(nangle): for k in xrange(12): dphi = 8.0*(random()-0.5) dtht = 8.0*(random()-0.5) psi = 360.0*random() phi = angles[i][0]+dphi tht = angles[i][1]+dtht s2x = 4.0*(random()-0.5) s2y = 4.0*(random()-0.5) params.append([phi, tht, psi, s2x, s2y]) ivol = iprj % nvol #imgsrc = randint(0,nvlms-1) imgsrc = iprj % nvlms proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y]) x = xstart + irow * width y = ystart + icol * width mic += pad(proj, 4096, 4096, 1, 0.0, x-2048, y-2048, 0) proj = proj + model_gauss_noise( sigma_proj, nx, nx ) if parm_CTF: proj = filt_ctf(proj, ctf) proj.set_attr_dict({"ctf":ctf, "ctf_applied":0}) proj = proj + filt_gaussl(model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss) proj.set_attr("origimgsrc",imgsrc) proj.set_attr("test_id", iprj) # flags describing the status of the image (1 = true, 0 = false) set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0]) set_params_proj(proj, [phi, tht, psi, s2x, s2y]) proj.write_image(stack_data, iprj) icol += 1 if icol == rowlen: icol = 0 irow += 1 iprj += 1 mic += model_gauss_noise(sigma_mic,4096,4096) if parm_CTF: #apply CTF mic = filt_ctf(mic, ctf) mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic) mic.write_image(micpref + "%1d.hdf" % (idef-3), 0) drop_spider_doc("params.txt", params) elif options.importctf != None: print ' IMPORTCTF ' from utilities import read_text_row,write_text_row from random import randint import subprocess grpfile = 'groupid%04d'%randint(1000,9999) ctfpfile = 'ctfpfile%04d'%randint(1000,9999) cterr = [options.defocuserror/100.0, options.astigmatismerror] ctfs = read_text_row(options.importctf) for kk in xrange(len(ctfs)): root,name = os.path.split(ctfs[kk][-1]) ctfs[kk][-1] = name[:-4] if(options.input[:4] != 'bdb:'): ERROR('Sorry, only bdb files implemented','importctf',1) d = options.input[4:] #try: str = d.index('*') #except: str = -1 from string import split import glob uu = os.path.split(d) uu = os.path.join(uu[0],'EMAN2DB',uu[1]+'.bdb') flist = glob.glob(uu) for i in xrange(len(flist)): root,name = os.path.split(flist[i]) root = root[:-7] name = name[:-4] fil = 'bdb:'+os.path.join(root,name) sourcemic = EMUtil.get_all_attributes(fil,'ptcl_source_image') nn = len(sourcemic) gctfp = [] groupid = [] for kk in xrange(nn): junk,name2 = os.path.split(sourcemic[kk]) name2 = name2[:-4] ctfp = [-1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] for ll in xrange(len(ctfs)): if(name2 == ctfs[ll][-1]): # found correct if(ctfs[ll][8]/ctfs[ll][0] <= cterr[0]): # acceptable defocus error ctfp = ctfs[ll][:8] if(ctfs[ll][10] > cterr[1] ): # error of astigmatism exceed the threshold, set astigmatism to zero. ctfp[6] = 0.0 ctfp[7] = 0.0 gctfp.append(ctfp) groupid.append(kk) break if(len(groupid) > 0): write_text_row(groupid, grpfile) write_text_row(gctfp, ctfpfile) cmd = "{} {} {} {}".format('e2bdb.py',fil,'--makevstack=bdb:'+root+'G'+name,'--list='+grpfile) #print cmd subprocess.call(cmd, shell=True) cmd = "{} {} {} {}".format('sxheader.py','bdb:'+root+'G'+name,'--params=ctf','--import='+ctfpfile) #print cmd subprocess.call(cmd, shell=True) else: print ' >>> Group ',name,' skipped.' cmd = "{} {} {}".format("rm -f",grpfile,ctfpfile) subprocess.call(cmd, shell=True) elif options.scale > 0.0: from utilities import read_text_row,write_text_row scale = options.scale nargs = len(args) if nargs != 2: print "Please provide names of input and output file!" return p = read_text_row(args[0]) for i in xrange(len(p)): p[i][3] /= scale p[i][4] /= scale write_text_row(p, args[1]) elif options.adaptive_mask: from utilities import get_im from morphology import adaptive_mask, binarize, erosion, dilation nsigma = options.nsigma ndilation = options.ndilation kernel_size = options.kernel_size gauss_standard_dev = options.gauss_standard_dev nargs = len(args) if nargs ==0: print " Create 3D mask from a given volume, either automatically or from the user provided threshold." elif nargs > 2: print "Too many inputs are given, try again!" return else: inputvol = get_im(args[0]) input_path, input_file_name = os.path.split(args[0]) input_file_name_root,ext=os.path.splitext(input_file_name) if nargs == 2: mask_file_name = args[1] else: mask_file_name = "adaptive_mask_for_"+input_file_name_root+".hdf" # Only hdf file is output. if options.threshold !=9999.: mask3d = binarize(inputvol, options.threshold) for i in xrange(options.ne): mask3d = erosion(mask3d) for i in xrange(options.nd): mask3d = dilation(mask3d) else: mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev) mask3d.write_image(mask_file_name) elif options.postprocess: from utilities import get_im from fundamentals import rot_avg_table from morphology import compute_bfactor,power from statistics import fsc from filter import filt_table, filt_gaussinv from EMAN2 import periodogram e1 = get_im(args[0],0) if e1.get_zsize()==1: nimage = EMUtil.get_image_count(args[0]) if options.mask !=None: m = get_im(options.mask) else: m = None for i in xrange(nimage): e1 = get_im(args[0],i) if m: e1 *=m guinerline = rot_avg_table(power(periodogram(e1),.5)) freq_max = 1/(2.*pixel_size) freq_min = 1./options.B_start b,junk=compute_bfactor(guinerline, freq_min, freq_max, pixel_size) tmp = b/pixel_size**2 sigma_of_inverse=sqrt(2./tmp) e1 = filt_gaussinv(e1,sigma_of_inverse) if options.low_pass_filter: from filter import filt_tanl e1 =filt_tanl(e1,options.ff, options.aa) e1.write_image(options.output) else: nargs = len(args) e1 = get_im(args[0]) if nargs >1: e2 = get_im(args[1]) if options.mask !=None: m = get_im(options.mask) else: m =None pixel_size = options.pixel_size from math import sqrt if m !=None: e1 *=m if nargs >1 :e2 *=m if options.fsc_weighted: frc = fsc(e1,e2,1) ## FSC is done on masked two images #### FSC weighting sqrt((2.*fsc)/(1+fsc)); fil = len(frc[1])*[None] for i in xrange(len(fil)): if frc[1][i]>=options.FSC_cutoff: tmp = frc[1][i] else: tmp = 0.0 fil[i] = sqrt(2.*tmp/(1.+tmp)) if nargs>1: e1 +=e2 if options.fsc_weighted: e1=filt_table(e1,fil) guinerline = rot_avg_table(power(periodogram(e1),.5)) freq_max = 1/(2.*pixel_size) freq_min = 1./options.B_start b,junk = compute_bfactor(guinerline, freq_min, freq_max, pixel_size) tmp = b/pixel_size**2 sigma_of_inverse=sqrt(2./tmp) e1 = filt_gaussinv(e1,sigma_of_inverse) if options.low_pass_filter: from filter import filt_tanl e1 =filt_tanl(e1,options.ff, options.aa) e1.write_image(options.output) elif options.window_stack: nargs = len(args) if nargs ==0: print " Reduce image size of a stack" return else: output_stack_name = None inputstack = args[0] if nargs ==2:output_stack_name = args[1] input_path,input_file_name=os.path.split(inputstack) input_file_name_root,ext=os.path.splitext(input_file_name) if input_file_name_root[0:3]=="bdb":stack_is_bdb= True else: stack_is_bdb= False if output_stack_name is None: if stack_is_bdb: output_stack_name ="bdb:reduced_"+input_file_name_root[4:] else:output_stack_name = "reduced_"+input_file_name_root+".hdf" # Only hdf file is output. nimage = EMUtil.get_image_count(inputstack) from fundamentals import window2d for i in xrange(nimage): image = EMData() image.read_image(inputstack,i) w = window2d(image,options.box,options.box) w.write_image(output_stack_name,i) else: ERROR("Please provide option name","sxprocess.py",1)
def main(): def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror): if mirror: m = 1 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0) else: m = 0 alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0) return alpha, sx, sy, m progname = os.path.basename(sys.argv[0]) usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=0.2 --aa=0.1 --sym=symmetry --CTF" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--ave2D", type="string" , default=False, help="write to the disk a stack of 2D averages") parser.add_option("--var2D", type="string" , default=False, help="write to the disk a stack of 2D variances") parser.add_option("--ave3D", type="string" , default=False, help="write to the disk reconstructed 3D average") parser.add_option("--var3D", type="string" , default=False, help="compute 3D variability (time consuming!)") parser.add_option("--img_per_grp", type="int" , default=10 , help="number of neighbouring projections") parser.add_option("--no_norm", action="store_true", default=False, help="do not use normalization") parser.add_option("--radiusvar", type="int" , default=-1 , help="radius for 3D var" ) parser.add_option("--npad", type="int" , default=2 , help="number of time to pad the original images") parser.add_option("--sym" , type="string" , default="c1" , help="symmetry") parser.add_option("--fl", type="float" , default=0.0 , help="stop-band frequency (Default - no filtration)") parser.add_option("--aa", type="float" , default=0.0 , help="fall off of the filter (Default - no filtration)") parser.add_option("--CTF", action="store_true", default=False, help="use CFT correction") parser.add_option("--VERBOSE", action="store_true", default=False, help="Long output for debugging") #parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version") #parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" ) #parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" ) #parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" ) #parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" ) parser.add_option("--VAR" , action="store_true", default=False, help="stack on input consists of 2D variances (Default False)") parser.add_option("--decimate", type="float", default=1.0, help="image decimate rate, a number large than 1. default is 1") parser.add_option("--window", type="int", default=0, help="reduce images to a small image size without changing pixel_size. Default value is zero.") #parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)") parser.add_option("--nvec", type="int" , default=0 , help="number of eigenvectors, default = 0 meaning no PCA calculated") parser.add_option("--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)") (options,args) = parser.parse_args() ##### from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD, MPI_TAG_UB from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX from applications import MPI_start_end from reconstruction import recons3d_em, recons3d_em_MPI from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI from utilities import print_begin_msg, print_end_msg, print_msg from utilities import read_text_row, get_image, get_im from utilities import bcast_EMData_to_all, bcast_number_to_all from utilities import get_symt # This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015 from EMAN2db import db_open_dict if options.symmetrize : try: sys.argv = mpi_init(len(sys.argv), sys.argv) try: number_of_proc = mpi_comm_size(MPI_COMM_WORLD) if( number_of_proc > 1 ): ERROR("Cannot use more than one CPU for symmetry prepration","sx3dvariability",1) except: pass except: pass # Input #instack = "Clean_NORM_CTF_start_wparams.hdf" #instack = "bdb:data" instack = args[0] sym = options.sym if( sym == "c1" ): ERROR("Thre is no need to symmetrize stack for C1 symmetry","sx3dvariability",1) if(instack[:4] !="bdb:"): stack = "bdb:data" delete_bdb(stack) cmdexecute("sxcpy.py "+instack+" "+stack) else: stack = instack qt = EMUtil.get_all_attributes(stack,'xform.projection') na = len(qt) ts = get_symt(sym) ks = len(ts) angsa = [None]*na for k in xrange(ks): delete_bdb("bdb:Q%1d"%k) cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k) DB = db_open_dict("bdb:Q%1d"%k) for i in xrange(na): ut = qt[i]*ts[k] DB.set_attr(i, "xform.projection", ut) #bt = ut.get_params("spider") #angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]] #write_text_row(angsa, 'ptsma%1d.txt'%k) #cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k) #cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k)) DB.close() delete_bdb("bdb:sdata") cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q") #cmdexecute("ls EMAN2DB/sdata*") a = get_im("bdb:sdata") a.set_attr("variabilitysymmetry",sym) a.write_image("bdb:sdata") else: sys.argv = mpi_init(len(sys.argv), sys.argv) myid = mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 1: stack = args[0] else: print( "usage: " + usage) print( "Please run '" + progname + " -h' for detailed options") return 1 t0 = time() # obsolete flags options.MPI = True options.nvec = 0 options.radiuspca = -1 options.iter = 40 options.abs = 0.0 options.squ = 0.0 if options.fl > 0.0 and options.aa == 0.0: ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid) if options.VAR and options.SND: ERROR("Only one of var and SND can be set!", "sx3dvariability", myid) exit() if options.VAR and (options.ave2D or options.ave3D or options.var2D): ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid) exit() #if options.SND and (options.ave2D or options.ave3D): # ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid) # exit() if options.nvec > 0 : ERROR("PCA option not implemented", "sx3dvariability", 1, myid) exit() if options.nvec > 0 and options.ave3D == None: ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid) exit() import string options.sym = options.sym.lower() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True if myid == main_node: print_begin_msg("sx3dvariability") print_msg("%-70s: %s\n"%("Input stack", stack)) img_per_grp = options.img_per_grp nvec = options.nvec radiuspca = options.radiuspca symbaselen = 0 if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() if options.sym != "c1" : imgdata = get_im(stack) try: i = imgdata.get_attr("variabilitysymmetry") if(i != options.sym): ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", myid=myid) except: ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", myid=myid) from utilities import get_symt i = len(get_symt(options.sym)) if((nima/i)*i != nima): ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", myid=myid) symbaselen = nima/i else: symbaselen = nima else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) Tracker ={} Tracker["nx"] =nx Tracker["ny"] =ny Tracker["total_stack"]=nima if options.decimate==1.: if options.window !=0: nx = options.window ny = options.window else: if options.window ==0: nx = int(nx/options.decimate) ny = int(ny/options.decimate) else: nx = int(options.window/options.decimate) ny = nx symbaselen = bcast_number_to_all(symbaselen) if radiuspca == -1: radiuspca = nx/2-2 if myid == main_node: print_msg("%-70s: %d\n"%("Number of projection", nima)) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) """ if options.SND: from projection import prep_vol, prgs from statistics import im_diff from utilities import get_im, model_circle, get_params_proj, set_params_proj from utilities import get_ctf, generate_ctf from filter import filt_ctf imgdata = EMData.read_images(stack, range(img_begin, img_end)) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) bcast_EMData_to_all(vol, myid) volft, kb = prep_vol(vol) mask = model_circle(nx/2-2, nx, ny) varList = [] for i in xrange(img_begin, img_end): phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin]) ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y]) if options.CTF: ctf_params = get_ctf(imgdata[i-img_begin]) ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params)) diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask) diff2 = diff*diff set_params_proj(diff2, [phi, theta, psi, s2x, s2y]) varList.append(diff2) mpi_barrier(MPI_COMM_WORLD) """ if options.VAR: #varList = EMData.read_images(stack, range(img_begin, img_end)) varList = [] this_image = EMData() for index_of_particle in xrange(img_begin,img_end): this_image.read_image(stack,index_of_particle) varList.append(image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF)) else: from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2 from utilities import model_blank, nearest_proj, model_circle from applications import pca from statistics import avgvar, avgvar_ctf, ccc from filter import filt_tanl from morphology import threshold, square_root from projection import project, prep_vol, prgs from sets import Set if myid == main_node: t1 = time() proj_angles = [] aveList = [] tab = EMUtil.get_all_attributes(stack, 'xform.projection') for i in xrange(nima): t = tab[i].get_params('spider') phi = t['phi'] theta = t['theta'] psi = t['psi'] x = theta if x > 90.0: x = 180.0 - x x = x*10000+psi proj_angles.append([x, t['phi'], t['theta'], t['psi'], i]) t2 = time() print_msg("%-70s: %d\n"%("Number of neighboring projections", img_per_grp)) print_msg("...... Finding neighboring projections\n") if options.VERBOSE: print "Number of images per group: ", img_per_grp print "Now grouping projections" proj_angles.sort() proj_angles_list = [0.0]*(nima*4) if myid == main_node: for i in xrange(nima): proj_angles_list[i*4] = proj_angles[i][1] proj_angles_list[i*4+1] = proj_angles[i][2] proj_angles_list[i*4+2] = proj_angles[i][3] proj_angles_list[i*4+3] = proj_angles[i][4] proj_angles_list = bcast_list_to_all(proj_angles_list, myid, main_node) proj_angles = [] for i in xrange(nima): proj_angles.append([proj_angles_list[i*4], proj_angles_list[i*4+1], proj_angles_list[i*4+2], int(proj_angles_list[i*4+3])]) del proj_angles_list proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end)) all_proj = Set() for im in proj_list: for jm in im: all_proj.add(proj_angles[jm][3]) all_proj = list(all_proj) if options.VERBOSE: print "On node %2d, number of images needed to be read = %5d"%(myid, len(all_proj)) index = {} for i in xrange(len(all_proj)): index[all_proj[i]] = i mpi_barrier(MPI_COMM_WORLD) if myid == main_node: print_msg("%-70s: %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2)) print_msg("%-70s: %d\n"%("Number of groups processed on the main node", len(proj_list))) if options.VERBOSE: print "Grouping projections took: ", (time()-t2)/60 , "[min]" print "Number of groups on main node: ", len(proj_list) mpi_barrier(MPI_COMM_WORLD) if myid == main_node: print_msg("...... calculating the stack of 2D variances \n") if options.VERBOSE: print "Now calculating the stack of 2D variances" proj_params = [0.0]*(nima*5) aveList = [] varList = [] if nvec > 0: eigList = [[] for i in xrange(nvec)] if options.VERBOSE: print "Begin to read images on processor %d"%(myid) ttt = time() #imgdata = EMData.read_images(stack, all_proj) img = EMData() imgdata = [] for index_of_proj in xrange(len(all_proj)): img.read_image(stack, all_proj[index_of_proj]) dmg = image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF) #print dmg.get_xsize(), "init" imgdata.append(dmg) if options.VERBOSE: print "Reading images on processor %d done, time = %.2f"%(myid, time()-ttt) print "On processor %d, we got %d images"%(myid, len(imgdata)) mpi_barrier(MPI_COMM_WORLD) ''' imgdata2 = EMData.read_images(stack, range(img_begin, img_end)) if options.fl > 0.0: for k in xrange(len(imgdata2)): imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa) if options.CTF: vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) else: vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1) if myid == main_node: vol.write_image("vol_ctf.hdf") print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf")) del vol, imgdata2 mpi_barrier(MPI_COMM_WORLD) ''' from applications import prepare_2d_forPCA from utilities import model_blank for i in xrange(len(proj_list)): ki = proj_angles[proj_list[i][0]][3] if ki >= symbaselen: continue mi = index[ki] phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi]) grp_imgdata = [] for j in xrange(img_per_grp): mj = index[proj_angles[proj_list[i][j]][3]] phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj]) alpha, sx, sy, mirror = params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror_list[i][j]) if thetaM <= 90: if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM-phi, 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM-phi), 0.0, 0.0, 1.0) else: if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM-phi), 0.0, 0.0, 1.0) else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM-phi)), 0.0, 0.0, 1.0) set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0]) grp_imgdata.append(imgdata[mj]) #print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize() if not options.no_norm: #print grp_imgdata[j].get_xsize() mask = model_circle(nx/2-2, nx, nx) for k in xrange(img_per_grp): ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False) grp_imgdata[k] -= ave grp_imgdata[k] /= std del mask if options.fl > 0.0: from filter import filt_ctf, filt_table from fundamentals import fft, window2d nx2 = 2*nx ny2 = 2*ny if options.CTF: from utilities import pad for k in xrange(img_per_grp): grp_imgdata[k] = window2d(fft( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa) ),nx,ny) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) else: for k in xrange(img_per_grp): grp_imgdata[k] = filt_tanl( grp_imgdata[k], options.fl, options.aa) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) else: from utilities import pad, read_text_file from filter import filt_ctf, filt_table from fundamentals import fft, window2d nx2 = 2*nx ny2 = 2*ny if options.CTF: from utilities import pad for k in xrange(img_per_grp): grp_imgdata[k] = window2d( fft( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1) ) , nx,ny) #grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny) #grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa) ''' if i < 10 and myid == main_node: for k in xrange(10): grp_imgdata[k].write_image("grp%03d.hdf"%i, k) ''' """ if myid == main_node and i==0: for pp in xrange(len(grp_imgdata)): grp_imgdata[pp].write_image("pp.hdf", pp) """ ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata) """ if myid == main_node and i==0: for pp in xrange(len(grp_imgdata)): grp_imgdata[pp].write_image("qq.hdf", pp) """ var = model_blank(nx,ny) for q in grp_imgdata: Util.add_img2( var, q ) Util.mul_scalar( var, 1.0/(len(grp_imgdata)-1)) # Switch to std dev var = square_root(threshold(var)) #if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a") #else: ave, var = avgvar(grp_imgdata, mode="a") """ if myid == main_node: ave.write_image("avgv.hdf",i) var.write_image("varv.hdf",i) """ set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0]) set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0]) aveList.append(ave) varList.append(var) if options.VERBOSE: print "%5.2f%% done on processor %d"%(i*100.0/len(proj_list), myid) if nvec > 0: eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True) for k in xrange(nvec): set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0]) eigList[k].append(eig[k]) """ if myid == 0 and i == 0: for k in xrange(nvec): eig[k].write_image("eig.hdf", k) """ del imgdata # To this point, all averages, variances, and eigenvectors are computed if options.ave2D: from fundamentals import fpol if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node : for im in xrange(len(aveList)): aveList[im].write_image(options.ave2D, km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im+i+70000) """ nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('members', map(int, members)) members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('pix_err', map(float, members)) members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD) ave.set_attr('refprojdir', map(float, members)) """ tmpvol=fpol(ave, Tracker["nx"],Tracker["nx"],Tracker["nx"]) tmpvol.write_image(options.ave2D, km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) for im in xrange(len(aveList)): send_EMData(aveList[im], main_node,im+myid+70000) """ members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) members = aveList[im].get_attr('pix_err') mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) except: mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) """ if options.ave3D: from fundamentals import fpol if options.VERBOSE: print "Reconstructing 3D average volume" ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(ave3D, myid) if myid == main_node: ave3D=fpol(ave3D,Tracker["nx"],Tracker["nx"],Tracker["nx"]) ave3D.write_image(options.ave3D) print_msg("%-70s: %s\n"%("Writing to the disk volume reconstructed from averages as", options.ave3D)) del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList if nvec > 0: for k in xrange(nvec): if options.VERBOSE: print "Reconstruction eigenvolumes", k cont = True ITER = 0 mask2d = model_circle(radiuspca, nx, nx) while cont: #print "On node %d, iteration %d"%(myid, ITER) eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad) bcast_EMData_to_all(eig3D, myid, main_node) if options.fl > 0.0: eig3D = filt_tanl(eig3D, options.fl, options.aa) if myid == main_node: eig3D.write_image("eig3d_%03d.hdf"%k, ITER) Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) ) eig3Df, kb = prep_vol(eig3D) del eig3D cont = False icont = 0 for l in xrange(len(eigList[k])): phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l]) proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y]) cl = ccc(proj, eigList[k][l], mask2d) if cl < 0.0: icont += 1 cont = True eigList[k][l] *= -1.0 u = int(cont) u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD) icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD) if myid == main_node: u = int(u[0]) print " Eigenvector: ",k," number changed ",int(icont[0]) else: u = 0 u = bcast_number_to_all(u, main_node) cont = bool(u) ITER += 1 del eig3Df, kb mpi_barrier(MPI_COMM_WORLD) del eigList, mask2d if options.ave3D: del ave3D if options.var2D: from fundamentals import fpol if myid == main_node: km = 0 for i in xrange(number_of_proc): if i == main_node : for im in xrange(len(varList)): tmpvol=fpol(varList[im], Tracker["nx"], Tracker["nx"],1) tmpvol.write_image(options.var2D, km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD) nl = int(nl[0]) for im in xrange(nl): ave = recv_EMData(i, im+i+70000) tmpvol=fpol(ave, Tracker["nx"], Tracker["nx"],1) tmpvol.write_image(options.var2D, km) km += 1 else: mpi_send(len(varList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD) for im in xrange(len(varList)): send_EMData(varList[im], main_node, im+myid+70000)# What with the attributes?? mpi_barrier(MPI_COMM_WORLD) if options.var3D: if myid == main_node and options.VERBOSE: print "Reconstructing 3D variability volume" t6 = time() radiusvar = options.radiusvar if( radiusvar < 0 ): radiusvar = nx//2 -3 res = recons3d_4nn_MPI(myid, varList, symmetry=options.sym, npad=options.npad) #res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ) if myid == main_node: from fundamentals import fpol res =fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"]) res.write_image(options.var3D) if myid == main_node: print_msg("%-70s: %.2f\n"%("Reconstructing 3D variability took [s]", time()-t6)) if options.VERBOSE: print "Reconstruction took: %.2f [min]"%((time()-t6)/60) if myid == main_node: print_msg("%-70s: %.2f\n"%("Total time for these computations [s]", time()-t0)) if options.VERBOSE: print "Total time for these computations: %.2f [min]"%((time()-t0)/60) print_end_msg("sx3dvariability") global_def.BATCH = False from mpi import mpi_finalize mpi_finalize()
def separate_class(classavgstack, instack, options, outdir='.', verbose=False): """ Main function overseeing various projection-comparison modes. Arguments: classavgstack : Input image stack classmap : Class-to-particle lookup table. Each (long) line contains particles assigned to a class, one file for all classes options : (list) Command-line options, run 'sxproj_compare.py -h' for an exhaustive list outdir : Output directory verbose : (boolean) Whether to write additional information to screen """ # Set output directory and log file name prepare_outdir(outdir, verbose) log, verbose = prepare_log(outdir, verbose) # Expand paths for outputs classmap = os.path.join(outdir, CLASSMAPFILE) classdoc = os.path.join(outdir, CLASSDOCPREFIX + '{0:03d}.txt') outbdb = os.path.join(outdir, CLASSSTACKPREFIX + '{0:03d}') outflt = os.path.join(outdir, FILTSTACKPREFIX + '{0:03d}') num_classes = EMUtil.get_image_count(classavgstack) # Generate class-to-particle lookup table and class-selection lists vomq(classavgstack, classmap, classdoc, log=log, verbose=verbose) print_log_msg("Writing each class to a separate stack", log, verbose) if options.shrink: print_log_msg( "Will downsample stacks by a factor of %s" % options.shrink, log, verbose) if options.filtrad: if options.pxsz: filtrad = options.pxsz / options.filtrad else: filtrad = options.filtrad print_log_msg("Will low-pass filter to %s px^-1" % options.shrink, log, verbose) tot_parts = 0 # Loop through classes for class_num in xrange(num_classes): # Write BDB stacks cmd = "e2bdb.py %s --makevstack bdb:%s --list %s" % ( instack, outbdb.format(class_num), classdoc.format(class_num)) print_log_msg(cmd, log, verbose) os.system(cmd) num_class_imgs = EMUtil.get_image_count('bdb:' + outbdb.format(class_num)) tot_parts += num_class_imgs # Optional filtered stack if options.filtrad or options.shrink: cmd = "e2proc2d.py bdb:%s bdb:%s --inplace" % ( outbdb.format(class_num), outflt.format(class_num)) # --inplace overwrites existing images if options.filtrad: cmd = cmd + " --process=filter.lowpass.gauss:cutoff_freq=%s" % filtrad if options.shrink: cmd = cmd + " --meanshrink=%s" % options.shrink print_log_msg(cmd, log, verbose) os.system(cmd) print_log_msg("\nDone! Separated %s particles from %s classes\n" % (tot_parts, num_classes), log, verbose=True)
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Forms chains of 2D images based on their similarities. Functionality: 4. Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent). Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers. The ways to use the program: 4.1 Use option initial to specify which image will be used as an initial seed to form the chain. sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25 4.2 If options initial is omitted, the program will determine which image best serves as initial seed to form the chain sxprocess.py input_stack.hdf output_stack.hdf --radius=25 4.3 Use option circular to form a circular chain. sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25 4.4 New circular code based on pairwise alignments sxprocess.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt 4.5 Circular ordering based on pairwise alignments sxprocess.py vols.hdf chain.hdf mask.hdf --dd --radius=25 """ parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False) parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False) parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False) parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)") parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") # import params for 2D alignment parser.add_option("--ou", type="int", default=-1, help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)") parser.add_option("--xr", type="int", default=0, help="range for translation search in x direction, search is +/xr (0)") parser.add_option("--yr", type="int", default=0, help="range for translation search in y direction, search is +/yr (0)") #parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)") parser.add_option("--pairwiseccc", type="string", default= None, help="Input/output pairwise ccc file") (options, args) = parser.parse_args() global_def.BATCH = True if options.dd: nargs = len(args) if nargs != 3: print "must provide name of input and two output files!" return stack = args[0] new_stack = args[1] from utilities import model_circle from statistics import ccc from statistics import mono lend = EMUtil.get_image_count(stack) lccc = [None]*(lend*(lend-1)/2) for i in xrange(lend-1): v1 = get_im( stack, i ) if( i == 0 and nargs == 2): nx = v1.get_xsize() ny = v1.get_ysize() nz = v1.get_ysize() if options.ou < 1 : radius = nx//2-2 else: radius = options.ou mask = model_circle(radius, nx, ny, nz) else: mask = get_im(args[2]) for j in xrange(i+1, lend): lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0] order = tsp(lccc) if(len(order) != lend): print " problem with data length" from sys import exit exit() print "Total sum of cccs :",TotalDistance(order, lccc) print "ordering :",order for i in xrange(lend): get_im(stack, order[i]).write_image( new_stack, i ) elif options.align: nargs = len(args) if nargs != 3: print "must provide name of input and two output files!" return from utilities import get_params2D, model_circle from fundamentals import rot_shift2D from statistics import ccc from time import time from alignment import align2d, align2d_scf from multi_shc import mult_transform stack = args[0] new_stack = args[1] d = EMData.read_images(stack) """ # will align anyway try: ttt = d[0].get_attr('xform.params2d') for i in xrange(len(d)): alpha, sx, sy, mirror, scale = get_params2D(d[i]) d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror) except: pass """ nx = d[0].get_xsize() ny = d[0].get_ysize() if options.ou < 1 : radius = nx//2-2 else: radius = options.ou mask = model_circle(radius, nx, ny) if(options.xr < 0): xrng = 0 else: xrng = options.xr if(options.yr < 0): yrng = xrng else: yrng = options.yr initial = max(options.initial, 0) from statistics import mono lend = len(d) lccc = [None]*(lend*(lend-1)/2) from utilities import read_text_row if options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) : st = time() for i in xrange(lend-1): for j in xrange(i+1, lend): # j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger) #alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F") alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius) lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir] #print " %4d %10.1f"%(i,time()-st) if(not os.path.exists(options.pairwiseccc)): from utilities import write_text_row write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc) elif(os.path.exists(options.pairwiseccc)): lccc = read_text_row(options.pairwiseccc) initial = int(lccc[0][0] + 0.1) del lccc[0] for i in xrange(len(lccc)): T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)}) lccc[i] = [lccc[i][0],T] tdummy = Transform({"type":"2D"}) maxsum = -1.023 for m in xrange(0,lend):#initial, initial+1): indc = range( lend ) lsnake = [[m, tdummy, 0.0]] del indc[m] lsum = 0.0 while len(indc) > 1: maxcit = -111. for i in xrange(len(indc)): cuc = lccc[mono(indc[i], lsnake[-1][0])][0] if cuc > maxcit: maxcit = cuc qi = indc[i] # Here we need transformation from the current to the previous, # meaning indc[i] -> lsnake[-1][0] T = lccc[mono(indc[i], lsnake[-1][0])][1] # If direction is from larger to smaller index, the transformation has to be inverted if( indc[i] > lsnake[-1][0] ): T = T.inverse() lsnake.append([qi,T, maxcit]) lsum += maxcit del indc[indc.index(qi)] T = lccc[mono(indc[-1], lsnake[-1][0])][1] if( indc[-1] > lsnake[-1][0]): T = T.inverse() lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]]) print " initial image and lsum ",m,lsum #print lsnake if(lsum > maxsum): maxsum = lsum init = m snake = [lsnake[i] for i in xrange(lend)] print " Initial image selected : ",init,maxsum," ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc) #for q in snake: print q from copy import deepcopy trans=deepcopy([snake[i][1] for i in xrange(len(snake))]) print [snake[i][0] for i in xrange(len(snake))]
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Forms chains of 2D images based on their similarities. Functionality: 4. Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent). Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers. The ways to use the program: 4.1 Use option initial to specify which image will be used as an initial seed to form the chain. sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25 4.2 If options initial is omitted, the program will determine which image best serves as initial seed to form the chain sxprocess.py input_stack.hdf output_stack.hdf --radius=25 4.3 Use option circular to form a circular chain. sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25 4.4 New circular code based on pairwise alignments sxprocess.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt 4.5 Circular ordering based on pairwise alignments sxprocess.py vols.hdf chain.hdf mask.hdf --dd --radius=25 """ parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False) parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False) parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False) parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)") parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") # import params for 2D alignment parser.add_option("--ou", type="int", default=-1, help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)") parser.add_option("--xr", type="int", default=0, help="range for translation search in x direction, search is +/xr (0)") parser.add_option("--yr", type="int", default=0, help="range for translation search in y direction, search is +/yr (0)") #parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)") parser.add_option("--pairwiseccc", type="string", default= None, help="Input/output pairwise ccc file") (options, args) = parser.parse_args() global_def.BATCH = True if options.dd: nargs = len(args) if nargs != 3: print "must provide name of input and two output files!" return stack = args[0] new_stack = args[1] from utilities import model_circle from statistics import ccc from statistics import mono lend = EMUtil.get_image_count(stack) lccc = [None]*(lend*(lend-1)/2) for i in xrange(lend-1): v1 = get_im( stack, i ) if( i == 0 and nargs == 2): nx = v1.get_xsize() ny = v1.get_ysize() nz = v1.get_ysize() if options.ou < 1 : radius = nx//2-2 else: radius = options.ou mask = model_circle(radius, nx, ny, nz) else: mask = get_im(args[2]) for j in xrange(i+1, lend): lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0] order = tsp(lccc) if(len(order) != lend): print " problem with data length" from sys import exit exit() print "Total sum of cccs :",TotalDistance(order, lccc) print "ordering :",order for i in xrange(lend): get_im(stack, order[i]).write_image( new_stack, i ) elif options.align: nargs = len(args) if nargs != 3: print "must provide name of input and two output files!" return from utilities import get_params2D, model_circle from fundamentals import rot_shift2D from statistics import ccc from time import time from alignment import align2d, align2d_scf from multi_shc import mult_transform stack = args[0] new_stack = args[1] d = EMData.read_images(stack) """ # will align anyway try: ttt = d[0].get_attr('xform.params2d') for i in xrange(len(d)): alpha, sx, sy, mirror, scale = get_params2D(d[i]) d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror) except: pass """ nx = d[0].get_xsize() ny = d[0].get_ysize() if options.ou < 1 : radius = nx//2-2 else: radius = options.ou mask = model_circle(radius, nx, ny) if(options.xr < 0): xrng = 0 else: xrng = options.xr if(options.yr < 0): yrng = xrng else: yrng = options.yr initial = max(options.initial, 0) from statistics import mono lend = len(d) lccc = [None]*(lend*(lend-1)/2) from utilities import read_text_row if options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) : st = time() for i in xrange(lend-1): for j in xrange(i+1, lend): # j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger) #alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F") alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius) lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir] #print " %4d %10.1f"%(i,time()-st) if(not os.path.exists(options.pairwiseccc)): from utilities import write_text_row write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc) elif(os.path.exists(options.pairwiseccc)): lccc = read_text_row(options.pairwiseccc) initial = int(lccc[0][0] + 0.1) del lccc[0] for i in xrange(len(lccc)): T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)}) lccc[i] = [lccc[i][0],T] tdummy = Transform({"type":"2D"}) maxsum = -1.023 for m in xrange(0,lend):#initial, initial+1): indc = range( lend ) lsnake = [[m, tdummy, 0.0]] del indc[m] lsum = 0.0 while len(indc) > 1: maxcit = -111. for i in xrange(len(indc)): cuc = lccc[mono(indc[i], lsnake[-1][0])][0] if cuc > maxcit: maxcit = cuc qi = indc[i] # Here we need transformation from the current to the previous, # meaning indc[i] -> lsnake[-1][0] T = lccc[mono(indc[i], lsnake[-1][0])][1] # If direction is from larger to smaller index, the transformation has to be inverted if( indc[i] > lsnake[-1][0] ): T = T.inverse() lsnake.append([qi,T, maxcit]) lsum += maxcit del indc[indc.index(qi)] T = lccc[mono(indc[-1], lsnake[-1][0])][1] if( indc[-1] > lsnake[-1][0]): T = T.inverse() lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]]) print " initial image and lsum ",m,lsum #print lsnake if(lsum > maxsum): maxsum = lsum init = m snake = [lsnake[i] for i in xrange(lend)] print " Initial image selected : ",init,maxsum," ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc) #for q in snake: print q from copy import deepcopy trans=deepcopy([snake[i][1] for i in xrange(len(snake))]) print [snake[i][0] for i in xrange(len(snake))] """ for m in xrange(lend): prms = trans[m].get_params("2D") print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2]) """ for k in xrange(lend-2,0,-1): T = snake[k][1] for i in xrange(k+1, lend): trans[i] = T*trans[i] # To add - apply all transformations and do the overall centering. for m in xrange(lend): prms = trans[m].get_params("2D") #print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2]) #rot_shift2D(d[snake[m][0]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image(new_stack, m) rot_shift2D(d[snake[m][0]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(new_stack, m) order = tsp(lccc) if(len(order) != lend): print " problem with data length" from sys import exit exit() print TotalDistance(order, lccc) print order ibeg = order.index(init) order = [order[(i+ibeg)%lend] for i in xrange(lend)] print TotalDistance(order, lccc) print order snake = [tdummy] for i in xrange(1,lend): # Here we need transformation from the current to the previous, # meaning order[i] -> order[i-1]] T = lccc[mono(order[i], order[i-1])][1] # If direction is from larger to smaller index, the transformation has to be inverted if( order[i] > order[i-1] ): T = T.inverse() snake.append(T) assert(len(snake) == lend) from copy import deepcopy trans = deepcopy(snake) for k in xrange(lend-2,0,-1): T = snake[k] for i in xrange(k+1, lend): trans[i] = T*trans[i] # Try to smooth the angles - complicated, I am afraid one would have to use angles forward and backwards # and find their average?? # In addition, one would have to recenter them """ trms = [] for m in xrange(lend): prms = trans[m].get_params("2D") trms.append([prms["alpha"], prms["mirror"]]) for i in xrange(3): for m in xrange(lend): mb = (m-1)%lend me = (m+1)%lend # angles order mb,m,me # calculate predicted angles mb->m """ for m in xrange(lend): prms = trans[m].get_params("2D") #rot_shift2D(d[order[m]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image("metro.hdf", m) rot_shift2D(d[order[m]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(args[2], m) """ # This was an effort to get number of loops, inconclusive, to say the least from numpy import outer, zeros, float32, sqrt lend = len(d) cor = zeros(lend,float32) cor = outer(cor, cor) for i in xrange(lend): cor[i][i] = 1.0 for i in xrange(lend-1): for j in xrange(i+1, lend): cor[i,j] = lccc[mono(i,j)][0] cor[j,i] = cor[i,j] lmbd, eigvec = pca(cor) from utilities import write_text_file nvec=20 print [lmbd[j] for j in xrange(nvec)] print " G" mm = [-1]*lend for i in xrange(lend): # row mi = -1.0e23 for j in xrange(nvec): qt = eigvec[j][i] if(abs(qt)>mi): mi = abs(qt) mm[i] = j for j in xrange(nvec): qt = eigvec[j][i] print round(qt,3), # eigenvector print mm[i] print for j in xrange(nvec): qt = [] for i in xrange(lend): if(mm[i] == j): qt.append(i) if(len(qt)>0): write_text_file(qt,"loop%02d.txt"%j) """ """ print [lmbd[j] for j in xrange(nvec)] print " B" mm = [-1]*lend for i in xrange(lend): # row mi = -1.0e23 for j in xrange(nvec): qt = eigvec[j][i]/sqrt(lmbd[j]) if(abs(qt)>mi): mi = abs(qt) mm[i] = j for j in xrange(nvec): qt = eigvec[j][i]/sqrt(lmbd[j]) print round(qt,3), # eigenvector print mm[i] print """ """ lend=3 cor = zeros(lend,float32) cor = outer(cor, cor) cor[0][0] =136.77 cor[0][1] = 79.15 cor[0][2] = 37.13 cor[1][0] = 79.15 cor[2][0] = 37.13 cor[1][1] = 50.04 cor[1][2] = 21.65 cor[2][1] = 21.65 cor[2][2] = 13.26 lmbd, eigvec = pca(cor) print lmbd print eigvec for i in xrange(lend): # row for j in xrange(lend): print eigvec[j][i], # eigenvector print print " B" for i in xrange(lend): # row for j in xrange(lend): print eigvec[j][i]/sqrt(lmbd[j]), # eigenvector print print " G" for i in xrange(lend): # row for j in xrange(lend): print eigvec[j][i]*sqrt(lmbd[j]), # eigenvector print """ else: nargs = len(args) if nargs != 2: print "must provide name of input and output file!" return from utilities import get_params2D, model_circle from fundamentals import rot_shift2D from statistics import ccc from time import time from alignment import align2d from multi_shc import mult_transform stack = args[0] new_stack = args[1] d = EMData.read_images(stack) try: ttt = d[0].get_attr('xform.params2d') for i in xrange(len(d)): alpha, sx, sy, mirror, scale = get_params2D(d[i]) d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror) except: pass nx = d[0].get_xsize() ny = d[0].get_ysize() if options.radius < 1 : radius = nx//2-2 else: radius = options.radius mask = model_circle(radius, nx, ny) init = options.initial if init > -1 : print " initial image: %d" % init temp = d[init].copy() temp.write_image(new_stack, 0) del d[init] k = 1 lsum = 0.0 while len(d) > 1: maxcit = -111. for i in xrange(len(d)): cuc = ccc(d[i], temp, mask) if cuc > maxcit: maxcit = cuc qi = i # print k, maxcit lsum += maxcit temp = d[qi].copy() del d[qi] temp.write_image(new_stack, k) k += 1 print lsum d[0].write_image(new_stack, k) else: if options.circular : print "Using options.circular" # figure the "best circular" starting image maxsum = -1.023 for m in xrange(len(d)): indc = range(len(d) ) lsnake = [-1]*(len(d)+1) lsnake[0] = m lsnake[-1] = m del indc[m] temp = d[m].copy() lsum = 0.0 direction = +1 k = 1 while len(indc) > 1: maxcit = -111. for i in xrange(len(indc)): cuc = ccc(d[indc[i]], temp, mask) if cuc > maxcit: maxcit = cuc qi = indc[i] lsnake[k] = qi lsum += maxcit del indc[indc.index(qi)] direction = -direction for i in xrange( 1,len(d) ): if( direction > 0 ): if(lsnake[i] == -1): temp = d[lsnake[i-1]].copy() #print " forw ",lsnake[i-1] k = i break else: if(lsnake[len(d) - i] == -1): temp = d[lsnake[len(d) - i +1]].copy() #print " back ",lsnake[len(d) - i +1] k = len(d) - i break lsnake[lsnake.index(-1)] = indc[-1] #print " initial image and lsum ",m,lsum #print lsnake if(lsum > maxsum): maxsum = lsum init = m snake = [lsnake[i] for i in xrange(len(d))] print " Initial image selected : ",init,maxsum print lsnake for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m) else: # figure the "best" starting image maxsum = -1.023 for m in xrange(len(d)): indc = range(len(d) ) lsnake = [m] del indc[m] temp = d[m].copy() lsum = 0.0 while len(indc) > 1: maxcit = -111. for i in xrange(len(indc)): cuc = ccc(d[indc[i]], temp, mask) if cuc > maxcit: maxcit = cuc qi = indc[i] lsnake.append(qi) lsum += maxcit temp = d[qi].copy() del indc[indc.index(qi)] lsnake.append(indc[-1]) #print " initial image and lsum ",m,lsum #print lsnake if(lsum > maxsum): maxsum = lsum init = m snake = [lsnake[i] for i in xrange(len(d))] print " Initial image selected : ",init,maxsum print lsnake for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m)
def runcheck(classavgstack, recon, outdir, inangles=None, selectdoc=None, displayYN=False, projstack='proj.hdf', outangles='angles.txt', outstack='comp-proj-reproj.hdf', normstack='comp-proj-reproj-norm.hdf'): print # Check if inputs exist check(classavgstack) check(recon) # Create directory if it doesn't exist if not os.path.isdir(outdir): os.makedirs(outdir) # os.mkdir() can only operate one directory deep print("mkdir -p %s" % outdir) # Expand path for outputs projstack = os.path.join(outdir, projstack) outangles = os.path.join(outdir, outangles) outstack = os.path.join(outdir, outstack) normstack = os.path.join(outdir, normstack) # Get number of images nimg0 = EMUtil.get_image_count(classavgstack) #print("nimg0: %s" % nimg0) # In case class averages include discarded images, apply selection file if selectdoc: goodavgs, extension = os.path.splitext(classavgstack) newclasses = goodavgs + "_kept" + extension # e2proc2d appends to existing files, so rename existing output if os.path.exists(newclasses): renamefile = newclasses + '.bak' os.rename(newclasses, renamefile) print("mv %s %s" % (newclasses, renamefile)) cmd7="e2proc2d.py %s %s --list=%s" % (classavgstack, newclasses, selectdoc) print cmd7 os.system(cmd7) # Update class-averages classavgstack = newclasses # Import Euler angles if inangles: cmd6="sxheader.py %s --params=xform.projection --import=%s" % (classavgstack, inangles) print cmd6 header(classavgstack, 'xform.projection', fimport=inangles) cmd1="sxheader.py %s --params=xform.projection --export=%s" % (classavgstack, outangles) print cmd1 #os.system(cmd1) try: header(classavgstack, 'xform.projection', fexport=outangles) except RuntimeError: print("\nERROR!! No projection angles found in class-average stack header!\n") exit() cmd2="sxproject3d.py %s %s --angles=%s" % (recon, projstack, outangles) print cmd2 #os.system(cmd2) project3d(recon, stack=projstack, listagls=outangles) imgcounter = 0 # montage will have double the number of images as number of class-averages result=[] # Number of images may have changed nimg1 = EMUtil.get_image_count(classavgstack) for imgnum in xrange(nimg1): #print imgnum classimg = get_im(classavgstack, imgnum) ccc1 = classimg.get_attr_default('cross-corr', -1.0) prjimg = get_im(projstack,imgnum) ccc1 = prjimg.get_attr_default('cross-corr', -1.0) cccoeff = ccc(prjimg,classimg) #print imgnum, cccoeff classimg.set_attr_dict({'cross-corr':cccoeff}) prjimg.set_attr_dict({'cross-corr':cccoeff}) prjimg.write_image(outstack,imgcounter) imgcounter += 1 classimg.write_image(outstack, imgcounter) imgcounter += 1 result.append(cccoeff) result1 = sum(result) #print result1 nimg2 = EMUtil.get_image_count(outstack) meanccc = result1/nimg1 print("Mean CCC is %s" % meanccc) for imgnum in xrange(nimg2): if (imgnum % 2 ==0): prjimg = get_im(outstack,imgnum) meanccc1 = prjimg.get_attr_default('mean-cross-corr', -1.0) prjimg.set_attr_dict({'mean-cross-corr':meanccc}) write_header(outstack,prjimg,imgnum) if (imgnum % 100) == 0: print imgnum # e2proc2d appends to existing files, so delete existing output if os.path.exists(normstack): os.remove(normstack) print("rm %s" % normstack) cmd5="e2proc2d.py %s %s --process=normalize" % (outstack, normstack) print cmd5 os.system(cmd5) # Optionally pop up e2display if displayYN: cmd8 = "e2display.py %s" % normstack print cmd8 os.system(cmd8) print("Done!")
def __new__(cls,filename,application,force_plot=False,force_2d=False,old=None): file_type = Util.get_filename_ext(filename) em_file_type = EMUtil.get_image_ext_type(file_type) if not file_exists(filename): return None if force_plot and force_2d: # ok this sucks but it suffices for the time being print("Error, the force_plot and force_2d options are mutually exclusive") return None if force_plot: from .emplot2d import EMPlot2DWidget if isinstance(old,EMPlot2DWidget): widget = old else: widget = EMPlot2DWidget(application=application) widget.set_data_from_file(filename) return widget if em_file_type != IMAGE_UNKNOWN or filename[:4] == "bdb:": n = EMUtil.get_image_count(filename) nx,ny,nz = gimme_image_dimensions3D(filename) if n > 1 and nz == 1: if force_2d: a = EMData() data=a.read_images(filename) else: data = None # This is like a flag - the ImageMXWidget only needs the file name elif nz == 1: data = [EMData(filename,0)] else: data = EMData() data.read_image(filename,0,not force_2d) # This should be 3-D. We read the header-only here data = [data] if data != None and len(data) == 1: data = data[0] if force_2d or isinstance(data,EMData) and data.get_zsize()==1: if isinstance(data,list) or data.get_ysize() != 1: from .emimage2d import EMImage2DWidget if isinstance(old,EMImage2DWidget): widget = old else: widget= EMImage2DWidget(application=application) else: from .emplot2d import EMPlot2DWidget if isinstance(old,EMPlot2DWidget): widget = old else: widget = EMPlot2DWidget(application=application) widget.set_data_from_file(filename) return widget elif isinstance(data,EMData): if isinstance(old,EMScene3D): widget = old else: widget = EMScene3D() # print n,data for ii in range(n): data=EMData(filename,ii) datai = EMDataItem3D(data, transform=Transform()) widget.insertNewNode(os.path.basename(filename), datai, parentnode=widget) isosurface = EMIsosurface(datai, transform=Transform()) widget.insertNewNode("Iso", isosurface, parentnode=datai) return widget elif data == None or isinstance(data,list): from .emimagemx import EMImageMXWidget if isinstance(old,EMImageMXWidget): widget = old else: widget = EMImageMXWidget(application=application) data = filename else: print(filename) raise # weirdness, this should never happen widget.set_data(data,filename) return widget else: from .emplot2d import EMPlot2DWidget if isinstance(old,EMPlot2DWidget): widget = old else: widget = EMPlot2DWidget(application=application) widget.set_data_from_file(filename) return widget
def main(): arglist = [] for arg in sys.argv: arglist.append( arg ) progname = os.path.basename( arglist[0] ) usage = progname + " prj_stack volume [begin end step] --CTF --npad=ntimes_padding --list=file --group=ID --snr=SNR --sym=symmetry --verbose=(0|1) --xysize --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--CTF", action="store_true", default=False, help="apply CTF correction") parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio" ) parser.add_option("--sym", type="string", default="c1", help="symmetry" ) parser.add_option("--list", type="string", help="file with list of images to be used in the first column" ) parser.add_option("--group", type="int", default=-1, help="perform reconstruction using images for a given group number (group is attribute in the header)" ) parser.add_option("--MPI", action="store_true", default=False, help="use MPI version ") parser.add_option("--npad", type="int", default=2, help="number of times padding (default 2)" ) parser.add_option("--verbose", type="int", default=0, help="verbose level: 0 no verbose, 1 verbose" ) parser.add_option("--xysize", type="int", default=-1, help="user expected size at xy direction" ) parser.add_option("--zsize", type="int", default=-1, help="user expected size at z direction" ) parser.add_option("--smearstep", type="float", default=0.0, help="Rotational smear step (default 0.0, no smear)" ) parser.add_option("--interpolation_method", type ="string", default="4nn", help="4nn, or tril: nearest neighbor, or trillinear interpolation" ) parser.add_option("--niter", type="int", default=10, help="number of iterations for iterative reconstruction" ) parser.add_option("--upweighted", action="store_true", default=False, help="apply background noise") parser.add_option("--compensate", action="store_true", default=False, help="compensate in reconstruction") parser.add_option("--chunk_id", type="int", default=-1, help="reconstruct both odd and even groups of particles") parser.add_option("--target_window_size", type="int", default=-1, help=" size of the targeted reconstruction ") (options,args) = parser.parse_args(arglist[1:]) if options.MPI: from mpi import mpi_init sys.argv = mpi_init(len(sys.argv), sys.argv) if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() if len(args) == 2: prj_stack = args[0] vol_stack = args[1] nimage = EMUtil.get_image_count( prj_stack ) pid_list = range(0, nimage) elif len(args) == 5: prj_stack = args[0] vol_stack = args[1] begin = atoi( args[2] ) end = atoi( args[3] ) step = atoi( args[4] ) pid_list = range(begin, end, step) else: ERROR("incomplete list of arguments","recon3d_n",1) exit() if(options.list and options.group > -1): ERROR("options group and list cannot be used together","recon3d_n",1) sys.exit() from applications import recons3d_n, recons3d_trl_MPI global_def.BATCH = True if options.interpolation_method == "4nn": recons3d_n(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\ options.sym, options.list, options.group, options.verbose, options.MPI,options.xysize, options.zsize, options.smearstep, options.upweighted, options.compensate,options.chunk_id) elif options.interpolation_method == "tril": if options.MPI is False: ERROR(" trillinear interpolation reconstruction has MPI version only!") sys.exit() recons3d_trl_MPI(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\ options.sym, options.verbose, options.niter, options.compensate, options.target_window_size) else: ERROR(" Wrong interpolation method. The current options are 4nn, and tril. 4nn is the defalut one. ") global_def.BATCH = False if options.MPI: from mpi import mpi_finalize mpi_finalize()
def main(): global debug progname = os.path.basename(sys.argv[0]) usage = """prog [options] <path or db> ... Various utilities related to BDB databases. examples : e2bdb.py -c Is perhaps the most critical function, as it cleans up the database cache. See the Wiki for more. e2bdb.py <path> -s will list the contents of the database in a directory in bdb: notation e2bdb.py <path> -l Will give useful summary info about stacks in a directory e2bdb.py <database> --dump Gives a mechanism to dump all of the metadata in a database, even if the database contains no images """ parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_argument( "--cleanup", "-c", action="store_true", default=False, help= "This option will clean up the database cache so files can safely be moved or accessed on another computer via NFS." ) parser.add_argument( "--force", "-F", action="store_true", default=False, help= "This will force an action that would normally fail due to failed checks." ) parser.add_argument( "--delete", action="store_true", default=False, help="This will delete (or at least empty) the named database(s)") parser.add_argument("--all", "-a", action="store_true", help="List per-particle info", default=False) parser.add_argument("--long", "-l", action="store_true", help="Long listing", default=False) parser.add_argument("--short", "-s", action="store_true", help="Dense listing of names only", default=False) parser.add_argument( "--filt", type=str, help="Only include dictionary names containing the specified string", default=None) parser.add_argument( "--filtexclude", type=str, help="Exclude dictionary names containing the specified string", default=None) parser.add_argument( "--match", type=str, help= "Only include dictionaries matching the provided Python regular expression", default=None) parser.add_argument( "--exclude", type=str, help="The name of a database containing a list of exclusion keys", default=None) parser.add_argument( "--dump", "-D", action="store_true", help= "List contents of an entire database, eg 'e2bdb.py -D refine_01#register", default=False) parser.add_argument( "--smalldump", action="store_true", help= "Lists contents of an entire database, but only list 2 items per dictionary to better see headers", default=False) parser.add_argument( "--extractplots", action="store_true", help= "If a database contains sets of plots, such as bdb:refine_xx#convergence.results, this will extract the plots as text files." ) parser.add_argument( "--check", action="store_true", help= "Check for self-consistency and errors in the structure of specified databases", default=False) parser.add_argument("--nocache", action="store_true", help="Don't use the database cache for this operation", default=False) parser.add_argument( "--merge", action="store_true", help= "This will merge the contents of BDB 2-N into BDB 1 (including BDB 1's contents)", default=False) parser.add_argument( "--makevstack", type=str, help= "Creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks", default=None) parser.add_argument( "--appendvstack", type=str, help= "Appends to/creates a 'virtual' BDB stack with its own metadata, but the binary data taken from the (filtered) list of stacks", default=None) parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help= "verbose level [0-9], higher number means higher level of verboseness") parser.add_argument( "--list", type=str, help= "Specify the name of a file with a list of images to use in creation of virtual stacks. Please see source for details.", default=None) parser.add_argument( "--exlist", type=str, help= "Specify the name of a file with a list of images to exclude in creation of virtual stacks. Please see source for details.", default=None) parser.add_argument( "--restore", type=str, help= "Write changes in the derived virtual stack back to the original stack", default=None) parser.add_argument( "--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-1) parser.add_argument( "--checkctf", action="store_true", help= "Verfies that all images in the file contain CTF information, and gives some basic statistics", default=False) parser.add_argument( "--step", type=str, default="0,1", help= "Specify <init>,<step>[,<max>]. Processes only a subset of the input data. For example, 0,2 would process only the even numbered particles" ) (options, args) = parser.parse_args() if options.nocache: EMAN2db.BDB_CACHE_DISABLE = True if options.cleanup: db_cleanup(options.force) sys.exit(0) try: options.step = int(options.step.split(",")[0]), int( options.step.split(",")[1]), int( options.step.split(",")[2]) # convert strings to tuple except: try: options.step = int(options.step.split(",")[0]), int( options.step.split(",")[1]) except: print("Invalid --step specification") sys.exit(1) if options.all: options.long = 1 if len(args) == 0: args.append("bdb:.") logid = 0 if options.makevstack: logid = E2init(sys.argv) vstack = db_open_dict(options.makevstack) vstackn = 0 elif options.appendvstack: logid = E2init(sys.argv) vstack = db_open_dict(options.appendvstack) vstackn = len(vstack) else: vstack = None if options.merge: print("WARNING: Merge mode\nCombining contents of: ", ", ".join(args[1:])) print("into ", args[0]) if input("Proceed (y/n) :").lower() != "y": print("Aborting") sys.exit(1) for i, path in enumerate(args): if path.lower()[:4] == "bdb:" and not "#" in path: path = "bdb:.#" + path[4:] if path.lower()[:4] != "bdb:": path = "bdb:" + path if i == 0: outdb = db_open_dict(path) continue indb = db_open_dict(path, True) for k in list(indb.keys()): outdb[k] = indb[k] print("Merging complete") sys.exit(0) for path in args: if path.lower()[:4] == "bdb:" and not "#" in path: uu = os.path.split(path) if (uu[0] == ''): path = "bdb:.#" + path[4:] else: path = uu[0] + "#" + uu[1] if path.lower()[:4] != "bdb:": path = "bdb:" + path if '#' in path: if len(args) > 1: print("\n", path, ":") path, dbs = path.rsplit("#", 1) path += "#" dbs = [dbs] else: if not '#' in path and path[-1] != '/': path += '#' if len(args) > 1: print("\n", path[:-1], ":") dbs = db_list_dicts(path) dbs.sort() if options.filt: dbs = [db for db in dbs if options.filt in db] if options.filtexclude: dbs = [db for db in dbs if options.filtexclude not in db] if options.match != None: dbs = [db for db in dbs if re.match(options.match, db)] if options.list: if options.makevstack == None and options.appendvstack == None: print( "ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart" ) sys.exit(1) vdata = open(options.list, 'r').readlines() n = len(vdata[0].split()) slist = [] for line in vdata: line = line.split() for i in range(n): val = int(line[i]) slist.append(val) del n, val, vdata if options.exlist: if options.makevstack == None: print( "ERROR, this option is used for virtual stack creation, please add makevstack or appendvstack options, and restart" ) sys.exit(1) vdata = open(options.exlist, 'r').readlines() n = len(vdata[0].split()) slist = [] for line in vdata: line = line.split() for i in range(n): val = int(line[i]) slist.append(val) n = EMUtil.get_image_count(args[0]) good = set(range(n)) - set(slist) slist = [i for i in good] slist.sort() del n, val, vdata, good if options.makevstack != None or options.appendvstack != None: vspath = os.path.realpath(vstack.path) + "/" if options.verbose > 2: print("vspath: ", vspath) for db in dbs: dct, keys = db_open_dict(path + db, ro=True, with_keys=True) if dct == vstack: continue if len(options.step) == 2: if keys == None: vals = list( range(options.step[0], len(dct), options.step[1])) else: vals = keys[options.step[0]::options.step[ 1]] # we apply --step even if we have a list of keys else: if keys == None: vals = list( range(options.step[0], options.step[2], options.step[1])) else: vals = keys[options.step[0]:options.step[2]:options.step[ 1]] # we apply --step even if we have a list of keys if options.list != None or options.exlist != None: vals = slist for n in vals: try: d = dct.get(n, nodata=1).get_attr_dict() except: traceback.print_exc() print("---\nerror reading ", db, n) continue # This block converts an absolute path to the actual data to a relative path try: dpath = os.path.realpath(dct.get_data_path(n)) if options.verbose > 2: print("dpath: ", dpath) if os.name == 'nt': vspath = vspath.replace("\\", '/') dpath = dpath.replace('\\', '/') rpath = makerelpath(vspath, dpath) if options.verbose > 2: print("rpath: ", rpath) except: print("error with data_path ", db, n) continue d["data_path"] = rpath d["data_n"] = n d["data_source"] = path + db if d["data_path"] == None: print("error with data_path ", db, n) continue vstack[vstackn] = d vstackn += 1 if vstackn % 100 == 0: try: print("\r ", vstackn, " ", end=' ') sys.stdout.flush() except: pass print("\r ", vstackn, " ") dct.close() try: maxname = max([len(s) for s in dbs]) except: print("Error reading ", path) if options.restore: nima = EMUtil.get_image_count(options.restore) IB = db_open_dict(options.restore) source_old = None if len(options.step) == 3: nima = min(options.step[2], nima) for i in range(options.step[0], nima, options.step[1]): source = IB.get_header(i) source_path = source["source_path"] ID = source["source_n"] if (source_old != source_path): if (source_old != None): DB.close() DB = db_open_dict(source_path, ro=True) source_old = source_path target = DB.get_header(ID) try: source["data_path"] = target["data_path"] source["data_n"] = target["data_n"] source["source_path"] = target["source_path"] source["source_n"] = target["source_n"] except: # top level does not have data_path del source['data_path'] del source['data_n'] source["source_path"] = target["source_path"] source["source_n"] = target["source_n"] DB.set_header(ID, source) DB.close() if options.extractplots: for db in dbs: print("#### Extracting plots from ", db) dct = db_open_dict(path + db, ro=True) #### Dump keys = list(dct.keys()) keys.sort() for k in keys: v = dct[k] try: ns = [len(i) for i in v] fsp = db + "-" + k + ".txt" print("%s (%d columns)" % (fsp, len(ns))) out = open(fsp, "w") for i in range(ns[0]): for j in range(len(ns)): out.write(str(v[j][i])) if j < len(ns) - 1: out.write("\t") out.write("\n") out.close() except: continue dct.close() if options.smalldump: for db in dbs: print("##### ", db) dct = db_open_dict(path + db, ro=True) #### Dump keys = list(dct.keys()) keys.sort() if len(options.step) == 3: keys = keys[:options.step[2]] for k in keys[options.step[0]::options.step[1]]: v = dct[k] print("%s : " % k, end=' ') if isinstance(v, list) or isinstance(v, tuple): for i in v: print("\n\t%s" % str(i), end=' ') print("") elif isinstance(v, dict): ks2 = list(v.keys()) ks2.sort() kc = 0 for i in ks2: if kc >= 2: print("...") break print("\n\t%s : %s" % (i, v[i]), end=' ') kc += 1 print("") else: print(str(v)) dct.close() if options.checkctf: for db in dbs: print("##### CTF -> ", db) dct = db_open_dict(path + db, ro=True) keys = list(dct.keys()) if len(options.step) == 3: keys = keys[:options.step[2]] defocus = set() for k in keys[options.step[0]::options.step[1]]: v = dct.get_header(k) try: ctf = v["ctf"] except: if k != "maxrec": print("CTF missing on image %s" % k) continue defocus.add(ctf.defocus) defocus = list(defocus) print("Defocuses found: ", end=' ') for i in defocus: print("%1.3f, " % i, end=' ') print("\n\nRange: %1.3f - %1.3f (%d unique values)" % (min(defocus), max(defocus), len(defocus))) if options.dump: for db in dbs: print("##### ", db) dct = db_open_dict(path + db, ro=True) #### Dump keys = list(dct.keys()) if len(options.step) == 3: keys = keys[:options.step[2]] keys.sort() for k in keys[options.step[0]::options.step[1]]: v = dct[k] print("%s : " % k, end=' ') if isinstance(v, list) or isinstance(v, tuple): for i in v: print("\n\t%s" % str(i), end=' ') print("") elif isinstance(v, dict): ks2 = list(v.keys()) ks2.sort() for i in ks2: print("\n\t%s : %s" % (i, v[i]), end=' ') print("") else: print(str(v)) dct.close() # long listing, one db per line elif options.long: width = maxname + 3 fmt = "%%-%ds %%-07d %%14s %%s" % width fmt2 = "%%-%ds (not an image stack)" % width total = [0, 0] for db in dbs: dct = db_open_dict(path + db, True) ### Info on all particles if options.all: mx = len(dct) if len(options.step) == 3: mx = min(mx, options.step[2]) for i in range(options.step[0], mx, options.step[1]): try: im = dct[i] if im == None: raise Exception except: continue print( "%d. %d x %d x %d\tA/pix=%1.2f\tM=%1.4f\tS=%1.4f\tSk=%1.4f" % (i, im["nx"], im["ny"], im["nz"], im["apix_x"], im["mean"], im["sigma"], im["skewness"]), end=' ') try: print("\t%s" % str(im["model_id"])) except: pass try: print("\tdf=%1.3f\tB=%1.1f" % (im["ctf"].defocus, im["ctf"].bfactor)) except: print(" ") first = EMData() try: first.read_image(path + db, 0, True) size = first.get_xsize() * first.get_ysize( ) * first.get_zsize() * len(dct) * 4 total[0] += len(dct) total[1] += size print(fmt % (db, len(dct), "%dx%dx%d apix: %1.2f" % (first.get_xsize(), first.get_ysize(), first.get_zsize(), first["apix_x"]), human_size(size)), end=' ') except: print(fmt2 % db) try: print("\tdf: %1.3f\tB: %1.0f" % (first["ctf"].defocus, first["ctf"].bfactor)) except: print("") dct.close() print(fmt % ("TOTAL", total[0], "", human_size(total[1]))) elif options.check: from pickle import loads for db in dbs: dct = db_open_dict(path + db, ro=True) dct.realopen() keys = list(dct.bdb.keys()) allkvp = {} for k in keys: s1, s2 = k.split("\x80", 1) # start of a pickled string. s2 = loads("\x80" + s2) # the pickled part if len( s1 ) > 0: # If anything unpickled, then it is an axbxc prefix identifying the location of a binary st = allkvp.setdefault( s1, set()) # set of all positions seen so far v = loads(dct.bdb.get(k)) # position in binary file if v in st: print( "Error: value %d seen multiple times in %s (%s,%s)" % (v, db, s1, s2)) st.add(v) print("%s : " % db, end=' ') for i in list(allkvp.keys()): if options.verbose > 0: print("%s %d/%d\t" % (i, len(allkvp[i]), int(max(allkvp[i])) + 1), end=' ') if len(allkvp[i]) != int(max(allkvp[i]) + 1): print( "\nMismatch found in %s. Could be normal if file has been rewritten multiple times, but is unusual" % db) if options.verbose > 0: print("") else: print(" done") dct.close() elif options.short: for db in dbs: print(path + db, end=' ') print(" ") elif not options.makevstack and not options.appendvstack: # Nicely formatted 'ls' style display cols = int(floor(old_div(80.0, (maxname + 3)))) width = old_div(80, cols) rows = int(ceil(old_div(float(len(dbs)), cols))) fmt = "%%-%ds" % width for r in range(rows): for c in range(cols): try: print(fmt % dbs[r + c * rows], end=' ') except: pass print(" ") if options.delete: if not options.force: print("You are requesting to delete the following databases:") for db in dbs: print(db, " ", end=' ') if input("\nAre you sure (y/n) ? ")[0].lower() != 'y': print("Aborted") sys.exit(1) for db in dbs: db_remove_dict(path + db) if logid: E2end(logid)
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Generic 2-D image processing programs. Functionality: 1. Phase flip a stack of images and write output to new file: sxprocess.py input_stack.hdf output_stack.hdf --phase_flip 2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size. The window size will change accordingly. sxprocess input.hdf output.hdf --changesize --ratio=0.5 3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes. sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024] 4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs: sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64 5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3): sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid 6. Retrieve original image numbers of images listed in ISAC output stack of averages: sxprocess.py select1.hdf ohk.txt 7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file. Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack. sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw 8. Generate a 1D rotationally averaged power spectrum of an image. sxprocess.py vol.hdf --rotwp=rotpw.txt # Output will contain three columns: (1) rotationally averaged power spectrum (2) logarithm of the rotationally averaged power spectrum (3) integer line number (from zero to approximately to half the image size) 9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data. sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt The output file is then imported and 3D transformed volume computed: sxheader.py bdb:p --params=xform.projection --import=output.txt mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI The reconstructed volume is in the position of the volume computed using the input.txt parameters and then transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz) 10. Import ctf parameters from the output of sxcter into windowed particle headers. There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph. In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'. Normally this is done by e2boxer.py during windowing. Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created. sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 # Output will be a vritual stack bdb:Gdata sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 To concatenate output files: cd directory e2bdb.py . --makevstack=bdb:allparticles --filt=G IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them. 11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor sxprocess.py orientationparams.txt scaledparams.txt scale=0.5 12. Generate adaptive mask from a given 3-D volume. """ parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--order", action="store_true", help= "Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False) parser.add_option("--order_lookup", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_metropolis", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_pca", action="store_true", help="Test/Debug.", default=False) parser.add_option( "--initial", type="int", default=-1, help= "Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)" ) parser.add_option( "--circular", action="store_true", help= "Select circular ordering (fisr image has to be similar to the last", default=False) parser.add_option( "--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") parser.add_option( "--changesize", action="store_true", help= "resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False) parser.add_option( "--ratio", type="float", default=1.0, help= "The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round" ) parser.add_option( "--pw", action="store_true", help= "compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False) parser.add_option( "--wn", type="int", default=-1, help= "Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))" ) parser.add_option("--phase_flip", action="store_true", help="Phase flip the input stack", default=False) parser.add_option( "--makedb", metavar="param1=value1:param2=value2", type="string", action="append", help= "One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0" ) parser.add_option( "--generate_projections", metavar="param1=value1:param2=value2", type="string", action="append", help= "Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64." ) parser.add_option( "--isacgroup", type="int", help= "Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1) parser.add_option( "--isacselect", action="store_true", help= "Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False) parser.add_option( "--params", type="string", default=None, help="Name of header of parameter, which one depends on specific option" ) parser.add_option( "--adjpw", action="store_true", help="Adjust rotationally averaged power spectrum of an image", default=False) parser.add_option( "--rotpw", type="string", default=None, help= "Name of the text file to contain rotationally averaged power spectrum of the input image." ) parser.add_option( "--transformparams", type="string", default=None, help= "Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters." ) # import ctf estimates done using cter parser.add_option("--input", type="string", default=None, help="Input particles.") parser.add_option( "--importctf", type="string", default=None, help="Name of the file containing CTF parameters produced by sxcter.") parser.add_option( "--defocuserror", type="float", default=1000000.0, help= "Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%" ) parser.add_option( "--astigmatismerror", type="float", default=360.0, help= "Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees." ) # import ctf estimates done using cter parser.add_option( "--scale", type="float", default=-1.0, help= "Divide shifts in the input 3D orientation parameters text file by the scale factor." ) # generate adaptive mask from an given 3-Db volue parser.add_option("--adaptive_mask", action="store_true", help="create adavptive 3-D mask from a given volume", default=False) parser.add_option( "--nsigma", type="float", default=1., help= "number of times of sigma of the input volume to obtain the the large density cluster" ) parser.add_option( "--ndilation", type="int", default=3, help= "number of times of dilation applied to the largest cluster of density" ) parser.add_option( "--kernel_size", type="int", default=11, help="convolution kernel for smoothing the edge of the mask") parser.add_option( "--gauss_standard_dev", type="int", default=9, help="stanadard deviation value to generate Gaussian edge") (options, args) = parser.parse_args() global_def.BATCH = True if options.phase_flip: nargs = len(args) if nargs != 2: print "must provide name of input and output file!" return from EMAN2 import Processor instack = args[0] outstack = args[1] nima = EMUtil.get_image_count(instack) from filter import filt_ctf for i in xrange(nima): img = EMData() img.read_image(instack, i) try: ctf = img.get_attr('ctf') except: print "no ctf information in input stack! Exiting..." return dopad = True sign = 1 binary = 1 # phase flip assert img.get_ysize() > 1 dict = ctf.to_dict() dz = dict["defocus"] cs = dict["cs"] voltage = dict["voltage"] pixel_size = dict["apix"] b_factor = dict["bfactor"] ampcont = dict["ampcont"] dza = dict["dfdiff"] azz = dict["dfang"] if dopad and not img.is_complex(): ip = 1 else: ip = 0 params = { "filter_type": Processor.fourier_filter_types.CTF_, "defocus": dz, "Cs": cs, "voltage": voltage, "Pixel_size": pixel_size, "B_factor": b_factor, "amp_contrast": ampcont, "dopad": ip, "binary": binary, "sign": sign, "dza": dza, "azz": azz } tmp = Processor.EMFourierFilter(img, params) tmp.set_attr_dict({"ctf": ctf}) tmp.write_image(outstack, i) elif options.changesize: nargs = len(args) if nargs != 2: ERROR("must provide name of input and output file!", "change size", 1) return from utilities import get_im instack = args[0] outstack = args[1] sub_rate = float(options.ratio) nima = EMUtil.get_image_count(instack) from fundamentals import resample for i in xrange(nima): resample(get_im(instack, i), sub_rate).write_image(outstack, i) elif options.isacgroup > -1: nargs = len(args) if nargs != 3: ERROR("Three files needed on input!", "isacgroup", 1) return from utilities import get_im instack = args[0] m = get_im(args[1], int(options.isacgroup)).get_attr("members") l = [] for k in m: l.append(int(get_im(args[0], k).get_attr(options.params))) from utilities import write_text_file write_text_file(l, args[2]) elif options.isacselect: nargs = len(args) if nargs != 2: ERROR("Two files needed on input!", "isacgroup", 1) return from utilities import get_im nima = EMUtil.get_image_count(args[0]) m = [] for k in xrange(nima): m += get_im(args[0], k).get_attr("members") m.sort() from utilities import write_text_file write_text_file(m, args[1]) elif options.pw: nargs = len(args) if nargs < 2: ERROR("must provide name of input and output file!", "pw", 1) return from utilities import get_im d = get_im(args[0]) nx = d.get_xsize() ny = d.get_ysize() if nargs == 3: mask = get_im(args[2]) wn = int(options.wn) if wn == -1: wn = max(nx, ny) else: if ((wn < nx) or (wn < ny)): ERROR("window size cannot be smaller than the image size", "pw", 1) n = EMUtil.get_image_count(args[0]) from utilities import model_blank, model_circle, pad from EMAN2 import periodogram p = model_blank(wn, wn) for i in xrange(n): d = get_im(args[0], i) if nargs == 3: d *= mask st = Util.infomask(d, None, True) d -= st[0] p += periodogram(pad(d, wn, wn, 1, 0.)) p /= n p.write_image(args[1]) elif options.adjpw: if len(args) < 3: ERROR( "filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)", "adjpw", 1) return img_stack = args[0] from math import sqrt from fundamentals import rops_table, fft from utilities import read_text_file, get_im from filter import filt_tanl, filt_table if (args[1][-3:] == 'txt'): rops_dst = read_text_file(args[1]) else: rops_dst = rops_table(get_im(args[1])) out_stack = args[2] if (len(args) > 4): fl = float(args[3]) aa = float(args[4]) else: fl = -1.0 aa = 0.0 nimage = EMUtil.get_image_count(img_stack) for i in xrange(nimage): img = fft(get_im(img_stack, i)) rops_src = rops_table(img) assert len(rops_dst) == len(rops_src) table = [0.0] * len(rops_dst) for j in xrange(len(rops_dst)): table[j] = sqrt(rops_dst[j] / rops_src[j]) if (fl > 0.0): img = filt_tanl(img, fl, aa) img = fft(filt_table(img, table)) img.write_image(out_stack, i) elif options.rotpw != None: if len(args) != 1: ERROR("Only one input permitted", "rotpw", 1) return from utilities import write_text_file, get_im from fundamentals import rops_table from math import log10 t = rops_table(get_im(args[0])) x = range(len(t)) r = [0.0] * len(x) for i in x: r[i] = log10(t[i]) write_text_file([t, r, x], options.rotpw) elif options.transformparams != None: if len(args) != 2: ERROR( "Please provide names of input and output files with orientation parameters", "transformparams", 1) return from utilities import read_text_row, write_text_row transf = [0.0] * 6 spl = options.transformparams.split(',') for i in xrange(len(spl)): transf[i] = float(spl[i]) write_text_row(rotate_shift_params(read_text_row(args[0]), transf), args[1]) elif options.makedb != None: nargs = len(args) if nargs != 1: print "must provide exactly one argument denoting database key under which the input params will be stored" return dbkey = args[0] print "database key under which params will be stored: ", dbkey gbdb = js_open_dict("e2boxercache/gauss_box_DB.json") parmstr = 'dummy:' + options.makedb[0] (processorname, param_dict) = parsemodopt(parmstr) dbdict = {} for pkey in param_dict: if (pkey == 'invert_contrast') or (pkey == 'use_variance'): if param_dict[pkey] == 'True': dbdict[pkey] = True else: dbdict[pkey] = False else: dbdict[pkey] = param_dict[pkey] gbdb[dbkey] = dbdict elif options.generate_projections: nargs = len(args) if nargs != 3: ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\ "sxprocess - generate projections",1) return inpstr = args[0] outstk = args[1] micpref = args[2] parmstr = 'dummy:' + options.generate_projections[0] (processorname, param_dict) = parsemodopt(parmstr) parm_CTF = False parm_format = 'bdb' parm_apix = 2.5 if 'CTF' in param_dict: if param_dict['CTF'] == 'True': parm_CTF = True if 'format' in param_dict: parm_format = param_dict['format'] if 'apix' in param_dict: parm_apix = float(param_dict['apix']) boxsize = 64 if 'boxsize' in param_dict: boxsize = int(param_dict['boxsize']) print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize scale_mult = 2500 sigma_add = 1.5 sigma_proj = 30.0 sigma2_proj = 17.5 sigma_gauss = 0.3 sigma_mic = 30.0 sigma2_mic = 17.5 sigma_gauss_mic = 0.3 if 'scale_mult' in param_dict: scale_mult = float(param_dict['scale_mult']) if 'sigma_add' in param_dict: sigma_add = float(param_dict['sigma_add']) if 'sigma_proj' in param_dict: sigma_proj = float(param_dict['sigma_proj']) if 'sigma2_proj' in param_dict: sigma2_proj = float(param_dict['sigma2_proj']) if 'sigma_gauss' in param_dict: sigma_gauss = float(param_dict['sigma_gauss']) if 'sigma_mic' in param_dict: sigma_mic = float(param_dict['sigma_mic']) if 'sigma2_mic' in param_dict: sigma2_mic = float(param_dict['sigma2_mic']) if 'sigma_gauss_mic' in param_dict: sigma_gauss_mic = float(param_dict['sigma_gauss_mic']) from filter import filt_gaussl, filt_ctf from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank, pad, model_gauss_noise, set_params2D, set_params_proj from projection import prep_vol, prgs seed(14567) delta = 29 angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S") nangle = len(angles) modelvol = [] nvlms = EMUtil.get_image_count(inpstr) from utilities import get_im for k in xrange(nvlms): modelvol.append(get_im(inpstr, k)) nx = modelvol[0].get_xsize() if nx != boxsize: ERROR("Requested box dimension does not match dimension of the input model.", \ "sxprocess - generate projections",1) nvol = 10 volfts = [[] for k in xrange(nvlms)] for k in xrange(nvlms): for i in xrange(nvol): sigma = sigma_add + random() # 1.5-2.5 addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40) scale = scale_mult * (0.5 + random()) vf, kb = prep_vol(modelvol[k] + scale * addon) volfts[k].append(vf) del vf, modelvol if parm_format == "bdb": stack_data = "bdb:" + outstk delete_bdb(stack_data) else: stack_data = outstk + ".hdf" Cs = 2.0 pixel = parm_apix voltage = 120.0 ampcont = 10.0 ibd = 4096 / 2 - boxsize iprj = 0 width = 240 xstart = 8 + boxsize / 2 ystart = 8 + boxsize / 2 rowlen = 17 from random import randint params = [] for idef in xrange(3, 8): irow = 0 icol = 0 mic = model_blank(4096, 4096) defocus = idef * 0.5 #0.2 if parm_CTF: astampl = defocus * 0.15 astangl = 50.0 ctf = generate_ctf([ defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl ]) for i in xrange(nangle): for k in xrange(12): dphi = 8.0 * (random() - 0.5) dtht = 8.0 * (random() - 0.5) psi = 360.0 * random() phi = angles[i][0] + dphi tht = angles[i][1] + dtht s2x = 4.0 * (random() - 0.5) s2y = 4.0 * (random() - 0.5) params.append([phi, tht, psi, s2x, s2y]) ivol = iprj % nvol #imgsrc = randint(0,nvlms-1) imgsrc = iprj % nvlms proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y]) x = xstart + irow * width y = ystart + icol * width mic += pad(proj, 4096, 4096, 1, 0.0, x - 2048, y - 2048, 0) proj = proj + model_gauss_noise(sigma_proj, nx, nx) if parm_CTF: proj = filt_ctf(proj, ctf) proj.set_attr_dict({"ctf": ctf, "ctf_applied": 0}) proj = proj + filt_gaussl( model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss) proj.set_attr("origimgsrc", imgsrc) proj.set_attr("test_id", iprj) # flags describing the status of the image (1 = true, 0 = false) set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0]) set_params_proj(proj, [phi, tht, psi, s2x, s2y]) proj.write_image(stack_data, iprj) icol += 1 if icol == rowlen: icol = 0 irow += 1 iprj += 1 mic += model_gauss_noise(sigma_mic, 4096, 4096) if parm_CTF: #apply CTF mic = filt_ctf(mic, ctf) mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic) mic.write_image(micpref + "%1d.hdf" % (idef - 3), 0) drop_spider_doc("params.txt", params) elif options.importctf != None: print ' IMPORTCTF ' from utilities import read_text_row, write_text_row from random import randint import subprocess grpfile = 'groupid%04d' % randint(1000, 9999) ctfpfile = 'ctfpfile%04d' % randint(1000, 9999) cterr = [options.defocuserror / 100.0, options.astigmatismerror] ctfs = read_text_row(options.importctf) for kk in xrange(len(ctfs)): root, name = os.path.split(ctfs[kk][-1]) ctfs[kk][-1] = name[:-4] if (options.input[:4] != 'bdb:'): ERROR('Sorry, only bdb files implemented', 'importctf', 1) d = options.input[4:] #try: str = d.index('*') #except: str = -1 from string import split import glob uu = os.path.split(d) uu = os.path.join(uu[0], 'EMAN2DB', uu[1] + '.bdb') flist = glob.glob(uu) for i in xrange(len(flist)): root, name = os.path.split(flist[i]) root = root[:-7] name = name[:-4] fil = 'bdb:' + os.path.join(root, name) sourcemic = EMUtil.get_all_attributes(fil, 'ptcl_source_image') nn = len(sourcemic) gctfp = [] groupid = [] for kk in xrange(nn): junk, name2 = os.path.split(sourcemic[kk]) name2 = name2[:-4] ctfp = [-1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] for ll in xrange(len(ctfs)): if (name2 == ctfs[ll][-1]): # found correct if (ctfs[ll][8] / ctfs[ll][0] <= cterr[0]): # acceptable defocus error ctfp = ctfs[ll][:8] if (ctfs[ll][10] > cterr[1]): # error of astigmatism exceed the threshold, set astigmatism to zero. ctfp[6] = 0.0 ctfp[7] = 0.0 gctfp.append(ctfp) groupid.append(kk) break if (len(groupid) > 0): write_text_row(groupid, grpfile) write_text_row(gctfp, ctfpfile) cmd = "{} {} {} {}".format( 'e2bdb.py', fil, '--makevstack=bdb:' + root + 'G' + name, '--list=' + grpfile) #print cmd subprocess.call(cmd, shell=True) cmd = "{} {} {} {}".format('sxheader.py', 'bdb:' + root + 'G' + name, '--params=ctf', '--import=' + ctfpfile) #print cmd subprocess.call(cmd, shell=True) else: print ' >>> Group ', name, ' skipped.' cmd = "{} {} {}".format("rm -f", grpfile, ctfpfile) subprocess.call(cmd, shell=True) elif options.scale > 0.0: from utilities import read_text_row, write_text_row scale = options.scale nargs = len(args) if nargs != 2: print "Please provide names of input and output file!" return p = read_text_row(args[0]) for i in xrange(len(p)): p[i][3] /= scale p[i][4] /= scale write_text_row(p, args[1]) elif options.adaptive_mask: from utilities import get_im from morphology import adaptive_mask nsigma = options.nsigma ndilation = options.ndilation kernel_size = options.kernel_size gauss_standard_dev = options.gauss_standard_dev nargs = len(args) if nargs > 2: print "Too many inputs are given, try again!" return else: inputvol = get_im(args[0]) input_path, input_file_name = os.path.split(args[0]) input_file_name_root, ext = os.path.splitext(input_file_name) if nargs == 2: mask_file_name = args[1] else: mask_file_name = "adaptive_mask_for" + input_file_name_root + ".hdf" # Only hdf file is output. mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev) mask3d.write_image(mask_file_name) else: ERROR("Please provide option name", "sxprocess.py", 1)
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " prj_stack volume [begin end step] --CTF --npad=ntimes_padding --list=file --group=ID --snr=SNR --sym=symmetry --verbose=(0|1) --xysize --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--CTF", action="store_true", default=False, help="apply CTF correction") parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio") parser.add_option("--sym", type="string", default="c1", help="symmetry") parser.add_option( "--list", type="string", help="file with list of images to be used in the first column") parser.add_option( "--group", type="int", default=-1, help= "perform reconstruction using images for a given group number (group is attribute in the header)" ) parser.add_option("--MPI", action="store_true", default=False, help="use MPI version ") parser.add_option("--npad", type="int", default=2, help="number of times padding (default 2)") parser.add_option("--verbose", type="int", default=0, help="verbose level: 0 no verbose, 1 verbose") parser.add_option("--xysize", type="int", default=-1, help="user expected size at xy direction") parser.add_option("--zsize", type="int", default=-1, help="user expected size at z direction") parser.add_option("--smearstep", type="float", default=0.0, help="Rotational smear step (default 0.0, no smear)") parser.add_option( "--interpolation_method", type="string", default="4nn", help="4nn, or tril: nearest neighbor, or trillinear interpolation") parser.add_option("--niter", type="int", default=10, help="number of iterations for iterative reconstruction") parser.add_option("--upweighted", action="store_true", default=False, help="apply background noise") parser.add_option("--compensate", action="store_true", default=False, help="compensate in reconstruction") parser.add_option("--chunk_id", type="int", default=-1, help="reconstruct both odd and even groups of particles") parser.add_option("--target_window_size", type="int", default=-1, help=" size of the targeted reconstruction ") (options, args) = parser.parse_args(arglist[1:]) if options.MPI: from mpi import mpi_init sys.argv = mpi_init(len(sys.argv), sys.argv) if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() if len(args) == 2: prj_stack = args[0] vol_stack = args[1] nimage = EMUtil.get_image_count(prj_stack) pid_list = list(range(0, nimage)) elif len(args) == 5: prj_stack = args[0] vol_stack = args[1] begin = atoi(args[2]) end = atoi(args[3]) step = atoi(args[4]) pid_list = list(range(begin, end, step)) else: ERROR("incomplete list of arguments", "recon3d_n", 1) exit() if (options.list and options.group > -1): ERROR("options group and list cannot be used together", "recon3d_n", 1) sys.exit() from applications import recons3d_n, recons3d_trl_MPI global_def.BATCH = True if options.interpolation_method == "4nn": recons3d_n(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\ options.sym, options.list, options.group, options.verbose, options.MPI,options.xysize, options.zsize, options.smearstep, options.upweighted, options.compensate,options.chunk_id) elif options.interpolation_method == "tril": if options.MPI is False: ERROR( " trillinear interpolation reconstruction has MPI version only!" ) sys.exit() recons3d_trl_MPI(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\ options.sym, options.verbose, options.niter, options.compensate, options.target_window_size) else: ERROR( " Wrong interpolation method. The current options are 4nn, and tril. 4nn is the defalut one. " ) global_def.BATCH = False if options.MPI: from mpi import mpi_finalize mpi_finalize()
def __init__(self,options): self.options = options self.using_cuda = EMUtil.cuda_available() and options.cuda self.nsoln = options.nsoln
def main(): import os import sys from optparse import OptionParser from global_def import SPARXVERSION import global_def arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage2 = progname + """ inputfile outputfile [options] Functionalities: 1. Helicise input volume and save the result to output volume: sxhelicon_utils.py input_vol.hdf output_vol.hdf --helicise --dp=27.6 --dphi=166.5 --fract=0.65 --rmax=70 --rmin=1 --apix=1.84 --sym=D1 2. Helicise pdb file and save the result to a new pdb file: sxhelicon_utils.py input.pdb output.pdb --helicisepdb --dp=27.6 --dphi=166.5 --nrepeats --apix=1.84 3. Generate two lists of image indices used to split segment stack into halves for helical fsc calculation. sxhelicon_utils.py bdb:big_stack --hfsc='flst' --filament_attr=filament 4. Map of filament distribution in the stack sxhelicon_utils.py bdb:big_stack --filinfo=info.txt The output file will contain four columns: 1 2 3 4 first image number last image number number of images in the filament name 5. Predict segments' orientation parameters based on distances between segments and known helical symmetry sxhelicon_utils.py bdb:big_stack --predict_helical=helical_params.txt --dp=27.6 --dphi=166.5 --apix=1.84 6. Generate disks from filament based reconstructions: sxheader.py stk.hdf --params=xform.projection --import=params.txt mpirun -np 2 sxhelicon_utils.py stk.hdf --gendisk='bdb:disk' --ref_nx=100 --ref_ny=100 --ref_nz=200 --apix=1.84 --dp=27.6 --dphi=166.715 --fract=0.67 --rmin=0 --rmax=64 --function="[.,nofunc,helical3c]" --sym="c1" --MPI 7. Stack disks based on helical symmetry parameters sxhelicon_utils.py disk_to_stack.hdf --stackdisk=stacked_disks.hdf --dphi=166.5 --dp=27.6 --ref_nx=160 --ref_ny=160 --ref_nz=225 --apix=1.84 8. Helical symmetry search: mpirun -np 3 sxhelicon_utils.py volf0010.hdf outsymsearch --symsearch --dp=27.6 --dphi=166.715 --apix=1.84 --fract=0.65 --rmin=0 --rmax=92.0 --datasym=datasym.txt --dp_step=0.92 --ndp=3 --dphi_step=1.0 --ndphi=10 --MPI """ parser = OptionParser(usage2, version=SPARXVERSION) #parser.add_option("--ir", type="float", default= -1, help="inner radius for rotational correlation > 0 (set to 1) (Angstroms)") parser.add_option( "--ou", type="float", default=-1, help= "outer radius for rotational 2D correlation < int(nx/2)-1 (set to the radius of the particle) (Angstroms)" ) parser.add_option( "--rs", type="int", default=1, help="step between rings in rotational correlation >0 (set to 1)") parser.add_option( "--xr", type="string", default="4 2 1 1 1", help= "range for translation search in x direction, search is +/-xr (Angstroms) " ) parser.add_option( "--txs", type="string", default="1 1 1 0.5 0.25", help= "step size of the translation search in x directions, search is -xr, -xr+ts, 0, xr-ts, xr (Angstroms)" ) parser.add_option("--delta", type="string", default="10 6 4 3 2", help="angular step of reference projections") parser.add_option("--an", type="string", default="-1", help="angular neighborhood for local searches") parser.add_option( "--maxit", type="int", default=30, help= "maximum number of iterations performed for each angular step (set to 30) " ) parser.add_option("--CTF", action="store_true", default=False, help="CTF correction") parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio of the data") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version") #parser.add_option("--fourvar", action="store_true", default=False, help="compute Fourier variance") parser.add_option("--apix", type="float", default=-1.0, help="pixel size in Angstroms") parser.add_option("--dp", type="float", default=-1.0, help="delta z - translation in Angstroms") parser.add_option("--dphi", type="float", default=-1.0, help="delta phi - rotation in degrees") parser.add_option("--rmin", type="float", default=0.0, help="minimal radius for hsearch (Angstroms)") parser.add_option("--rmax", type="float", default=80.0, help="maximal radius for hsearch (Angstroms)") parser.add_option("--fract", type="float", default=0.7, help="fraction of the volume used for helical search") parser.add_option("--sym", type="string", default="c1", help="symmetry of the structure") parser.add_option("--function", type="string", default="helical", help="name of the reference preparation function") parser.add_option("--npad", type="int", default=2, help="padding size for 3D reconstruction") parser.add_option("--debug", action="store_true", default=False, help="debug") parser.add_option("--volalixshift", action="store_true", default=False, help="Use volalixshift refinement") parser.add_option( "--searchxshift", type="float", default=0.0, help= "search range for x-shift determination: +/- searchxshift (Angstroms)") parser.add_option( "--nearby", type="float", default=6.0, help= "neighborhood within which to search for peaks in 1D ccf for x-shift search (Angstroms)" ) # filinfo parser.add_option( "--filinfo", type="string", default="", help= "Store in an output text file infomration about distribution of filaments in the stack." ) # diskali parser.add_option("--diskali", action="store_true", default=False, help="volume alignment") parser.add_option( "--zstep", type="float", default=1, help="Step size for translational search along z (Angstroms)") # helicise parser.add_option( "--helicise", action="store_true", default=False, help="helicise input volume and save results to output volume") parser.add_option( "--hfsc", type="string", default="", help= "Generate two lists of image indices used to split segment stack into halves for helical fsc calculation. The lists will be stored in two text files named using file_prefix with '_even' and '_odd' suffixes, respectively." ) parser.add_option( "--filament_attr", type="string", default="filament", help="attribute under which filament identification is stored") parser.add_option( "--predict_helical", type="string", default="", help="Generate projection parameters consistent with helical symmetry") # helicise pdb parser.add_option( "--helicisepdb", action="store_true", default=False, help="Helicise pdb file and save the result to a new pdb file") parser.add_option( "--nrepeats", type="int", default=50, help= "Number of time the helical symmetry will be applied to the input file" ) # input options for generating disks parser.add_option( "--gendisk", type="string", default="", help="Name of file under which generated disks will be saved to") parser.add_option("--ref_nx", type="int", default=-1, help="nx=ny volume size") parser.add_option( "--ref_nz", type="int", default=-1, help="nz volume size - computed disks will be nx x ny x rise/apix") parser.add_option( "--new_pixel_size", type="float", default=-1, help= "desired pixel size of the output disks. The default is -1, in which case there is no resampling (unless --match_pixel_rise flag is True)." ) parser.add_option( "--maxerror", type="float", default=0.1, help= "proportional to the maximum amount of error to tolerate between (dp/new_pixel_size) and int(dp/new_pixel_size ), where new_pixel_size is the pixel size calculated when the option --match_pixel_rise flag is True." ) parser.add_option( "--match_pixel_rise", action="store_true", default=False, help= "calculate new pixel size such that the rise is approximately integer number of pixels given the new pixel size. This will be the pixel size of the output disks." ) # get consistency parser.add_option( "--consistency", type="string", default="", help="Name of parameters to get consistency statistics for") parser.add_option("--phithr", type="float", default=2.0, help="phi threshold for consistency check") parser.add_option("--ythr", type="float", default=2.0, help="y threshold (in Angstroms) for consistency check") parser.add_option( "--segthr", type="int", default=3, help="minimum number of segments/filament for consistency check") # stack disks parser.add_option( "--stackdisk", type="string", default="", help="Name of file under which output volume will be saved to.") parser.add_option("--ref_ny", type="int", default=-1, help="ny of output volume size. Default is ref_nx") # symmetry search parser.add_option("--symsearch", action="store_true", default=False, help="Do helical symmetry search.") parser.add_option( "--ndp", type="int", default=12, help= "In symmetrization search, number of delta z steps equals to 2*ndp+1") parser.add_option( "--ndphi", type="int", default=12, help= "In symmetrization search, number of dphi steps equals to 2*ndphi+1") parser.add_option( "--dp_step", type="float", default=0.1, help="delta z step for symmetrization [Angstroms] (default 0.1)") parser.add_option( "--dphi_step", type="float", default=0.1, help="dphi step for symmetrization [degrees] (default 0.1)") parser.add_option("--datasym", type="string", default="datasym.txt", help="symdoc") parser.add_option( "--symdoc", type="string", default="", help="text file containing helical symmetry parameters dp and dphi") # filament statistics in the stack (options, args) = parser.parse_args(arglist[1:]) if len(args) < 1 or len(args) > 5: print("Various helical reconstruction related functionalities: " + usage2) print("Please run '" + progname + " -h' for detailed options") else: if len(options.hfsc) > 0: if len(args) != 1: print("Incorrect number of parameters") sys.exit() from applications import imgstat_hfsc imgstat_hfsc(args[0], options.hfsc, options.filament_attr) sys.exit() elif len(options.filinfo) > 0: if len(args) != 1: print("Incorrect number of parameters") sys.exit() from EMAN2 import EMUtil filams = EMUtil.get_all_attributes(args[0], "filament") ibeg = 0 filcur = filams[0] n = len(filams) inf = [] i = 1 while (i <= n): if (i < n): fis = filams[i] else: fis = "" if (fis != filcur): iend = i - 1 inf.append([ibeg, iend, iend - ibeg + 1, filcur]) ibeg = i filcur = fis i += 1 from utilities import write_text_row write_text_row(inf, options.filinfo) sys.exit() if len(options.stackdisk) > 0: if len(args) != 1: print("Incorrect number of parameters") sys.exit() dpp = (float(options.dp) / options.apix) rise = int(dpp) if (abs(float(rise) - dpp) > 1.0e-3): print(" dpp has to be integer multiplicity of the pixel size") sys.exit() from utilities import get_im v = get_im(args[0]) from applications import stack_disks ref_ny = options.ref_ny if ref_ny < 0: ref_ny = options.ref_nx sv = stack_disks(v, options.ref_nx, ref_ny, options.ref_nz, options.dphi, rise) sv.write_image(options.stackdisk) sys.exit() if len(options.consistency) > 0: if len(args) != 1: print("Incorrect number of parameters") sys.exit() from development import consistency_params consistency_params(args[0], options.consistency, options.dphi, options.dp, options.apix, phithr=options.phithr, ythr=options.ythr, THR=options.segthr) sys.exit() rminp = int((float(options.rmin) / options.apix) + 0.5) rmaxp = int((float(options.rmax) / options.apix) + 0.5) from utilities import get_input_from_string, get_im xr = get_input_from_string(options.xr) txs = get_input_from_string(options.txs) irp = 1 if options.ou < 0: oup = -1 else: oup = int((options.ou / options.apix) + 0.5) xrp = '' txsp = '' for i in xrange(len(xr)): xrp += " " + str(float(xr[i]) / options.apix) for i in xrange(len(txs)): txsp += " " + str(float(txs[i]) / options.apix) searchxshiftp = int((options.searchxshift / options.apix) + 0.5) nearbyp = int((options.nearby / options.apix) + 0.5) zstepp = int((options.zstep / options.apix) + 0.5) if options.MPI: from mpi import mpi_init, mpi_finalize sys.argv = mpi_init(len(sys.argv), sys.argv) if len(options.predict_helical) > 0: if len(args) != 1: print("Incorrect number of parameters") sys.exit() if options.dp < 0: print( "Helical symmetry paramter rise --dp should not be negative" ) sys.exit() from applications import predict_helical_params predict_helical_params(args[0], options.dp, options.dphi, options.apix, options.predict_helical) sys.exit() if options.helicise: if len(args) != 2: print("Incorrect number of parameters") sys.exit() if options.dp < 0: print( "Helical symmetry paramter rise --dp should not be negative" ) sys.exit() from utilities import get_im, sym_vol vol = get_im(args[0]) vol = sym_vol(vol, options.sym) hvol = vol.helicise(options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp) hvol = sym_vol(hvol, options.sym) hvol.write_image(args[1]) sys.exit() if options.helicisepdb: if len(args) != 2: print("Incorrect number of parameters") sys.exit() if options.dp < 0: print( "Helical symmetry paramter rise --dp should not be negative" ) sys.exit() from math import cos, sin, radians from copy import deepcopy import numpy from numpy import zeros, dot, float32 dp = options.dp dphi = options.dphi nperiod = options.nrepeats infile = open(args[0], "r") pall = infile.readlines() infile.close() p = [] pos = [] lkl = -1 for i in xrange(len(pall)): if ((pall[i])[:4] == 'ATOM'): if (lkl == -1): lkl = i p.append(pall[i]) pos.append(i) n = len(p) X = zeros((3, len(p)), dtype=float32) X_new = zeros((3, len(p)), dtype=float32) for i in xrange(len(p)): element = deepcopy(p[i]) X[0, i] = float(element[30:38]) X[1, i] = float(element[38:46]) X[2, i] = float(element[46:54]) pnew = [] for j in xrange(-nperiod, nperiod + 1): for i in xrange(n): pnew.append(deepcopy(p[i])) dphi = radians(dphi) m = zeros((3, 3), dtype=float32) t = zeros((3, 1), dtype=float32) m[2][2] = 1.0 t[0, 0] = 0.0 t[1, 0] = 0.0 for j in xrange(-nperiod, nperiod + 1): if j != 0: rd = j * dphi m[0][0] = cos(rd) m[0][1] = sin(rd) m[1][0] = -m[0][1] m[1][1] = m[0][0] t[2, 0] = j * dp X_new = dot(m, X) + t for i in xrange(n): pnew[j * n + i] = pnew[j * n + i][:30] + "%8.3f" % (float( X_new[0, i])) + "%8.3f" % (float( X_new[1, i])) + "%8.3f" % (float( X_new[2, i])) + pnew[j * n + i][54:] outfile = open(args[1], "w") outfile.writelines(pall[0:lkl]) outfile.writelines(pnew) outfile.writelines("END\n") outfile.close() sys.exit() if options.volalixshift: if options.maxit > 1: print( "Inner iteration for x-shift determinatin is restricted to 1" ) sys.exit() if len(args) < 4: mask = None else: mask = args[3] from applications import volalixshift_MPI global_def.BATCH = True volalixshift_MPI(args[0], args[1], args[2], searchxshiftp, options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp, mask, options.maxit, options.CTF, options.snr, options.sym, options.function, options.npad, options.debug, nearbyp) global_def.BATCH = False if options.diskali: #if options.maxit > 1: # print "Inner iteration for disk alignment is restricted to 1" # sys.exit() if len(args) < 4: mask = None else: mask = args[3] global_def.BATCH = True if (options.sym[:1] == "d" or options.sym[:1] == "D"): from development import diskaliD_MPI diskaliD_MPI(args[0], args[1], args[2], mask, options.dp, options.dphi, options.apix, options.function, zstepp, options.fract, rmaxp, rminp, options.CTF, options.maxit, options.sym) else: from applications import diskali_MPI diskali_MPI(args[0], args[1], args[2], mask, options.dp, options.dphi, options.apix, options.function, zstepp, options.fract, rmaxp, rminp, options.CTF, options.maxit, options.sym) global_def.BATCH = False if options.symsearch: if len(options.symdoc) < 1: if options.dp < 0 or options.dphi < 0: print( "Enter helical symmetry parameters either using --symdoc or --dp and --dphi" ) sys.exit() if options.dp < 0 or options.dphi < 0: # read helical symmetry parameters from symdoc from utilities import read_text_row hparams = read_text_row(options.symdoc) dp = hparams[0][0] dphi = hparams[0][1] else: dp = options.dp dphi = options.dphi from applications import symsearch_MPI if len(args) < 3: mask = None else: mask = args[2] global_def.BATCH = True symsearch_MPI(args[0], args[1], mask, dp, options.ndp, options.dp_step, dphi, options.ndphi, options.dphi_step, rminp, rmaxp, options.fract, options.sym, options.function, options.datasym, options.apix, options.debug) global_def.BATCH = False elif len(options.gendisk) > 0: from applications import gendisks_MPI global_def.BATCH = True if len(args) == 1: mask3d = None else: mask3d = args[1] if options.dp < 0: print( "Helical symmetry paramter rise --dp must be explictly set!" ) sys.exit() gendisks_MPI(args[0], mask3d, options.ref_nx, options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp, options.CTF, options.function, options.sym, options.gendisk, options.maxerror, options.new_pixel_size, options.match_pixel_rise) global_def.BATCH = False if options.MPI: from mpi import mpi_finalize mpi_finalize()
def __init__(self, options): self.options = options self.using_cuda = EMUtil.cuda_available() and options.cuda self.nsoln = options.nsoln