def shrink_or_enlarge_or_stay_the_same(images, shrink_ratio, target_nx, target_radius, newx, nima): from fundamentals import resample from utilities import pad if newx >= target_nx: msk = model_circle(target_radius, target_nx, target_nx) else: msk = model_circle(newx // 2 - 2, newx, newx) do_not_stay_the_same_size = shrink_ratio != 1.0 cut_to_window = newx > target_nx need_padding = newx < target_nx for im in xrange(nima): if do_not_stay_the_same_size: images[im] = resample(images[im], shrink_ratio) if cut_to_window: images[im] = Util.window(images[im], target_nx, target_nx, 1) p = Util.infomask(images[im], msk, False) images[im] -= p[0] p = Util.infomask(images[im], msk, True) images[im] /= p[1] if need_padding: images[im] = pad(images[im], target_nx, target_nx, 1, 0.0)
def shrink_or_enlarge_or_stay_the_same(images, shrink_ratio, target_nx, target_radius, newx, nima): from fundamentals import resample from utilities import pad if newx >= target_nx : msk = model_circle(target_radius, target_nx, target_nx) else: msk = model_circle(newx//2-2, newx, newx) do_not_stay_the_same_size = shrink_ratio != 1.0 cut_to_window = newx > target_nx need_padding = newx < target_nx for im in xrange(nima): if do_not_stay_the_same_size : images[im] = resample(images[im], shrink_ratio) if cut_to_window : images[im] = Util.window(images[im], target_nx, target_nx, 1) p = Util.infomask(images[im], msk, False) images[im] -= p[0] p = Util.infomask(images[im], msk, True) images[im] /= p[1] if need_padding : images[im] = pad(images[im], target_nx, target_nx, 1, 0.0)
def main(args): progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" + " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" + " --generation=generation --rand_seed=rand_seed>" ) parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--radius", type="int", help="particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)") parser.add_option("--target_radius", type="int", default=29, help="target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)") parser.add_option("--target_nx", type="int", default=76, help="target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1' (default 76)") parser.add_option("--img_per_grp", type="int", default=100, help="number of images per class: in the ideal case (essentially maximum size of class) (default 100)") parser.add_option("--CTF", action="store_true", default=False, help="apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)") parser.add_option("--ir", type="int", default=1, help="inner ring: of the resampling to polar coordinates. units - pixels (default 1)") parser.add_option("--rs", type="int", default=1, help="ring step: of the resampling to polar coordinates. units - pixels (default 1)") parser.add_option("--xr", type="int", default=1, help="x range: of translational search. By default, set by the program. (default 1)") parser.add_option("--yr", type="int", default=-1, help="y range: of translational search. By default, same as xr. (default -1)") parser.add_option("--ts", type="float", default=1.0, help="search step: of translational search: units - pixels (default 1.0)") parser.add_option("--maxit", type="int", default=30, help="number of iterations for reference-free alignment: (default 30)") #parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)") parser.add_option("--center_method", type="int", default=-1, help="method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)") parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment: (default 90.0)") parser.add_option("--FL", type="float", default=0.2, help="lowest stopband: frequency used in the tangent filter (default 0.2)") parser.add_option("--FH", type="float", default=0.3, help="highest stopband: frequency used in the tangent filter (default 0.3)") parser.add_option("--FF", type="float", default=0.2, help="fall-off of the tangent filter: (default 0.2)") parser.add_option("--init_iter", type="int", default=3, help="SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)") parser.add_option("--main_iter", type="int", default=3, help="SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)") parser.add_option("--iter_reali", type="int", default=1, help="SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)") parser.add_option("--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase: (default 1)") parser.add_option("--max_round", type="int", default=20, help="maximum rounds: of generating candidate class averages in the first phase (default 20)") parser.add_option("--match_second", type="int", default=5, help="number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)") parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability: (default 5)") parser.add_option("--thld_err", type="float", default=0.7, help="threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)") parser.add_option("--indep_run", type="int", default=4, help="level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching. Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)") parser.add_option("--thld_grp", type="int", default=10, help="minimum size of reproducible class (default 10)") parser.add_option("--n_generations", type="int", default=100, help="maximum number of generations: program stops when reaching this total number of generations: (default 100)") #parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)") parser.add_option("--rand_seed", type="int", help="random seed set before calculations: useful for testing purposes. By default, total randomness (type int)") parser.add_option("--new", action="store_true", default=False, help="use new code: (default False)") parser.add_option("--debug", action="store_true", default=False, help="debug info printout: (default False)") # must be switched off in production parser.add_option("--use_latest_master_directory",action="store_true", default=False, help="use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)") parser.add_option("--restart_section", type="string", default=' ', help="restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')") parser.add_option("--stop_after_candidates", action="store_true", default=False, help="stop after candidates: stops after the 'candidate_class_averages' section. (default False)") ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP) parser.add_option("--skip_prealignment", action="store_true", default=False, help="skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)") required_option_list = ['radius'] (options, args) = parser.parse_args(args) if options.return_options: return parser if len(args) > 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" sys.exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True from isac import iter_isac from fundamentals import rot_shift2D, resample from utilities import pad, combine_params2 command_line_provided_stack_filename = args[0] main_node = 0 mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) nproc = mpi_comm_size(MPI_COMM_WORLD) mpi_barrier(MPI_COMM_WORLD) if(myid == main_node): print "****************************************************************" Util.version() print "****************************************************************" sys.stdout.flush() mpi_barrier(MPI_COMM_WORLD) # Making sure all required options appeared. for required_option in required_option_list: if not options.__dict__[required_option]: print "\n ==%s== mandatory option is missing.\n"%required_option print "Please run '" + progname + " -h' for detailed options" return 1 radi = options.radius target_radius = options.target_radius target_nx = options.target_nx center_method = options.center_method if(radi < 1): ERROR("Particle radius has to be provided!","sxisac",1,myid) use_latest_master_directory = options.use_latest_master_directory stop_after_candidates = options.stop_after_candidates # program_state_stack.restart_location_title_from_command_line = options.restart_section from utilities import qw program_state_stack.PROGRAM_STATE_VARIABLES = set(qw(""" isac_generation """)) # create or reuse master directory masterdir = "" stack_processed_by_ali2d_base__filename = "" stack_processed_by_ali2d_base__filename__without_master_dir = "" error_status = 0 if len(args) == 2: masterdir = args[1] elif len(args) == 1: if use_latest_master_directory: all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)] import re; r = re.compile("^master.*$") all_dirs = filter(r.match, all_dirs) if len(all_dirs)>0: # all_dirs = max(all_dirs, key=os.path.getctime) masterdir = max(all_dirs, key=os.path.getmtime) #Create folder for all results or check if there is one created already if(myid == main_node): if( masterdir == ""): timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime()) masterdir = "master"+timestring cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) elif not os.path.exists(masterdir): # os.path.exists(masterdir) does not exist masterdir = args[1] cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) if(args[0][:4] == "bdb:"): filename = args[0][4:] else: filename = args[0][:-4] filename = os.path.basename(filename) stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(masterdir, filename ) stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename if_error_then_all_processes_exit_program(error_status) # send masterdir to all processes masterdir = send_string_to_all(masterdir) if myid == 0: if options.restart_section != " ": if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)): stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)) import re if "," in options.restart_section: parsed_restart_section_option = options.restart_section.split(",") stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"]) generation_str_format = parsed_restart_section_option[1] if generation_str_format != "": isac_generation_from_command_line = int(generation_str_format) stored_state[-1]["isac_generation"] = isac_generation_from_command_line else: isac_generation_from_command_line = 1 if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"]) if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack) else: print "Please remove the restart_section option from the command line. The program must be started from the beginning." mpi_finalize() sys.exit() else: isac_generation_from_command_line = -1 program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)) stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename) stack_processed_by_ali2d_base__filename__without_master_dir = \ send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir) # previous code 2016-05-05--20-14-12-153 # # PARAMETERS OF THE PROCEDURE # if( options.xr == -1 ): # # Default values # # target_nx = 76 # # target_radius = 29 # target_xr = 1 # else: # nx//2 # # Check below! # target_xr = options.xr # # target_nx = 76 + target_xr - 1 # subtract one, which is default # target_nx += target_xr - 1 # subtract one, which is default # # target_radius = 29 target_xr = options.xr target_nx += target_xr - 1 # subtract one, which is default if (options.yr == -1): yr = options.xr else: yr = options.yr mpi_barrier(MPI_COMM_WORLD) # Initialization of stacks if(myid == main_node): print "command_line_provided_stack_filename", command_line_provided_stack_filename number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename) else: number_of_images_in_stack = 0 number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node) nxrsteps = 4 init2dir = os.path.join(masterdir,"2dalignment") # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() if not os.path.exists(os.path.join(init2dir, "Finished_initial_2d_alignment.txt")): if(myid == 0): import subprocess from logger import Logger, BaseLogger_Files # Create output directory log2d = Logger(BaseLogger_Files()) log2d.prefix = os.path.join(init2dir) cmd = "mkdir -p "+log2d.prefix outcome = subprocess.call(cmd, shell=True) log2d.prefix += "/" # outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True) else: outcome = 0 log2d = None if(myid == main_node): a = get_im(command_line_provided_stack_filename) nnxo = a.get_xsize() else: nnxo = 0 nnxo = bcast_number_to_all(nnxo, source_node = main_node) image_start, image_end = MPI_start_end(number_of_images_in_stack, nproc, myid) if options.skip_prealignment: params2d = [[0.0,0.0,0.0,0] for i in xrange(image_start, image_end)] else: original_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end)) # We assume the target radius will be 29, and xr = 1. shrink_ratio = float(target_radius)/float(radi) for im in xrange(len(original_images)): if(shrink_ratio != 1.0): original_images[im] = resample(original_images[im], shrink_ratio) nx = original_images[0].get_xsize() # nx = int(nx*shrink_ratio + 0.5) txrm = (nx - 2*(target_radius+1))//2 if(txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d"%(radi), "sxisac",1, myid) if(txrm/nxrsteps>0): tss = "" txr = "" while(txrm/nxrsteps>0): tts=txrm/nxrsteps tss += " %d"%tts txr += " %d"%(tts*nxrsteps) txrm =txrm//2 else: tss = "1" txr = "%d"%txrm # print "nx, txr, txrm, tss", nx, txr, txrm, tss # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() # section ali2d_base params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \ False, 90.0, center_method, 14, options.CTF, 1.0, False, \ "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False) del original_images for i in xrange(len(params2d)): alpha, sx, sy, mirror = combine_params2(0, params2d[i][1],params2d[i][2], 0, -params2d[i][0], 0, 0, 0) sx /= shrink_ratio sy /= shrink_ratio params2d[i][0] = 0.0 params2d[i][1] = sx params2d[i][2] = sy params2d[i][3] = 0 #set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0]) mpi_barrier(MPI_COMM_WORLD) tmp = params2d[:] tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD) if( myid == main_node ): if options.skip_prealignment: print "=========================================" print "Even though there is no alignment step, '%s' params are set to zero for later use."%os.path.join(init2dir, "initial2Dparams.txt") print "=========================================" write_text_row(tmp,os.path.join(init2dir, "initial2Dparams.txt")) del tmp mpi_barrier(MPI_COMM_WORLD) # We assume the target image size will be target_nx, radius will be 29, and xr = 1. # Note images can be also padded, in which case shrink_ratio > 1. shrink_ratio = float(target_radius)/float(radi) aligned_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end)) nx = aligned_images[0].get_xsize() nima = len(aligned_images) newx = int(nx*shrink_ratio + 0.5) while not os.path.exists(os.path.join(init2dir, "initial2Dparams.txt")): import time time.sleep(1) mpi_barrier(MPI_COMM_WORLD) params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt")) params = params[image_start:image_end] msk = model_circle(radi, nx, nx) for im in xrange(nima): st = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= st[0] if options.CTF: aligned_images[im] = filt_ctf(aligned_images[im], aligned_images[im].get_attr("ctf"), binary = True) if(shrink_ratio < 1.0): if newx > target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) #aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx : msk = model_circle(newx//2-2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif(shrink_ratio == 1.0): if newx > target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx : msk = model_circle(newx//2-2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) #aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif(shrink_ratio > 1.0): if newx > target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx : msk = model_circle(newx//2-2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) del msk gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid) number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node) if( myid == main_node ): for i in range(number_of_images_in_stack): aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i) # It has to be explicitly closed from EMAN2db import db_open_dict DB = db_open_dict(stack_processed_by_ali2d_base__filename) DB.close() fp = open(os.path.join(masterdir,"README_shrink_ratio.txt"), "w") output_text = """ Since, for processing purposes, isac changes the image dimensions, adjustment of pixel size needs to be made in subsequent steps, (e.g. running sxviper.py). The shrink ratio for this particular isac run is -------- %.5f %.5f -------- To get the pixel size for the isac output the user needs to divide the original pixel size by the above value. This info is saved in the following file: README_shrink_ratio.txt """%(shrink_ratio, radi) fp.write(output_text); fp.flush() ;fp.close() print output_text fp = open(os.path.join(init2dir, "Finished_initial_2d_alignment.txt"), "w"); fp.flush() ;fp.close() else: if( myid == main_node ): print "Skipping 2d alignment since it was already done!" mpi_barrier(MPI_COMM_WORLD) # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() os.chdir(masterdir) if program_state_stack(locals(), getframeinfo(currentframe())): # if 1: pass if (myid == main_node): cmdexecute("sxheader.py --consecutive --params=originalid %s"%stack_processed_by_ali2d_base__filename__without_master_dir) cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir)) if (myid == main_node): main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d") print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no if isac_generation_from_command_line < 0: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1): if i == isac_generation_from_command_line+1: backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%i + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i) # it includes both command line and json file my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1] if "restart" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line ) elif "candidate_class_averages" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line) # cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line) elif "reproducible_class_averages" in my_restart_section: cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line) cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line) else: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 0 isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0] isac_generation = isac_generation_from_command_line - 1 if (myid == main_node): if isac_generation == 0: cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation) write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation)) write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation)) # Stopping criterion should be inside the program. while True: isac_generation += 1 if isac_generation > options.n_generations: break data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation program_state_stack.restart_location_title = "restart" if program_state_stack(locals(), getframeinfo(currentframe())): if (myid == main_node): cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation) # reference the original stack list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1)) cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\ stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file)) mpi_barrier(MPI_COMM_WORLD) os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation) program_state_stack.restart_location_title = "candidate_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new) # program_state_stack.restart_location_title = "stopped_program1" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack.restart_location_title = "stop_after_candidates" program_state_stack(locals(), getframeinfo(currentframe())) if stop_after_candidates: mpi_finalize() sys.exit() exit_program = 0 if(myid == main_node): if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation): print "This generation (%d) no class average candidates were generated! Finishing."%isac_generation exit_program = 1 exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if exit_program: os.chdir("..") break program_state_stack.restart_location_title = "reproducible_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new) pass os.chdir("..") if(myid == main_node): accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_accounted.txt"%(isac_generation))) number_of_accounted_images = len(accounted_images) un_accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_unaccounted.txt"%(isac_generation))) number_of_un_accounted_images = len(un_accounted_images) else: number_of_accounted_images = 0 number_of_un_accounted_images = 0 number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) number_of_un_accounted_images = int(mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if number_of_accounted_images == 0: if(myid == main_node): print "This generation (%d) there are no accounted images! Finishing."%isac_generation break while (myid == main_node): def files_are_missing(isac_generation): for i in xrange(1, isac_generation + 1): if not os.path.exists("generation_%04d/class_averages_generation_%d.hdf"%(i,i)): print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting."%(i,i) return 1 return 0 if files_are_missing(isac_generation): break cmdexecute("rm -f class_averages.hdf") cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation + 1)], "class_averages.hdf") break if number_of_un_accounted_images == 0: if(myid == main_node): print "This generation (%d) there are no un accounted images! Finishing."%isac_generation break program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall") mpi_barrier(MPI_COMM_WORLD) mpi_finalize()
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Generic 2-D image processing programs. Functionality: 1. Phase flip a stack of images and write output to new file: sxprocess.py input_stack.hdf output_stack.hdf --phase_flip 2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size. The window size will change accordingly. sxprocess input.hdf output.hdf --changesize --ratio=0.5 3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes or a 3-D volume. sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024] 4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs: sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64 5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3): sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid 6. Retrieve original image numbers of images listed in ISAC output stack of averages: sxprocess.py select1.hdf ohk.txt 7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file. Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack. sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw 8. Generate a 1D rotationally averaged power spectrum of an image. sxprocess.py vol.hdf --rotwp=rotpw.txt # Output will contain three columns: (1) rotationally averaged power spectrum (2) logarithm of the rotationally averaged power spectrum (3) integer line number (from zero to approximately to half the image size) 9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data. sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt The output file is then imported and 3D transformed volume computed: sxheader.py bdb:p --params=xform.projection --import=output.txt mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI The reconstructed volume is in the position of the volume computed using the input.txt parameters and then transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz) 10. Import ctf parameters from the output of sxcter into windowed particle headers. There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph. In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'. Normally this is done by e2boxer.py during windowing. Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created. sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 # Output will be a vritual stack bdb:Gdata sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 To concatenate output files: cd directory e2bdb.py . --makevstack=bdb:allparticles --filt=G IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them. 11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor sxprocess.py orientationparams.txt scaledparams.txt scale=0.5 12. Generate 3D mask from a given 3-D volume automatically or using threshold provided by user. 13. Postprocess 3-D or 2-D images: for 3-D volumes: calculate FSC with provided mask; weight summed volume with FSC; estimate B-factor from FSC weighted summed two volumes; apply negative B-factor to the weighted volume. for 2-D images: calculate B-factor and apply negative B-factor to 2-D images. 14. Winow stack file -reduce size of images without changing the pixel size. """ parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--order", action="store_true", help="Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False) parser.add_option("--order_lookup", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_metropolis", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_pca", action="store_true", help="Test/Debug.", default=False) parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)") parser.add_option("--circular", action="store_true", help="Select circular ordering (fisr image has to be similar to the last", default=False) parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") parser.add_option("--changesize", action="store_true", help="resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False) parser.add_option("--ratio", type="float", default=1.0, help="The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round") parser.add_option("--pw", action="store_true", help="compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False) parser.add_option("--wn", type="int", default=-1, help="Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))") parser.add_option("--phase_flip", action="store_true", help="Phase flip the input stack", default=False) parser.add_option("--makedb", metavar="param1=value1:param2=value2", type="string", action="append", help="One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0") parser.add_option("--generate_projections", metavar="param1=value1:param2=value2", type="string", action="append", help="Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64.") parser.add_option("--isacgroup", type="int", help="Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1) parser.add_option("--isacselect", action="store_true", help="Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False) parser.add_option("--params", type="string", default=None, help="Name of header of parameter, which one depends on specific option") parser.add_option("--adjpw", action="store_true", help="Adjust rotationally averaged power spectrum of an image", default=False) parser.add_option("--rotpw", type="string", default=None, help="Name of the text file to contain rotationally averaged power spectrum of the input image.") parser.add_option("--transformparams", type="string", default=None, help="Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters.") # import ctf estimates done using cter parser.add_option("--input", type="string", default= None, help="Input particles.") parser.add_option("--importctf", type="string", default= None, help="Name of the file containing CTF parameters produced by sxcter.") parser.add_option("--defocuserror", type="float", default=1000000.0, help="Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%") parser.add_option("--astigmatismerror", type="float", default=360.0, help="Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees.") # import ctf estimates done using cter parser.add_option("--scale", type="float", default=-1.0, help="Divide shifts in the input 3D orientation parameters text file by the scale factor.") # generate adaptive mask from an given 3-D volume parser.add_option("--adaptive_mask", action="store_true", help="create adavptive 3-D mask from a given volume", default=False) parser.add_option("--nsigma", type="float", default= 1., help="number of times of sigma of the input volume to obtain the the large density cluster") parser.add_option("--ndilation", type="int", default= 3, help="number of times of dilation applied to the largest cluster of density") parser.add_option("--kernel_size", type="int", default= 11, help="convolution kernel for smoothing the edge of the mask") parser.add_option("--gauss_standard_dev", type="int", default= 9, help="stanadard deviation value to generate Gaussian edge") parser.add_option("--threshold", type="float", default= 9999., help="threshold provided by user to binarize input volume") parser.add_option("--ne", type="int", default= 0, help="number of times to erode the binarized input image") parser.add_option("--nd", type="int", default= 0, help="number of times to dilate the binarized input image") parser.add_option("--postprocess", action="store_true", help="postprocess unfiltered odd, even 3-D volumes",default=False) parser.add_option("--fsc_weighted", action="store_true", help="postprocess unfiltered odd, even 3-D volumes") parser.add_option("--low_pass_filter", action="store_true", default=False, help="postprocess unfiltered odd, even 3-D volumes") parser.add_option("--ff", type="float", default=.25, help="low pass filter stop band frequency in absolute unit") parser.add_option("--aa", type="float", default=.1, help="low pass filter falloff" ) parser.add_option("--mask", type="string", help="input mask file", default=None) parser.add_option("--output", type="string", help="output file name", default="postprocessed.hdf") parser.add_option("--pixel_size", type="float", help="pixel size of the data", default=1.0) parser.add_option("--B_start", type="float", help="starting frequency in Angstrom for B-factor estimation", default=10.) parser.add_option("--FSC_cutoff", type="float", help="stop frequency in Angstrom for B-factor estimation", default=0.143) parser.add_option("--2d", action="store_true", help="postprocess isac 2-D averaged images",default=False) parser.add_option("--window_stack", action="store_true", help="window stack images using a smaller window size", default=False) parser.add_option("--box", type="int", default= 0, help="the new window size ") (options, args) = parser.parse_args() global_def.BATCH = True if options.phase_flip: nargs = len(args) if nargs != 2: print "must provide name of input and output file!" return from EMAN2 import Processor instack = args[0] outstack = args[1] nima = EMUtil.get_image_count(instack) from filter import filt_ctf for i in xrange(nima): img = EMData() img.read_image(instack, i) try: ctf = img.get_attr('ctf') except: print "no ctf information in input stack! Exiting..." return dopad = True sign = 1 binary = 1 # phase flip assert img.get_ysize() > 1 dict = ctf.to_dict() dz = dict["defocus"] cs = dict["cs"] voltage = dict["voltage"] pixel_size = dict["apix"] b_factor = dict["bfactor"] ampcont = dict["ampcont"] dza = dict["dfdiff"] azz = dict["dfang"] if dopad and not img.is_complex(): ip = 1 else: ip = 0 params = {"filter_type": Processor.fourier_filter_types.CTF_, "defocus" : dz, "Cs": cs, "voltage": voltage, "Pixel_size": pixel_size, "B_factor": b_factor, "amp_contrast": ampcont, "dopad": ip, "binary": binary, "sign": sign, "dza": dza, "azz":azz} tmp = Processor.EMFourierFilter(img, params) tmp.set_attr_dict({"ctf": ctf}) tmp.write_image(outstack, i) elif options.changesize: nargs = len(args) if nargs != 2: ERROR("must provide name of input and output file!", "change size", 1) return from utilities import get_im instack = args[0] outstack = args[1] sub_rate = float(options.ratio) nima = EMUtil.get_image_count(instack) from fundamentals import resample for i in xrange(nima): resample(get_im(instack, i), sub_rate).write_image(outstack, i) elif options.isacgroup>-1: nargs = len(args) if nargs != 3: ERROR("Three files needed on input!", "isacgroup", 1) return from utilities import get_im instack = args[0] m=get_im(args[1],int(options.isacgroup)).get_attr("members") l = [] for k in m: l.append(int(get_im(args[0],k).get_attr(options.params))) from utilities import write_text_file write_text_file(l, args[2]) elif options.isacselect: nargs = len(args) if nargs != 2: ERROR("Two files needed on input!", "isacgroup", 1) return from utilities import get_im nima = EMUtil.get_image_count(args[0]) m = [] for k in xrange(nima): m += get_im(args[0],k).get_attr("members") m.sort() from utilities import write_text_file write_text_file(m, args[1]) elif options.pw: nargs = len(args) if nargs < 2: ERROR("must provide name of input and output file!", "pw", 1) return from utilities import get_im, write_text_file from fundamentals import rops_table d = get_im(args[0]) ndim = d.get_ndim() if ndim ==3: pw = rops_table(d) write_text_file(pw, args[1]) else: nx = d.get_xsize() ny = d.get_ysize() if nargs ==3: mask = get_im(args[2]) wn = int(options.wn) if wn == -1: wn = max(nx, ny) else: if( (wn<nx) or (wn<ny) ): ERROR("window size cannot be smaller than the image size","pw",1) n = EMUtil.get_image_count(args[0]) from utilities import model_blank, model_circle, pad from EMAN2 import periodogram p = model_blank(wn,wn) for i in xrange(n): d = get_im(args[0], i) if nargs==3: d *=mask st = Util.infomask(d, None, True) d -= st[0] p += periodogram(pad(d, wn, wn, 1, 0.)) p /= n p.write_image(args[1]) elif options.adjpw: if len(args) < 3: ERROR("filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)","adjpw",1) return img_stack = args[0] from math import sqrt from fundamentals import rops_table, fft from utilities import read_text_file, get_im from filter import filt_tanl, filt_table if( args[1][-3:] == 'txt'): rops_dst = read_text_file( args[1] ) else: rops_dst = rops_table(get_im( args[1] )) out_stack = args[2] if(len(args) >4): fl = float(args[3]) aa = float(args[4]) else: fl = -1.0 aa = 0.0 nimage = EMUtil.get_image_count( img_stack ) for i in xrange(nimage): img = fft(get_im(img_stack, i) ) rops_src = rops_table(img) assert len(rops_dst) == len(rops_src) table = [0.0]*len(rops_dst) for j in xrange( len(rops_dst) ): table[j] = sqrt( rops_dst[j]/rops_src[j] ) if( fl > 0.0): img = filt_tanl(img, fl, aa) img = fft(filt_table(img, table)) img.write_image(out_stack, i) elif options.rotpw != None: if len(args) != 1: ERROR("Only one input permitted","rotpw",1) return from utilities import write_text_file, get_im from fundamentals import rops_table from math import log10 t = rops_table(get_im(args[0])) x = range(len(t)) r = [0.0]*len(x) for i in x: r[i] = log10(t[i]) write_text_file([t,r,x],options.rotpw) elif options.transformparams != None: if len(args) != 2: ERROR("Please provide names of input and output files with orientation parameters","transformparams",1) return from utilities import read_text_row, write_text_row transf = [0.0]*6 spl=options.transformparams.split(',') for i in xrange(len(spl)): transf[i] = float(spl[i]) write_text_row( rotate_shift_params(read_text_row(args[0]), transf) , args[1]) elif options.makedb != None: nargs = len(args) if nargs != 1: print "must provide exactly one argument denoting database key under which the input params will be stored" return dbkey = args[0] print "database key under which params will be stored: ", dbkey gbdb = js_open_dict("e2boxercache/gauss_box_DB.json") parmstr = 'dummy:'+options.makedb[0] (processorname, param_dict) = parsemodopt(parmstr) dbdict = {} for pkey in param_dict: if (pkey == 'invert_contrast') or (pkey == 'use_variance'): if param_dict[pkey] == 'True': dbdict[pkey] = True else: dbdict[pkey] = False else: dbdict[pkey] = param_dict[pkey] gbdb[dbkey] = dbdict elif options.generate_projections: nargs = len(args) if nargs != 3: ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\ "sxprocess - generate projections",1) return inpstr = args[0] outstk = args[1] micpref = args[2] parmstr = 'dummy:'+options.generate_projections[0] (processorname, param_dict) = parsemodopt(parmstr) parm_CTF = False parm_format = 'bdb' parm_apix = 2.5 if 'CTF' in param_dict: if param_dict['CTF'] == 'True': parm_CTF = True if 'format' in param_dict: parm_format = param_dict['format'] if 'apix' in param_dict: parm_apix = float(param_dict['apix']) boxsize = 64 if 'boxsize' in param_dict: boxsize = int(param_dict['boxsize']) print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize scale_mult = 2500 sigma_add = 1.5 sigma_proj = 30.0 sigma2_proj = 17.5 sigma_gauss = 0.3 sigma_mic = 30.0 sigma2_mic = 17.5 sigma_gauss_mic = 0.3 if 'scale_mult' in param_dict: scale_mult = float(param_dict['scale_mult']) if 'sigma_add' in param_dict: sigma_add = float(param_dict['sigma_add']) if 'sigma_proj' in param_dict: sigma_proj = float(param_dict['sigma_proj']) if 'sigma2_proj' in param_dict: sigma2_proj = float(param_dict['sigma2_proj']) if 'sigma_gauss' in param_dict: sigma_gauss = float(param_dict['sigma_gauss']) if 'sigma_mic' in param_dict: sigma_mic = float(param_dict['sigma_mic']) if 'sigma2_mic' in param_dict: sigma2_mic = float(param_dict['sigma2_mic']) if 'sigma_gauss_mic' in param_dict: sigma_gauss_mic = float(param_dict['sigma_gauss_mic']) from filter import filt_gaussl, filt_ctf from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank,pad,model_gauss_noise,set_params2D, set_params_proj from projection import prep_vol,prgs seed(14567) delta = 29 angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S") nangle = len(angles) modelvol = [] nvlms = EMUtil.get_image_count(inpstr) from utilities import get_im for k in xrange(nvlms): modelvol.append(get_im(inpstr,k)) nx = modelvol[0].get_xsize() if nx != boxsize: ERROR("Requested box dimension does not match dimension of the input model.", \ "sxprocess - generate projections",1) nvol = 10 volfts = [[] for k in xrange(nvlms)] for k in xrange(nvlms): for i in xrange(nvol): sigma = sigma_add + random() # 1.5-2.5 addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40 ) scale = scale_mult * (0.5+random()) vf, kb = prep_vol(modelvol[k] + scale*addon) volfts[k].append(vf) del vf, modelvol if parm_format == "bdb": stack_data = "bdb:"+outstk delete_bdb(stack_data) else: stack_data = outstk + ".hdf" Cs = 2.0 pixel = parm_apix voltage = 120.0 ampcont = 10.0 ibd = 4096/2-boxsize iprj = 0 width = 240 xstart = 8 + boxsize/2 ystart = 8 + boxsize/2 rowlen = 17 from random import randint params = [] for idef in xrange(3, 8): irow = 0 icol = 0 mic = model_blank(4096, 4096) defocus = idef * 0.5#0.2 if parm_CTF: astampl=defocus*0.15 astangl=50.0 ctf = generate_ctf([defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl]) for i in xrange(nangle): for k in xrange(12): dphi = 8.0*(random()-0.5) dtht = 8.0*(random()-0.5) psi = 360.0*random() phi = angles[i][0]+dphi tht = angles[i][1]+dtht s2x = 4.0*(random()-0.5) s2y = 4.0*(random()-0.5) params.append([phi, tht, psi, s2x, s2y]) ivol = iprj % nvol #imgsrc = randint(0,nvlms-1) imgsrc = iprj % nvlms proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y]) x = xstart + irow * width y = ystart + icol * width mic += pad(proj, 4096, 4096, 1, 0.0, x-2048, y-2048, 0) proj = proj + model_gauss_noise( sigma_proj, nx, nx ) if parm_CTF: proj = filt_ctf(proj, ctf) proj.set_attr_dict({"ctf":ctf, "ctf_applied":0}) proj = proj + filt_gaussl(model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss) proj.set_attr("origimgsrc",imgsrc) proj.set_attr("test_id", iprj) # flags describing the status of the image (1 = true, 0 = false) set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0]) set_params_proj(proj, [phi, tht, psi, s2x, s2y]) proj.write_image(stack_data, iprj) icol += 1 if icol == rowlen: icol = 0 irow += 1 iprj += 1 mic += model_gauss_noise(sigma_mic,4096,4096) if parm_CTF: #apply CTF mic = filt_ctf(mic, ctf) mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic) mic.write_image(micpref + "%1d.hdf" % (idef-3), 0) drop_spider_doc("params.txt", params) elif options.importctf != None: print ' IMPORTCTF ' from utilities import read_text_row,write_text_row from random import randint import subprocess grpfile = 'groupid%04d'%randint(1000,9999) ctfpfile = 'ctfpfile%04d'%randint(1000,9999) cterr = [options.defocuserror/100.0, options.astigmatismerror] ctfs = read_text_row(options.importctf) for kk in xrange(len(ctfs)): root,name = os.path.split(ctfs[kk][-1]) ctfs[kk][-1] = name[:-4] if(options.input[:4] != 'bdb:'): ERROR('Sorry, only bdb files implemented','importctf',1) d = options.input[4:] #try: str = d.index('*') #except: str = -1 from string import split import glob uu = os.path.split(d) uu = os.path.join(uu[0],'EMAN2DB',uu[1]+'.bdb') flist = glob.glob(uu) for i in xrange(len(flist)): root,name = os.path.split(flist[i]) root = root[:-7] name = name[:-4] fil = 'bdb:'+os.path.join(root,name) sourcemic = EMUtil.get_all_attributes(fil,'ptcl_source_image') nn = len(sourcemic) gctfp = [] groupid = [] for kk in xrange(nn): junk,name2 = os.path.split(sourcemic[kk]) name2 = name2[:-4] ctfp = [-1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] for ll in xrange(len(ctfs)): if(name2 == ctfs[ll][-1]): # found correct if(ctfs[ll][8]/ctfs[ll][0] <= cterr[0]): # acceptable defocus error ctfp = ctfs[ll][:8] if(ctfs[ll][10] > cterr[1] ): # error of astigmatism exceed the threshold, set astigmatism to zero. ctfp[6] = 0.0 ctfp[7] = 0.0 gctfp.append(ctfp) groupid.append(kk) break if(len(groupid) > 0): write_text_row(groupid, grpfile) write_text_row(gctfp, ctfpfile) cmd = "{} {} {} {}".format('e2bdb.py',fil,'--makevstack=bdb:'+root+'G'+name,'--list='+grpfile) #print cmd subprocess.call(cmd, shell=True) cmd = "{} {} {} {}".format('sxheader.py','bdb:'+root+'G'+name,'--params=ctf','--import='+ctfpfile) #print cmd subprocess.call(cmd, shell=True) else: print ' >>> Group ',name,' skipped.' cmd = "{} {} {}".format("rm -f",grpfile,ctfpfile) subprocess.call(cmd, shell=True) elif options.scale > 0.0: from utilities import read_text_row,write_text_row scale = options.scale nargs = len(args) if nargs != 2: print "Please provide names of input and output file!" return p = read_text_row(args[0]) for i in xrange(len(p)): p[i][3] /= scale p[i][4] /= scale write_text_row(p, args[1]) elif options.adaptive_mask: from utilities import get_im from morphology import adaptive_mask, binarize, erosion, dilation nsigma = options.nsigma ndilation = options.ndilation kernel_size = options.kernel_size gauss_standard_dev = options.gauss_standard_dev nargs = len(args) if nargs ==0: print " Create 3D mask from a given volume, either automatically or from the user provided threshold." elif nargs > 2: print "Too many inputs are given, try again!" return else: inputvol = get_im(args[0]) input_path, input_file_name = os.path.split(args[0]) input_file_name_root,ext=os.path.splitext(input_file_name) if nargs == 2: mask_file_name = args[1] else: mask_file_name = "adaptive_mask_for_"+input_file_name_root+".hdf" # Only hdf file is output. if options.threshold !=9999.: mask3d = binarize(inputvol, options.threshold) for i in xrange(options.ne): mask3d = erosion(mask3d) for i in xrange(options.nd): mask3d = dilation(mask3d) else: mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev) mask3d.write_image(mask_file_name) elif options.postprocess: from utilities import get_im from fundamentals import rot_avg_table from morphology import compute_bfactor,power from statistics import fsc from filter import filt_table, filt_gaussinv from EMAN2 import periodogram e1 = get_im(args[0],0) if e1.get_zsize()==1: nimage = EMUtil.get_image_count(args[0]) if options.mask !=None: m = get_im(options.mask) else: m = None for i in xrange(nimage): e1 = get_im(args[0],i) if m: e1 *=m guinerline = rot_avg_table(power(periodogram(e1),.5)) freq_max = 1/(2.*pixel_size) freq_min = 1./options.B_start b,junk=compute_bfactor(guinerline, freq_min, freq_max, pixel_size) tmp = b/pixel_size**2 sigma_of_inverse=sqrt(2./tmp) e1 = filt_gaussinv(e1,sigma_of_inverse) if options.low_pass_filter: from filter import filt_tanl e1 =filt_tanl(e1,options.ff, options.aa) e1.write_image(options.output) else: nargs = len(args) e1 = get_im(args[0]) if nargs >1: e2 = get_im(args[1]) if options.mask !=None: m = get_im(options.mask) else: m =None pixel_size = options.pixel_size from math import sqrt if m !=None: e1 *=m if nargs >1 :e2 *=m if options.fsc_weighted: frc = fsc(e1,e2,1) ## FSC is done on masked two images #### FSC weighting sqrt((2.*fsc)/(1+fsc)); fil = len(frc[1])*[None] for i in xrange(len(fil)): if frc[1][i]>=options.FSC_cutoff: tmp = frc[1][i] else: tmp = 0.0 fil[i] = sqrt(2.*tmp/(1.+tmp)) if nargs>1: e1 +=e2 if options.fsc_weighted: e1=filt_table(e1,fil) guinerline = rot_avg_table(power(periodogram(e1),.5)) freq_max = 1/(2.*pixel_size) freq_min = 1./options.B_start b,junk = compute_bfactor(guinerline, freq_min, freq_max, pixel_size) tmp = b/pixel_size**2 sigma_of_inverse=sqrt(2./tmp) e1 = filt_gaussinv(e1,sigma_of_inverse) if options.low_pass_filter: from filter import filt_tanl e1 =filt_tanl(e1,options.ff, options.aa) e1.write_image(options.output) elif options.window_stack: nargs = len(args) if nargs ==0: print " Reduce image size of a stack" return else: output_stack_name = None inputstack = args[0] if nargs ==2:output_stack_name = args[1] input_path,input_file_name=os.path.split(inputstack) input_file_name_root,ext=os.path.splitext(input_file_name) if input_file_name_root[0:3]=="bdb":stack_is_bdb= True else: stack_is_bdb= False if output_stack_name is None: if stack_is_bdb: output_stack_name ="bdb:reduced_"+input_file_name_root[4:] else:output_stack_name = "reduced_"+input_file_name_root+".hdf" # Only hdf file is output. nimage = EMUtil.get_image_count(inputstack) from fundamentals import window2d for i in xrange(nimage): image = EMData() image.read_image(inputstack,i) w = window2d(image,options.box,options.box) w.write_image(output_stack_name,i) else: ERROR("Please provide option name","sxprocess.py",1)
def main(args): progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" + " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" + " --generation=generation --rand_seed=rand_seed>" ) parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--radius", type="int", default=-1, help="<Particle radius>, it has to be provided.") parser.add_option("--img_per_grp", type="int", default=100, help="<number of images per group> in the ideal case (essentially maximum size of class) (100)") parser.add_option("--CTF", action="store_true", default=False, help="<CTF flag>, if set the data will be phase-flipped") parser.add_option("--ir", type="int", default=1, help="<inner ring> of the resampling to polar coordinates (1)") parser.add_option("--rs", type="int", default=1, help="<ring step> of the resampling to polar coordinates (1)") parser.add_option("--xr", type="int", default=-1, help="<x range> of translational search (By default set by the program) (advanced)") parser.add_option("--yr", type="int", default=-1, help="<y range> of translational search (same as xr) (advanced)") parser.add_option("--ts", type="float", default=1.0, help="<search step> of translational search (1.0)") parser.add_option("--maxit", type="int", default=30, help="number of iterations for reference-free alignment (30)") #parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)") parser.add_option("--center_method", type="int", default=7, help="<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)") parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment ") parser.add_option("--FL", type="float", default=0.2, help="<lowest stopband> frequency used in the tangent filter (0.2)") parser.add_option("--FH", type="float", default=0.3, help="<highest stopband> frequency used in the tangent filter (0.3)") parser.add_option("--FF", type="float", default=0.2, help="<fall-off of the tangent> filter (0.2)") parser.add_option("--init_iter", type="int", default=3, help="<init_iter> number of iterations of ISAC program in initialization (3)") parser.add_option("--main_iter", type="int", default=3, help="<main_iter> number of iterations of ISAC program in main part (3)") parser.add_option("--iter_reali", type="int", default=1, help="<iter_reali> number of iterations in ISAC before checking stability (1)") parser.add_option("--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase (1)") parser.add_option("--max_round", type="int", default=20, help="maximum rounds of generating candidate averages in the first phase (20)") parser.add_option("--match_second", type="int", default=5, help="number of iterations to run 2-way (or 3-way) matching in the second phase (5)") parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability (5)") parser.add_option("--thld_err", type="float", default=0.7, help="the threshold of pixel error when checking stability (0.7)") parser.add_option("--indep_run", type="int", default=4, help="number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)") parser.add_option("--thld_grp", type="int", default=10, help="minimum size of class (10)") parser.add_option("--n_generations", type="int", default=100, help="<n_generations> program stops when reaching this total number of generations (advanced)") #parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)") parser.add_option("--rand_seed", type="int", default=None, help="random seed set before calculations, useful for testing purposes (default None - total randomness)") parser.add_option("--new", action="store_true", default=False, help="use new code (default = False)") parser.add_option("--debug", action="store_true", default=False, help="debug info printout (default = False)") # must be switched off in production parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False) parser.add_option("--restart_section", type="string", default="", help="<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)") parser.add_option("--stop_after_candidates", action="store_true", default=False, help="<stop_after_candidates> stops after the 'candidate_class_averages' section") parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP) (options, args) = parser.parse_args(args) if options.return_options: return parser if len(args) > 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" sys.exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() from isac import iter_isac global_def.BATCH = True global_def.BATCH = True command_line_provided_stack_filename = args[0] global_def.BATCH = True main_node = 0 mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) nproc = mpi_comm_size(MPI_COMM_WORLD) radi = options.radius center_method = options.center_method if(radi < 1): ERROR("Particle radius has to be provided!","sxisac",1,myid) use_latest_master_directory = options.use_latest_master_directory stop_after_candidates = options.stop_after_candidates program_state_stack.restart_location_title_from_command_line = options.restart_section from utilities import qw program_state_stack.PROGRAM_STATE_VARIABLES = set(qw(""" isac_generation """)) # create or reuse master directory masterdir = "" stack_processed_by_ali2d_base__filename = "" stack_processed_by_ali2d_base__filename__without_master_dir = "" error_status = 0 if len(args) == 2: masterdir = args[1] elif len(args) == 1: if use_latest_master_directory: all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)] import re; r = re.compile("^master.*$") all_dirs = filter(r.match, all_dirs) if len(all_dirs)>0: # all_dirs = max(all_dirs, key=os.path.getctime) masterdir = max(all_dirs, key=os.path.getmtime) #Create folder for all results or check if there is one created already if(myid == main_node): if( masterdir == ""): timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime()) masterdir = "master"+timestring cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) elif not os.path.exists(masterdir): # os.path.exists(masterdir) does not exist masterdir = args[1] cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) if(args[0][:4] == "bdb:"): filename = args[0][4:] else: filename = args[0][:-4] filename = os.path.basename(filename) stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(masterdir, filename ) stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename if_error_all_processes_quit_program(error_status) # send masterdir to all processes masterdir = send_string_to_all(masterdir) if myid == 0: if options.restart_section != "": if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)): stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)) import re if "," in options.restart_section: parsed_restart_section_option = options.restart_section.split(",") stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"]) generation_str_format = parsed_restart_section_option[1] if generation_str_format != "": isac_generation_from_command_line = int(generation_str_format) stored_state[-1]["isac_generation"] = isac_generation_from_command_line else: isac_generation_from_command_line = 1 if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"]) if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack) else: print "Please remove the restart_section option from the command line. The program must be started from the beginning." mpi_finalize() sys.exit() else: isac_generation_from_command_line = -1 program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)) stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename) stack_processed_by_ali2d_base__filename__without_master_dir = \ send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir) # PARAMETERS OF THE PROCEDURE if( options.xr == -1 ): # Default values target_nx = 76 target_radius = 29 target_xr = 1 else: # nx//2 # Check below! target_xr = options.xr target_nx = 76 + target_xr - 1 # subtract one, which is default target_radius = 29 mpi_barrier(MPI_COMM_WORLD) # Initialization of stacks if(myid == main_node): number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename) else: number_of_images_in_stack = 0 number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node) nxrsteps = 4 init2dir = os.path.join(masterdir,"2dalignment") if(myid == 0): import subprocess from logger import Logger, BaseLogger_Files # Create output directory log2d = Logger(BaseLogger_Files()) log2d.prefix = os.path.join(init2dir) cmd = "mkdir -p "+log2d.prefix outcome = subprocess.call(cmd, shell=True) log2d.prefix += "/" # outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True) else: outcome = 0 log2d = None if(myid == main_node): a = get_im(command_line_provided_stack_filename) nnxo = a.get_xsize() else: nnxo = 0 nnxo = bcast_number_to_all(nnxo, source_node = main_node) txrm = (nnxo - 2*(radi+1))//2 if(txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d"%(radi), "sxisac",1, myid) if(txrm/nxrsteps>0): tss = "" txr = "" while(txrm/nxrsteps>0): tts=txrm/nxrsteps tss += " %d"%tts txr += " %d"%(tts*nxrsteps) txrm =txrm//2 else: tss = "1" txr = "%d"%txrm # section ali2d_base # centering method is set to #7 params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \ False, 90.0, center_method, 14, options.CTF, 1.0, False, \ "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False) if( myid == main_node ): write_text_row(params2d,os.path.join(init2dir, "initial2Dparams.txt")) del params2d mpi_barrier(MPI_COMM_WORLD) # We assume the target image size will be target_nx, radius will be 29, and xr = 1. # Note images can be also padded, in which case shrink_ratio > 1. shrink_ratio = float(target_radius)/float(radi) nx = aligned_images[0].get_xsize() nima = len(aligned_images) newx = int(nx*shrink_ratio + 0.5) from fundamentals import rot_shift2D, resample from utilities import pad, combine_params2 if(shrink_ratio < 1.0): if newx > target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx : msk = model_circle(newx//2-2, newx, newx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif(shrink_ratio == 1.0): if newx > target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx : msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx : msk = model_circle(nx//2-2, newx, newx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) #aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif(shrink_ratio > 1.0): target_radius = radi msk = model_circle(target_radius, nx, nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) del msk gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid) number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node) if( myid == main_node ): for i in range(number_of_images_in_stack): aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i) # It has to be explicitly closed from EMAN2db import db_open_dict DB = db_open_dict(stack_processed_by_ali2d_base__filename) DB.close() fp = open("README_shrink_ratio.txt", "w") output_text = """ Since, for processing purposes, isac changes the image dimensions, adjustment of pixel size needs to be made in subsequent steps, (e.g. running sxviper.py). The shrink ratio for this particular isac run is -------- %.5f -------- To get the pixel size for the isac output the user needs to divide the original pixel size by the above value. This info is saved in the following file: README_shrink_ratio.txt """%shrink_ratio fp.write(output_text); fp.flush() ;fp.close() print output_text mpi_barrier(MPI_COMM_WORLD) global_def.BATCH = True os.chdir(masterdir) if program_state_stack(locals(), getframeinfo(currentframe())): # if 1: pass if (myid == main_node): cmdexecute("sxheader.py --consecutive --params=originalid %s"%stack_processed_by_ali2d_base__filename__without_master_dir) cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir)) if (myid == main_node): main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d") print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no if isac_generation_from_command_line < 0: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1): if i == isac_generation_from_command_line+1: backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%i + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i) # it includes both command line and json file my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1] if "restart" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line ) elif "candidate_class_averages" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no) cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line) # cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line) elif "reproducible_class_averages" in my_restart_section: cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line) cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line) else: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 0 isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0] isac_generation = isac_generation_from_command_line - 1 if (myid == main_node): if isac_generation == 0: cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation) write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation)) write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation)) # Stopping criterion should be inside the program. while True: isac_generation += 1 if isac_generation > options.n_generations: break data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation if(myid == main_node): accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_accounted.txt"%(isac_generation - 1))) number_of_accounted_images = len(accounted_images) # unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1))) # number_of_unaccounted_images = len(unaccounted_images) else: number_of_accounted_images = 0 number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if number_of_accounted_images == 0: os.chdir("..") break program_state_stack.restart_location_title = "restart" if program_state_stack(locals(), getframeinfo(currentframe())): if (myid == main_node): cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation) # reference the original stack list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1)) cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\ stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file)) mpi_barrier(MPI_COMM_WORLD) os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation) program_state_stack.restart_location_title = "candidate_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new) # program_state_stack.restart_location_title = "stopped_program1" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack.restart_location_title = "stop_after_candidates" program_state_stack(locals(), getframeinfo(currentframe())) if stop_after_candidates: mpi_finalize() sys.exit() exit_program = 0 if(myid == main_node): if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation): print "This generation (%d) no class averages were generated!"%isac_generation exit_program = 1 exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if exit_program: os.chdir("..") break program_state_stack.restart_location_title = "reproducible_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new) pass os.chdir("..") if (myid == main_node): cmdexecute("rm -f class_averages.hdf") cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation)], "class_averages.hdf") # program_state_stack.restart_location_title = "stopped_program2" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall") mpi_finalize()
def main(args): progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" + " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" + " --generation=generation --rand_seed=rand_seed>") parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--radius", type="int", help= "particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)" ) parser.add_option( "--target_radius", type="int", default=29, help= "target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)" ) parser.add_option( "--target_nx", type="int", default=76, help= "target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1' (default 76)" ) parser.add_option( "--img_per_grp", type="int", default=100, help= "number of images per class: in the ideal case (essentially maximum size of class) (default 100)" ) parser.add_option( "--CTF", action="store_true", default=False, help= "apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)" ) parser.add_option( "--ir", type="int", default=1, help= "inner ring: of the resampling to polar coordinates. units - pixels (default 1)" ) parser.add_option( "--rs", type="int", default=1, help= "ring step: of the resampling to polar coordinates. units - pixels (default 1)" ) parser.add_option( "--xr", type="int", default=1, help= "x range: of translational search. By default, set by the program. (default 1)" ) parser.add_option( "--yr", type="int", default=-1, help= "y range: of translational search. By default, same as xr. (default -1)" ) parser.add_option( "--ts", type="float", default=1.0, help= "search step: of translational search: units - pixels (default 1.0)") parser.add_option( "--maxit", type="int", default=30, help="number of iterations for reference-free alignment: (default 30)") #parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)") parser.add_option( "--center_method", type="int", default=-1, help= "method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)" ) parser.add_option( "--dst", type="float", default=90.0, help="discrete angle used in within group alignment: (default 90.0)") parser.add_option( "--FL", type="float", default=0.2, help= "lowest stopband: frequency used in the tangent filter (default 0.2)") parser.add_option( "--FH", type="float", default=0.3, help= "highest stopband: frequency used in the tangent filter (default 0.3)") parser.add_option("--FF", type="float", default=0.2, help="fall-off of the tangent filter: (default 0.2)") parser.add_option( "--init_iter", type="int", default=3, help= "SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)" ) parser.add_option( "--main_iter", type="int", default=3, help= "SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)" ) parser.add_option( "--iter_reali", type="int", default=1, help= "SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)" ) parser.add_option( "--match_first", type="int", default=1, help= "number of iterations to run 2-way matching in the first phase: (default 1)" ) parser.add_option( "--max_round", type="int", default=20, help= "maximum rounds: of generating candidate class averages in the first phase (default 20)" ) parser.add_option( "--match_second", type="int", default=5, help= "number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)" ) parser.add_option( "--stab_ali", type="int", default=5, help="number of alignments when checking stability: (default 5)") parser.add_option( "--thld_err", type="float", default=0.7, help= "threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)" ) parser.add_option( "--indep_run", type="int", default=4, help= "level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching. Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)" ) parser.add_option("--thld_grp", type="int", default=10, help="minimum size of reproducible class (default 10)") parser.add_option( "--n_generations", type="int", default=10, help= "maximum number of generations: program stops when reaching this total number of generations: (default 10)" ) #parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)") parser.add_option( "--rand_seed", type="int", help= "random seed set before calculations: useful for testing purposes. By default, total randomness (type int)" ) parser.add_option("--new", action="store_true", default=False, help="use new code: (default False)") parser.add_option("--debug", action="store_true", default=False, help="debug info printout: (default False)") # must be switched off in production parser.add_option( "--use_latest_master_directory", action="store_true", default=False, help= "use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)" ) parser.add_option( "--restart_section", type="string", default=' ', help= "restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')" ) parser.add_option( "--stop_after_candidates", action="store_true", default=False, help= "stop after candidates: stops after the 'candidate_class_averages' section. (default False)" ) ##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help=SUPPRESS_HELP) parser.add_option( "--skip_prealignment", action="store_true", default=False, help= "skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)" ) required_option_list = ['radius'] (options, args) = parser.parse_args(args) if options.return_options: return parser if len(args) > 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" sys.exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() global_def.BATCH = True from isac import iter_isac from fundamentals import rot_shift2D, resample from utilities import pad, combine_params2 command_line_provided_stack_filename = args[0] main_node = 0 mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) nproc = mpi_comm_size(MPI_COMM_WORLD) mpi_barrier(MPI_COMM_WORLD) if (myid == main_node): print "****************************************************************" Util.version() print "****************************************************************" sys.stdout.flush() mpi_barrier(MPI_COMM_WORLD) # Making sure all required options appeared. for required_option in required_option_list: if not options.__dict__[required_option]: print "\n ==%s== mandatory option is missing.\n" % required_option print "Please run '" + progname + " -h' for detailed options" return 1 radi = options.radius target_radius = options.target_radius target_nx = options.target_nx center_method = options.center_method if (radi < 1): ERROR("Particle radius has to be provided!", "sxisac", 1, myid) use_latest_master_directory = options.use_latest_master_directory stop_after_candidates = options.stop_after_candidates # program_state_stack.restart_location_title_from_command_line = options.restart_section from utilities import qw program_state_stack.PROGRAM_STATE_VARIABLES = set( qw(""" isac_generation """)) # create or reuse master directory masterdir = "" stack_processed_by_ali2d_base__filename = "" stack_processed_by_ali2d_base__filename__without_master_dir = "" error_status = 0 if len(args) == 2: masterdir = args[1] elif len(args) == 1: if use_latest_master_directory: all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)] import re r = re.compile("^master.*$") all_dirs = filter(r.match, all_dirs) if len(all_dirs) > 0: # all_dirs = max(all_dirs, key=os.path.getctime) masterdir = max(all_dirs, key=os.path.getmtime) #Create folder for all results or check if there is one created already if (myid == main_node): if (masterdir == ""): timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime()) masterdir = "master" + timestring cmd = "{} {}".format("mkdir", masterdir) junk = cmdexecute(cmd) elif not os.path.exists(masterdir): # os.path.exists(masterdir) does not exist masterdir = args[1] cmd = "{} {}".format("mkdir", masterdir) junk = cmdexecute(cmd) if (args[0][:4] == "bdb:"): filename = args[0][4:] else: filename = args[0][:-4] filename = os.path.basename(filename) stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join( masterdir, filename) stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename if_error_then_all_processes_exit_program(error_status) # send masterdir to all processes masterdir = send_string_to_all(masterdir) if myid == 0: if options.restart_section != " ": if os.path.exists(os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)): stored_stack, stored_state = restore_program_stack_and_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) import re if "," in options.restart_section: parsed_restart_section_option = options.restart_section.split( ",") stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % parsed_restart_section_option[0], stored_state[-1]["location_in_program"]) generation_str_format = parsed_restart_section_option[1] if generation_str_format != "": isac_generation_from_command_line = int( generation_str_format) stored_state[-1][ "isac_generation"] = isac_generation_from_command_line else: isac_generation_from_command_line = 1 if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % options.restart_section, stored_state[-1]["location_in_program"]) if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] store_program_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE), stored_state, stored_stack) else: print "Please remove the restart_section option from the command line. The program must be started from the beginning." mpi_finalize() sys.exit() else: isac_generation_from_command_line = -1 program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) stack_processed_by_ali2d_base__filename = send_string_to_all( stack_processed_by_ali2d_base__filename) stack_processed_by_ali2d_base__filename__without_master_dir = \ send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir) # previous code 2016-05-05--20-14-12-153 # # PARAMETERS OF THE PROCEDURE # if( options.xr == -1 ): # # Default values # # target_nx = 76 # # target_radius = 29 # target_xr = 1 # else: # nx//2 # # Check below! # target_xr = options.xr # # target_nx = 76 + target_xr - 1 # subtract one, which is default # target_nx += target_xr - 1 # subtract one, which is default # # target_radius = 29 target_xr = options.xr target_nx += target_xr - 1 # subtract one, which is default if (options.yr == -1): yr = options.xr else: yr = options.yr mpi_barrier(MPI_COMM_WORLD) # Initialization of stacks if (myid == main_node): print "command_line_provided_stack_filename", command_line_provided_stack_filename number_of_images_in_stack = EMUtil.get_image_count( command_line_provided_stack_filename) else: number_of_images_in_stack = 0 number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node=main_node) nxrsteps = 4 init2dir = os.path.join(masterdir, "2dalignment") # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() if not os.path.exists( os.path.join(init2dir, "Finished_initial_2d_alignment.txt")): if (myid == 0): import subprocess from logger import Logger, BaseLogger_Files # Create output directory log2d = Logger(BaseLogger_Files()) log2d.prefix = os.path.join(init2dir) cmd = "mkdir -p " + log2d.prefix outcome = subprocess.call(cmd, shell=True) log2d.prefix += "/" # outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True) else: outcome = 0 log2d = None if (myid == main_node): a = get_im(command_line_provided_stack_filename) nnxo = a.get_xsize() else: nnxo = 0 nnxo = bcast_number_to_all(nnxo, source_node=main_node) image_start, image_end = MPI_start_end(number_of_images_in_stack, nproc, myid) if options.skip_prealignment: params2d = [[0.0, 0.0, 0.0, 0] for i in xrange(image_start, image_end)] else: original_images = EMData.read_images( command_line_provided_stack_filename, range(image_start, image_end)) # We assume the target radius will be 29, and xr = 1. shrink_ratio = float(target_radius) / float(radi) for im in xrange(len(original_images)): if (shrink_ratio != 1.0): original_images[im] = resample(original_images[im], shrink_ratio) nx = original_images[0].get_xsize() # nx = int(nx*shrink_ratio + 0.5) txrm = (nx - 2 * (target_radius + 1)) // 2 if (txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d" % (radi), "sxisac", 1, myid) if (txrm / nxrsteps > 0): tss = "" txr = "" while (txrm / nxrsteps > 0): tts = txrm / nxrsteps tss += " %d" % tts txr += " %d" % (tts * nxrsteps) txrm = txrm // 2 else: tss = "1" txr = "%d" % txrm # print "nx, txr, txrm, tss", nx, txr, txrm, tss # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() # section ali2d_base params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \ False, 90.0, center_method, 14, options.CTF, 1.0, False, \ "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False) del original_images for i in xrange(len(params2d)): alpha, sx, sy, mirror = combine_params2( 0, params2d[i][1], params2d[i][2], 0, -params2d[i][0], 0, 0, 0) sx /= shrink_ratio sy /= shrink_ratio params2d[i][0] = 0.0 params2d[i][1] = sx params2d[i][2] = sy params2d[i][3] = 0 #set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0]) mpi_barrier(MPI_COMM_WORLD) tmp = params2d[:] tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD) if (myid == main_node): if options.skip_prealignment: print "=========================================" print "Even though there is no alignment step, '%s' params are set to zero for later use." % os.path.join( init2dir, "initial2Dparams.txt") print "=========================================" write_text_row(tmp, os.path.join(init2dir, "initial2Dparams.txt")) del tmp mpi_barrier(MPI_COMM_WORLD) # We assume the target image size will be target_nx, radius will be 29, and xr = 1. # Note images can be also padded, in which case shrink_ratio > 1. shrink_ratio = float(target_radius) / float(radi) aligned_images = EMData.read_images( command_line_provided_stack_filename, range(image_start, image_end)) nx = aligned_images[0].get_xsize() nima = len(aligned_images) newx = int(nx * shrink_ratio + 0.5) while not os.path.exists(os.path.join(init2dir, "initial2Dparams.txt")): import time time.sleep(1) mpi_barrier(MPI_COMM_WORLD) params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt")) params = params[image_start:image_end] msk = model_circle(radi, nx, nx) for im in xrange(nima): st = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= st[0] if options.CTF: aligned_images[im] = filt_ctf( aligned_images[im], aligned_images[im].get_attr("ctf"), binary=True) if (shrink_ratio < 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) #aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(newx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio == 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(newx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) #aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio > 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(newx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts #alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) #alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) del msk gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid) number_of_images_in_stack = bcast_number_to_all( number_of_images_in_stack, source_node=main_node) if (myid == main_node): for i in range(number_of_images_in_stack): aligned_images[i].write_image( stack_processed_by_ali2d_base__filename, i) # It has to be explicitly closed from EMAN2db import db_open_dict DB = db_open_dict(stack_processed_by_ali2d_base__filename) DB.close() fp = open(os.path.join(masterdir, "README_shrink_ratio.txt"), "w") output_text = """ Since, for processing purposes, isac changes the image dimensions, adjustment of pixel size needs to be made in subsequent steps, (e.g. running sxviper.py). The shrink ratio for this particular isac run is -------- %.5f %.5f -------- To get the pixel size for the isac output the user needs to divide the original pixel size by the above value. This info is saved in the following file: README_shrink_ratio.txt """ % (shrink_ratio, radi) fp.write(output_text) fp.flush() fp.close() print output_text fp = open( os.path.join(init2dir, "Finished_initial_2d_alignment.txt"), "w") fp.flush() fp.close() else: if (myid == main_node): print "Skipping 2d alignment since it was already done!" mpi_barrier(MPI_COMM_WORLD) # from mpi import mpi_finalize # mpi_finalize() # sys.stdout.flush() # sys.exit() os.chdir(masterdir) if program_state_stack(locals(), getframeinfo(currentframe())): # if 1: pass if (myid == main_node): junk = cmdexecute( "sxheader.py --consecutive --params=originalid %s" % stack_processed_by_ali2d_base__filename__without_master_dir) junk = cmdexecute( "e2bdb.py %s --makevstack=%s_000" % (stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir)) if (myid == main_node): main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d") print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no if isac_generation_from_command_line < 0: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = -1 if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: for i in xrange(isac_generation_from_command_line + 1, main_dir_no + 1): if i == isac_generation_from_command_line + 1: backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) junk = cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i + " 000_backup" + "%05d" % backup_dir_no) junk = cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % i) # it includes both command line and json file my_restart_section = stored_state[-1]["location_in_program"].split( "___")[-1] if "restart" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) junk = cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) junk = cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % isac_generation_from_command_line) elif "candidate_class_averages" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) junk = cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line) # junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line) elif "reproducible_class_averages" in my_restart_section: junk = cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*" % isac_generation_from_command_line) junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*" % isac_generation_from_command_line) else: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 0 isac_generation_from_command_line = mpi_bcast( isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0] isac_generation = isac_generation_from_command_line - 1 if (myid == main_node): if isac_generation == 0: junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) write_text_file( [1], os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_accounted.txt" % isac_generation)) write_text_file( range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_unaccounted.txt" % isac_generation)) # Stopping criterion should be inside the program. while True: isac_generation += 1 if isac_generation > options.n_generations: break data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d" % isac_generation program_state_stack.restart_location_title = "restart" if program_state_stack(locals(), getframeinfo(currentframe())): if (myid == main_node): junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) # reference the original stack list_file = os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1), "generation_%d_unaccounted.txt" % (isac_generation - 1)) junk = cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\ stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file)) mpi_barrier(MPI_COMM_WORLD) os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation) program_state_stack.restart_location_title = "candidate_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new) # program_state_stack.restart_location_title = "stopped_program1" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack.restart_location_title = "stop_after_candidates" program_state_stack(locals(), getframeinfo(currentframe())) if stop_after_candidates: mpi_finalize() sys.exit() exit_program = 0 if (myid == main_node): if not os.path.exists( "class_averages_candidate_generation_%d.hdf" % isac_generation): print "This generation (%d) no class average candidates were generated! Finishing." % isac_generation exit_program = 1 exit_program = int( mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if exit_program: os.chdir("..") break program_state_stack.restart_location_title = "reproducible_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new) pass os.chdir("..") if (myid == main_node): accounted_images = read_text_file( os.path.join(NAME_OF_MAIN_DIR + "%04d" % (isac_generation), "generation_%d_accounted.txt" % (isac_generation))) number_of_accounted_images = len(accounted_images) un_accounted_images = read_text_file( os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation), "generation_%d_unaccounted.txt" % (isac_generation))) number_of_un_accounted_images = len(un_accounted_images) else: number_of_accounted_images = 0 number_of_un_accounted_images = 0 number_of_accounted_images = int( mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) number_of_un_accounted_images = int( mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if number_of_accounted_images == 0: if (myid == main_node): print "This generation (%d) there are no accounted images! Finishing." % isac_generation break while (myid == main_node): def files_are_missing(isac_generation): for i in xrange(1, isac_generation + 1): if not os.path.exists( "generation_%04d/class_averages_generation_%d.hdf" % (i, i)): print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting." % ( i, i) return 1 return 0 if files_are_missing(isac_generation): break junk = cmdexecute("rm -f class_averages.hdf") cpy([ "generation_%04d/class_averages_generation_%d.hdf" % (i, i) for i in xrange(1, isac_generation + 1) ], "class_averages.hdf") break if number_of_un_accounted_images == 0: if (myid == main_node): print "This generation (%d) there are no un accounted images! Finishing." % isac_generation break program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall") mpi_barrier(MPI_COMM_WORLD) mpi_finalize()
def main(args): progname = os.path.basename(sys.argv[0]) usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" + " --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" + " --generation=generation --rand_seed=rand_seed>") parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--radius", type="int", default=-1, help="<Particle radius>, it has to be provided.") parser.add_option( "--img_per_grp", type="int", default=100, help= "<number of images per group> in the ideal case (essentially maximum size of class) (100)" ) parser.add_option("--CTF", action="store_true", default=False, help="<CTF flag>, if set the data will be phase-flipped") parser.add_option( "--ir", type="int", default=1, help="<inner ring> of the resampling to polar coordinates (1)") parser.add_option( "--rs", type="int", default=1, help="<ring step> of the resampling to polar coordinates (1)") parser.add_option( "--xr", type="int", default=-1, help= "<x range> of translational search (By default set by the program) (advanced)" ) parser.add_option( "--yr", type="int", default=-1, help="<y range> of translational search (same as xr) (advanced)") parser.add_option("--ts", type="float", default=1.0, help="<search step> of translational search (1.0)") parser.add_option( "--maxit", type="int", default=30, help="number of iterations for reference-free alignment (30)") #parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)") parser.add_option( "--center_method", type="int", default=7, help= "<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)" ) parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment ") parser.add_option( "--FL", type="float", default=0.2, help="<lowest stopband> frequency used in the tangent filter (0.2)") parser.add_option( "--FH", type="float", default=0.3, help="<highest stopband> frequency used in the tangent filter (0.3)") parser.add_option("--FF", type="float", default=0.2, help="<fall-off of the tangent> filter (0.2)") parser.add_option( "--init_iter", type="int", default=3, help= "<init_iter> number of iterations of ISAC program in initialization (3)" ) parser.add_option( "--main_iter", type="int", default=3, help="<main_iter> number of iterations of ISAC program in main part (3)" ) parser.add_option( "--iter_reali", type="int", default=1, help= "<iter_reali> number of iterations in ISAC before checking stability (1)" ) parser.add_option( "--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase (1)" ) parser.add_option( "--max_round", type="int", default=20, help= "maximum rounds of generating candidate averages in the first phase (20)" ) parser.add_option( "--match_second", type="int", default=5, help= "number of iterations to run 2-way (or 3-way) matching in the second phase (5)" ) parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability (5)") parser.add_option( "--thld_err", type="float", default=0.7, help="the threshold of pixel error when checking stability (0.7)") parser.add_option( "--indep_run", type="int", default=4, help= "number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)" ) parser.add_option("--thld_grp", type="int", default=10, help="minimum size of class (10)") parser.add_option( "--n_generations", type="int", default=100, help= "<n_generations> program stops when reaching this total number of generations (advanced)" ) #parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)") parser.add_option( "--rand_seed", type="int", default=None, help= "random seed set before calculations, useful for testing purposes (default None - total randomness)" ) parser.add_option("--new", action="store_true", default=False, help="use new code (default = False)") parser.add_option("--debug", action="store_true", default=False, help="debug info printout (default = False)") # must be switched off in production parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False) parser.add_option( "--restart_section", type="string", default="", help= "<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)" ) parser.add_option( "--stop_after_candidates", action="store_true", default=False, help= "<stop_after_candidates> stops after the 'candidate_class_averages' section" ) parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help=SUPPRESS_HELP) (options, args) = parser.parse_args(args) if options.return_options: return parser if len(args) > 2: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" sys.exit() if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() from isac import iter_isac global_def.BATCH = True global_def.BATCH = True command_line_provided_stack_filename = args[0] global_def.BATCH = True main_node = 0 mpi_init(0, []) myid = mpi_comm_rank(MPI_COMM_WORLD) nproc = mpi_comm_size(MPI_COMM_WORLD) radi = options.radius center_method = options.center_method if (radi < 1): ERROR("Particle radius has to be provided!", "sxisac", 1, myid) use_latest_master_directory = options.use_latest_master_directory stop_after_candidates = options.stop_after_candidates program_state_stack.restart_location_title_from_command_line = options.restart_section from utilities import qw program_state_stack.PROGRAM_STATE_VARIABLES = set( qw(""" isac_generation """)) # create or reuse master directory masterdir = "" stack_processed_by_ali2d_base__filename = "" stack_processed_by_ali2d_base__filename__without_master_dir = "" error_status = 0 if len(args) == 2: masterdir = args[1] elif len(args) == 1: if use_latest_master_directory: all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)] import re r = re.compile("^master.*$") all_dirs = filter(r.match, all_dirs) if len(all_dirs) > 0: # all_dirs = max(all_dirs, key=os.path.getctime) masterdir = max(all_dirs, key=os.path.getmtime) #Create folder for all results or check if there is one created already if (myid == main_node): if (masterdir == ""): timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime()) masterdir = "master" + timestring cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) elif not os.path.exists(masterdir): # os.path.exists(masterdir) does not exist masterdir = args[1] cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) if (args[0][:4] == "bdb:"): filename = args[0][4:] else: filename = args[0][:-4] filename = os.path.basename(filename) stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join( masterdir, filename) stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename if_error_all_processes_quit_program(error_status) # send masterdir to all processes masterdir = send_string_to_all(masterdir) if myid == 0: if options.restart_section != "": if os.path.exists(os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)): stored_stack, stored_state = restore_program_stack_and_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) import re if "," in options.restart_section: parsed_restart_section_option = options.restart_section.split( ",") stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % parsed_restart_section_option[0], stored_state[-1]["location_in_program"]) generation_str_format = parsed_restart_section_option[1] if generation_str_format != "": isac_generation_from_command_line = int( generation_str_format) stored_state[-1][ "isac_generation"] = isac_generation_from_command_line else: isac_generation_from_command_line = 1 if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] else: isac_generation_from_command_line = -1 stored_state[-1]["location_in_program"] = re.sub( r"___.*$", "___%s" % options.restart_section, stored_state[-1]["location_in_program"]) if "isac_generation" in stored_state[-1]: del stored_state[-1]["isac_generation"] store_program_state( os.path.join(masterdir, NAME_OF_JSON_STATE_FILE), stored_state, stored_stack) else: print "Please remove the restart_section option from the command line. The program must be started from the beginning." mpi_finalize() sys.exit() else: isac_generation_from_command_line = -1 program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir, NAME_OF_JSON_STATE_FILE)) stack_processed_by_ali2d_base__filename = send_string_to_all( stack_processed_by_ali2d_base__filename) stack_processed_by_ali2d_base__filename__without_master_dir = \ send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir) # PARAMETERS OF THE PROCEDURE if (options.xr == -1): # Default values target_nx = 76 target_radius = 29 target_xr = 1 else: # nx//2 # Check below! target_xr = options.xr target_nx = 76 + target_xr - 1 # subtract one, which is default target_radius = 29 mpi_barrier(MPI_COMM_WORLD) # Initialization of stacks if (myid == main_node): number_of_images_in_stack = EMUtil.get_image_count( command_line_provided_stack_filename) else: number_of_images_in_stack = 0 number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node=main_node) nxrsteps = 4 init2dir = os.path.join(masterdir, "2dalignment") if (myid == 0): import subprocess from logger import Logger, BaseLogger_Files # Create output directory log2d = Logger(BaseLogger_Files()) log2d.prefix = os.path.join(init2dir) cmd = "mkdir -p " + log2d.prefix outcome = subprocess.call(cmd, shell=True) log2d.prefix += "/" # outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True) else: outcome = 0 log2d = None if (myid == main_node): a = get_im(command_line_provided_stack_filename) nnxo = a.get_xsize() else: nnxo = 0 nnxo = bcast_number_to_all(nnxo, source_node=main_node) txrm = (nnxo - 2 * (radi + 1)) // 2 if (txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d" % (radi), "sxisac", 1, myid) if (txrm / nxrsteps > 0): tss = "" txr = "" while (txrm / nxrsteps > 0): tts = txrm / nxrsteps tss += " %d" % tts txr += " %d" % (tts * nxrsteps) txrm = txrm // 2 else: tss = "1" txr = "%d" % txrm # section ali2d_base # centering method is set to #7 params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \ False, 90.0, center_method, 14, options.CTF, 1.0, False, \ "ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False) if (myid == main_node): write_text_row(params2d, os.path.join(init2dir, "initial2Dparams.txt")) del params2d mpi_barrier(MPI_COMM_WORLD) # We assume the target image size will be target_nx, radius will be 29, and xr = 1. # Note images can be also padded, in which case shrink_ratio > 1. shrink_ratio = float(target_radius) / float(radi) nx = aligned_images[0].get_xsize() nima = len(aligned_images) newx = int(nx * shrink_ratio + 0.5) from fundamentals import rot_shift2D, resample from utilities import pad, combine_params2 if (shrink_ratio < 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(nx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio == 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(nx // 2 - 2, newx, newx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = resample(aligned_images[im], shrink_ratio) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) elif (shrink_ratio > 1.0): if newx > target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx == target_nx: msk = model_circle(target_radius, target_nx, target_nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] elif newx < target_nx: msk = model_circle(target_radius, nx, nx) for im in xrange(nima): # Here we should use only shifts alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im]) alpha, sx, sy, mirror = combine_params2( 0, sx, sy, 0, -alpha, 0, 0, 0) aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0) p = Util.infomask(aligned_images[im], msk, False) aligned_images[im] -= p[0] p = Util.infomask(aligned_images[im], msk, True) aligned_images[im] /= p[1] aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0) del msk gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid) number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node=main_node) if (myid == main_node): for i in range(number_of_images_in_stack): aligned_images[i].write_image( stack_processed_by_ali2d_base__filename, i) # It has to be explicitly closed from EMAN2db import db_open_dict DB = db_open_dict(stack_processed_by_ali2d_base__filename) DB.close() mpi_barrier(MPI_COMM_WORLD) global_def.BATCH = True os.chdir(masterdir) if program_state_stack(locals(), getframeinfo(currentframe())): # if 1: pass if (myid == main_node): cmdexecute( "sxheader.py --consecutive --params=originalid %s" % stack_processed_by_ali2d_base__filename__without_master_dir) cmdexecute( "e2bdb.py %s --makevstack=%s_000" % (stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir)) if (myid == main_node): main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d") print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no if isac_generation_from_command_line < 0: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = -1 if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no: for i in xrange(isac_generation_from_command_line + 1, main_dir_no + 1): if i == isac_generation_from_command_line + 1: backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i + " 000_backup" + "%05d" % backup_dir_no) cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % i) # it includes both command line and json file my_restart_section = stored_state[-1]["location_in_program"].split( "___")[-1] if "restart" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) cmdexecute( "rm " + "EMAN2DB/" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d.bdb" % isac_generation_from_command_line) elif "candidate_class_averages" in my_restart_section: if "backup_dir_no" not in locals(): backup_dir_no = get_nonexistent_directory_increment_value( "./", "000_backup", myformat="%05d", start_value=1) cmdexecute("mkdir -p " + "000_backup" + "%05d" % backup_dir_no) cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line + " 000_backup" + "%05d" % backup_dir_no) cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation_from_command_line) # cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line) elif "reproducible_class_averages" in my_restart_section: cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*" % isac_generation_from_command_line) cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*" % isac_generation_from_command_line) else: if os.path.exists(NAME_OF_JSON_STATE_FILE): stored_stack, stored_state = restore_program_stack_and_state( NAME_OF_JSON_STATE_FILE) if "isac_generation" in stored_state[-1]: isac_generation_from_command_line = stored_state[-1][ "isac_generation"] else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 1 else: isac_generation_from_command_line = 0 isac_generation_from_command_line = mpi_bcast( isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0] isac_generation = isac_generation_from_command_line - 1 if (myid == main_node): if isac_generation == 0: cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) write_text_file( [1], os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_accounted.txt" % isac_generation)) write_text_file( range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation, "generation_%d_unaccounted.txt" % isac_generation)) # Stopping criterion should be inside the program. while True: isac_generation += 1 if isac_generation > options.n_generations: break data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[ 4:] + "_%03d" % isac_generation if (myid == main_node): accounted_images = read_text_file( os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1), "generation_%d_accounted.txt" % (isac_generation - 1))) number_of_accounted_images = len(accounted_images) # unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1))) # number_of_unaccounted_images = len(unaccounted_images) else: number_of_accounted_images = 0 number_of_accounted_images = int( mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if number_of_accounted_images == 0: os.chdir("..") break program_state_stack.restart_location_title = "restart" if program_state_stack(locals(), getframeinfo(currentframe())): if (myid == main_node): cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d" % isac_generation) # reference the original stack list_file = os.path.join( NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1), "generation_%d_unaccounted.txt" % (isac_generation - 1)) cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\ stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file)) mpi_barrier(MPI_COMM_WORLD) os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation) program_state_stack.restart_location_title = "candidate_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new) # program_state_stack.restart_location_title = "stopped_program1" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack.restart_location_title = "stop_after_candidates" program_state_stack(locals(), getframeinfo(currentframe())) if stop_after_candidates: mpi_finalize() sys.exit() exit_program = 0 if (myid == main_node): if not os.path.exists( "class_averages_candidate_generation_%d.hdf" % isac_generation): print "This generation (%d) no class averages were generated!" % isac_generation exit_program = 1 exit_program = int( mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]) if exit_program: os.chdir("..") break program_state_stack.restart_location_title = "reproducible_class_averages" if program_state_stack(locals(), getframeinfo(currentframe())): iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\ options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \ options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \ options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new) pass os.chdir("..") if (myid == main_node): cmdexecute("rm -f class_averages.hdf") cpy([ "generation_%04d/class_averages_generation_%d.hdf" % (i, i) for i in xrange(1, isac_generation) ], "class_averages.hdf") # program_state_stack.restart_location_title = "stopped_program2" # program_state_stack(locals(), getframeinfo(currentframe())) program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall") mpi_finalize()
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Generic 2-D image processing programs. Functionality: 1. Phase flip a stack of images and write output to new file: sxprocess.py input_stack.hdf output_stack.hdf --phase_flip 2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size. The window size will change accordingly. sxprocess input.hdf output.hdf --changesize --ratio=0.5 3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes. sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024] 4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs: sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64 5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3): sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid 6. Retrieve original image numbers of images listed in ISAC output stack of averages: sxprocess.py select1.hdf ohk.txt 7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file. Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack. sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw 8. Generate a 1D rotationally averaged power spectrum of an image. sxprocess.py vol.hdf --rotwp=rotpw.txt # Output will contain three columns: (1) rotationally averaged power spectrum (2) logarithm of the rotationally averaged power spectrum (3) integer line number (from zero to approximately to half the image size) 9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data. sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt The output file is then imported and 3D transformed volume computed: sxheader.py bdb:p --params=xform.projection --import=output.txt mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI The reconstructed volume is in the position of the volume computed using the input.txt parameters and then transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz) 10. Import ctf parameters from the output of sxcter into windowed particle headers. There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph. In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'. Normally this is done by e2boxer.py during windowing. Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created. sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 # Output will be a vritual stack bdb:Gdata sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 To concatenate output files: cd directory e2bdb.py . --makevstack=bdb:allparticles --filt=G IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them. 11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor sxprocess.py orientationparams.txt scaledparams.txt scale=0.5 12. Generate adaptive mask from a given 3-D volume. """ parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--order", action="store_true", help= "Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False) parser.add_option("--order_lookup", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_metropolis", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_pca", action="store_true", help="Test/Debug.", default=False) parser.add_option( "--initial", type="int", default=-1, help= "Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)" ) parser.add_option( "--circular", action="store_true", help= "Select circular ordering (fisr image has to be similar to the last", default=False) parser.add_option( "--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") parser.add_option( "--changesize", action="store_true", help= "resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False) parser.add_option( "--ratio", type="float", default=1.0, help= "The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round" ) parser.add_option( "--pw", action="store_true", help= "compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False) parser.add_option( "--wn", type="int", default=-1, help= "Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))" ) parser.add_option("--phase_flip", action="store_true", help="Phase flip the input stack", default=False) parser.add_option( "--makedb", metavar="param1=value1:param2=value2", type="string", action="append", help= "One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0" ) parser.add_option( "--generate_projections", metavar="param1=value1:param2=value2", type="string", action="append", help= "Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64." ) parser.add_option( "--isacgroup", type="int", help= "Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1) parser.add_option( "--isacselect", action="store_true", help= "Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False) parser.add_option( "--params", type="string", default=None, help="Name of header of parameter, which one depends on specific option" ) parser.add_option( "--adjpw", action="store_true", help="Adjust rotationally averaged power spectrum of an image", default=False) parser.add_option( "--rotpw", type="string", default=None, help= "Name of the text file to contain rotationally averaged power spectrum of the input image." ) parser.add_option( "--transformparams", type="string", default=None, help= "Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters." ) # import ctf estimates done using cter parser.add_option("--input", type="string", default=None, help="Input particles.") parser.add_option( "--importctf", type="string", default=None, help="Name of the file containing CTF parameters produced by sxcter.") parser.add_option( "--defocuserror", type="float", default=1000000.0, help= "Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%" ) parser.add_option( "--astigmatismerror", type="float", default=360.0, help= "Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees." ) # import ctf estimates done using cter parser.add_option( "--scale", type="float", default=-1.0, help= "Divide shifts in the input 3D orientation parameters text file by the scale factor." ) # generate adaptive mask from an given 3-Db volue parser.add_option("--adaptive_mask", action="store_true", help="create adavptive 3-D mask from a given volume", default=False) parser.add_option( "--nsigma", type="float", default=1., help= "number of times of sigma of the input volume to obtain the the large density cluster" ) parser.add_option( "--ndilation", type="int", default=3, help= "number of times of dilation applied to the largest cluster of density" ) parser.add_option( "--kernel_size", type="int", default=11, help="convolution kernel for smoothing the edge of the mask") parser.add_option( "--gauss_standard_dev", type="int", default=9, help="stanadard deviation value to generate Gaussian edge") (options, args) = parser.parse_args() global_def.BATCH = True if options.phase_flip: nargs = len(args) if nargs != 2: print "must provide name of input and output file!" return from EMAN2 import Processor instack = args[0] outstack = args[1] nima = EMUtil.get_image_count(instack) from filter import filt_ctf for i in xrange(nima): img = EMData() img.read_image(instack, i) try: ctf = img.get_attr('ctf') except: print "no ctf information in input stack! Exiting..." return dopad = True sign = 1 binary = 1 # phase flip assert img.get_ysize() > 1 dict = ctf.to_dict() dz = dict["defocus"] cs = dict["cs"] voltage = dict["voltage"] pixel_size = dict["apix"] b_factor = dict["bfactor"] ampcont = dict["ampcont"] dza = dict["dfdiff"] azz = dict["dfang"] if dopad and not img.is_complex(): ip = 1 else: ip = 0 params = { "filter_type": Processor.fourier_filter_types.CTF_, "defocus": dz, "Cs": cs, "voltage": voltage, "Pixel_size": pixel_size, "B_factor": b_factor, "amp_contrast": ampcont, "dopad": ip, "binary": binary, "sign": sign, "dza": dza, "azz": azz } tmp = Processor.EMFourierFilter(img, params) tmp.set_attr_dict({"ctf": ctf}) tmp.write_image(outstack, i) elif options.changesize: nargs = len(args) if nargs != 2: ERROR("must provide name of input and output file!", "change size", 1) return from utilities import get_im instack = args[0] outstack = args[1] sub_rate = float(options.ratio) nima = EMUtil.get_image_count(instack) from fundamentals import resample for i in xrange(nima): resample(get_im(instack, i), sub_rate).write_image(outstack, i) elif options.isacgroup > -1: nargs = len(args) if nargs != 3: ERROR("Three files needed on input!", "isacgroup", 1) return from utilities import get_im instack = args[0] m = get_im(args[1], int(options.isacgroup)).get_attr("members") l = [] for k in m: l.append(int(get_im(args[0], k).get_attr(options.params))) from utilities import write_text_file write_text_file(l, args[2]) elif options.isacselect: nargs = len(args) if nargs != 2: ERROR("Two files needed on input!", "isacgroup", 1) return from utilities import get_im nima = EMUtil.get_image_count(args[0]) m = [] for k in xrange(nima): m += get_im(args[0], k).get_attr("members") m.sort() from utilities import write_text_file write_text_file(m, args[1]) elif options.pw: nargs = len(args) if nargs < 2: ERROR("must provide name of input and output file!", "pw", 1) return from utilities import get_im d = get_im(args[0]) nx = d.get_xsize() ny = d.get_ysize() if nargs == 3: mask = get_im(args[2]) wn = int(options.wn) if wn == -1: wn = max(nx, ny) else: if ((wn < nx) or (wn < ny)): ERROR("window size cannot be smaller than the image size", "pw", 1) n = EMUtil.get_image_count(args[0]) from utilities import model_blank, model_circle, pad from EMAN2 import periodogram p = model_blank(wn, wn) for i in xrange(n): d = get_im(args[0], i) if nargs == 3: d *= mask st = Util.infomask(d, None, True) d -= st[0] p += periodogram(pad(d, wn, wn, 1, 0.)) p /= n p.write_image(args[1]) elif options.adjpw: if len(args) < 3: ERROR( "filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)", "adjpw", 1) return img_stack = args[0] from math import sqrt from fundamentals import rops_table, fft from utilities import read_text_file, get_im from filter import filt_tanl, filt_table if (args[1][-3:] == 'txt'): rops_dst = read_text_file(args[1]) else: rops_dst = rops_table(get_im(args[1])) out_stack = args[2] if (len(args) > 4): fl = float(args[3]) aa = float(args[4]) else: fl = -1.0 aa = 0.0 nimage = EMUtil.get_image_count(img_stack) for i in xrange(nimage): img = fft(get_im(img_stack, i)) rops_src = rops_table(img) assert len(rops_dst) == len(rops_src) table = [0.0] * len(rops_dst) for j in xrange(len(rops_dst)): table[j] = sqrt(rops_dst[j] / rops_src[j]) if (fl > 0.0): img = filt_tanl(img, fl, aa) img = fft(filt_table(img, table)) img.write_image(out_stack, i) elif options.rotpw != None: if len(args) != 1: ERROR("Only one input permitted", "rotpw", 1) return from utilities import write_text_file, get_im from fundamentals import rops_table from math import log10 t = rops_table(get_im(args[0])) x = range(len(t)) r = [0.0] * len(x) for i in x: r[i] = log10(t[i]) write_text_file([t, r, x], options.rotpw) elif options.transformparams != None: if len(args) != 2: ERROR( "Please provide names of input and output files with orientation parameters", "transformparams", 1) return from utilities import read_text_row, write_text_row transf = [0.0] * 6 spl = options.transformparams.split(',') for i in xrange(len(spl)): transf[i] = float(spl[i]) write_text_row(rotate_shift_params(read_text_row(args[0]), transf), args[1]) elif options.makedb != None: nargs = len(args) if nargs != 1: print "must provide exactly one argument denoting database key under which the input params will be stored" return dbkey = args[0] print "database key under which params will be stored: ", dbkey gbdb = js_open_dict("e2boxercache/gauss_box_DB.json") parmstr = 'dummy:' + options.makedb[0] (processorname, param_dict) = parsemodopt(parmstr) dbdict = {} for pkey in param_dict: if (pkey == 'invert_contrast') or (pkey == 'use_variance'): if param_dict[pkey] == 'True': dbdict[pkey] = True else: dbdict[pkey] = False else: dbdict[pkey] = param_dict[pkey] gbdb[dbkey] = dbdict elif options.generate_projections: nargs = len(args) if nargs != 3: ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\ "sxprocess - generate projections",1) return inpstr = args[0] outstk = args[1] micpref = args[2] parmstr = 'dummy:' + options.generate_projections[0] (processorname, param_dict) = parsemodopt(parmstr) parm_CTF = False parm_format = 'bdb' parm_apix = 2.5 if 'CTF' in param_dict: if param_dict['CTF'] == 'True': parm_CTF = True if 'format' in param_dict: parm_format = param_dict['format'] if 'apix' in param_dict: parm_apix = float(param_dict['apix']) boxsize = 64 if 'boxsize' in param_dict: boxsize = int(param_dict['boxsize']) print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize scale_mult = 2500 sigma_add = 1.5 sigma_proj = 30.0 sigma2_proj = 17.5 sigma_gauss = 0.3 sigma_mic = 30.0 sigma2_mic = 17.5 sigma_gauss_mic = 0.3 if 'scale_mult' in param_dict: scale_mult = float(param_dict['scale_mult']) if 'sigma_add' in param_dict: sigma_add = float(param_dict['sigma_add']) if 'sigma_proj' in param_dict: sigma_proj = float(param_dict['sigma_proj']) if 'sigma2_proj' in param_dict: sigma2_proj = float(param_dict['sigma2_proj']) if 'sigma_gauss' in param_dict: sigma_gauss = float(param_dict['sigma_gauss']) if 'sigma_mic' in param_dict: sigma_mic = float(param_dict['sigma_mic']) if 'sigma2_mic' in param_dict: sigma2_mic = float(param_dict['sigma2_mic']) if 'sigma_gauss_mic' in param_dict: sigma_gauss_mic = float(param_dict['sigma_gauss_mic']) from filter import filt_gaussl, filt_ctf from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank, pad, model_gauss_noise, set_params2D, set_params_proj from projection import prep_vol, prgs seed(14567) delta = 29 angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S") nangle = len(angles) modelvol = [] nvlms = EMUtil.get_image_count(inpstr) from utilities import get_im for k in xrange(nvlms): modelvol.append(get_im(inpstr, k)) nx = modelvol[0].get_xsize() if nx != boxsize: ERROR("Requested box dimension does not match dimension of the input model.", \ "sxprocess - generate projections",1) nvol = 10 volfts = [[] for k in xrange(nvlms)] for k in xrange(nvlms): for i in xrange(nvol): sigma = sigma_add + random() # 1.5-2.5 addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40) scale = scale_mult * (0.5 + random()) vf, kb = prep_vol(modelvol[k] + scale * addon) volfts[k].append(vf) del vf, modelvol if parm_format == "bdb": stack_data = "bdb:" + outstk delete_bdb(stack_data) else: stack_data = outstk + ".hdf" Cs = 2.0 pixel = parm_apix voltage = 120.0 ampcont = 10.0 ibd = 4096 / 2 - boxsize iprj = 0 width = 240 xstart = 8 + boxsize / 2 ystart = 8 + boxsize / 2 rowlen = 17 from random import randint params = [] for idef in xrange(3, 8): irow = 0 icol = 0 mic = model_blank(4096, 4096) defocus = idef * 0.5 #0.2 if parm_CTF: astampl = defocus * 0.15 astangl = 50.0 ctf = generate_ctf([ defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl ]) for i in xrange(nangle): for k in xrange(12): dphi = 8.0 * (random() - 0.5) dtht = 8.0 * (random() - 0.5) psi = 360.0 * random() phi = angles[i][0] + dphi tht = angles[i][1] + dtht s2x = 4.0 * (random() - 0.5) s2y = 4.0 * (random() - 0.5) params.append([phi, tht, psi, s2x, s2y]) ivol = iprj % nvol #imgsrc = randint(0,nvlms-1) imgsrc = iprj % nvlms proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y]) x = xstart + irow * width y = ystart + icol * width mic += pad(proj, 4096, 4096, 1, 0.0, x - 2048, y - 2048, 0) proj = proj + model_gauss_noise(sigma_proj, nx, nx) if parm_CTF: proj = filt_ctf(proj, ctf) proj.set_attr_dict({"ctf": ctf, "ctf_applied": 0}) proj = proj + filt_gaussl( model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss) proj.set_attr("origimgsrc", imgsrc) proj.set_attr("test_id", iprj) # flags describing the status of the image (1 = true, 0 = false) set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0]) set_params_proj(proj, [phi, tht, psi, s2x, s2y]) proj.write_image(stack_data, iprj) icol += 1 if icol == rowlen: icol = 0 irow += 1 iprj += 1 mic += model_gauss_noise(sigma_mic, 4096, 4096) if parm_CTF: #apply CTF mic = filt_ctf(mic, ctf) mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic) mic.write_image(micpref + "%1d.hdf" % (idef - 3), 0) drop_spider_doc("params.txt", params) elif options.importctf != None: print ' IMPORTCTF ' from utilities import read_text_row, write_text_row from random import randint import subprocess grpfile = 'groupid%04d' % randint(1000, 9999) ctfpfile = 'ctfpfile%04d' % randint(1000, 9999) cterr = [options.defocuserror / 100.0, options.astigmatismerror] ctfs = read_text_row(options.importctf) for kk in xrange(len(ctfs)): root, name = os.path.split(ctfs[kk][-1]) ctfs[kk][-1] = name[:-4] if (options.input[:4] != 'bdb:'): ERROR('Sorry, only bdb files implemented', 'importctf', 1) d = options.input[4:] #try: str = d.index('*') #except: str = -1 from string import split import glob uu = os.path.split(d) uu = os.path.join(uu[0], 'EMAN2DB', uu[1] + '.bdb') flist = glob.glob(uu) for i in xrange(len(flist)): root, name = os.path.split(flist[i]) root = root[:-7] name = name[:-4] fil = 'bdb:' + os.path.join(root, name) sourcemic = EMUtil.get_all_attributes(fil, 'ptcl_source_image') nn = len(sourcemic) gctfp = [] groupid = [] for kk in xrange(nn): junk, name2 = os.path.split(sourcemic[kk]) name2 = name2[:-4] ctfp = [-1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] for ll in xrange(len(ctfs)): if (name2 == ctfs[ll][-1]): # found correct if (ctfs[ll][8] / ctfs[ll][0] <= cterr[0]): # acceptable defocus error ctfp = ctfs[ll][:8] if (ctfs[ll][10] > cterr[1]): # error of astigmatism exceed the threshold, set astigmatism to zero. ctfp[6] = 0.0 ctfp[7] = 0.0 gctfp.append(ctfp) groupid.append(kk) break if (len(groupid) > 0): write_text_row(groupid, grpfile) write_text_row(gctfp, ctfpfile) cmd = "{} {} {} {}".format( 'e2bdb.py', fil, '--makevstack=bdb:' + root + 'G' + name, '--list=' + grpfile) #print cmd subprocess.call(cmd, shell=True) cmd = "{} {} {} {}".format('sxheader.py', 'bdb:' + root + 'G' + name, '--params=ctf', '--import=' + ctfpfile) #print cmd subprocess.call(cmd, shell=True) else: print ' >>> Group ', name, ' skipped.' cmd = "{} {} {}".format("rm -f", grpfile, ctfpfile) subprocess.call(cmd, shell=True) elif options.scale > 0.0: from utilities import read_text_row, write_text_row scale = options.scale nargs = len(args) if nargs != 2: print "Please provide names of input and output file!" return p = read_text_row(args[0]) for i in xrange(len(p)): p[i][3] /= scale p[i][4] /= scale write_text_row(p, args[1]) elif options.adaptive_mask: from utilities import get_im from morphology import adaptive_mask nsigma = options.nsigma ndilation = options.ndilation kernel_size = options.kernel_size gauss_standard_dev = options.gauss_standard_dev nargs = len(args) if nargs > 2: print "Too many inputs are given, try again!" return else: inputvol = get_im(args[0]) input_path, input_file_name = os.path.split(args[0]) input_file_name_root, ext = os.path.splitext(input_file_name) if nargs == 2: mask_file_name = args[1] else: mask_file_name = "adaptive_mask_for" + input_file_name_root + ".hdf" # Only hdf file is output. mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev) mask3d.write_image(mask_file_name) else: ERROR("Please provide option name", "sxprocess.py", 1)