def sanity_checks(command_args, input_vol): if os.path.isfile( os.path.join(command_args.output_dir, command_args.prefix + '_mask.hdf')): if not command_args.overwrite: ERROR( "Output mask already exists! Please provide the overwrite option if you want to overwrite the existing mask." ) if command_args.s_nx is not None and command_args.s_ny is None and command_args.s_ny is None and command_args.second_mask_shape: command_args.s_ny = command_args.s_nx command_args.s_nz = command_args.s_nx elif command_args.s_nx is not None and command_args.s_ny is not None and command_args.s_ny is not None and command_args.second_mask_shape: pass elif command_args.second_mask_shape: ERROR("You need to specify s_nx only or s_nx and s_ny and s_nz") if command_args.second_mask_shape in ('cylinder', 'sphere'): nx = input_vol.get_xsize() ny = input_vol.get_ysize() nz = input_vol.get_zsize() if command_args.s_radius > nx // 2 or command_args.s_radius > ny // 2: ERROR("Provided radius is larger than input image dimensions!") if command_args.second_mask_shape is not None: nx = input_vol.get_xsize() ny = input_vol.get_ysize() nz = input_vol.get_zsize() if command_args.s_nx > nx or command_args.s_ny > ny or command_args.s_nz > nz: ERROR( "Provided s_nx, s_ny, s_nz mask dimension is larger than input image dimensions!" )
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " stack dendoname <maskfile> --link=kind_of_link --dist=kind_of_dist --dissimilar" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--link", type='string', default="single", help="Kind of linkage: single, complete, average (default single)") parser.add_option("--dist", type='string', default="sim_SqEuc", help="Kind of distance: SqEuc, CCC (default SqEuc)") parser.add_option( "--dissimilar", action='store_true', default=False, help="Change the distance to the negative value (default False)") chk_link = ['single', 'complete', 'average'] chk_dist = ['SqEuc', 'CCC'] (options, args) = parser.parse_args() if len(args) < 2 or len(args) > 3: sxprint("Usage: " + usage) sxprint("Please run '" + progname + " -h' for detailed options") ERROR( "Invalid number of parameters. Please see usage information above." ) return elif options.link not in chk_link: ERROR("Kind of linkage unknown") return elif options.dist not in chk_dist: ERROR("Kind of distance unknown") return else: if len(args) == 2: maskname = None else: maskname = args[2] if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import HAC_clustering sp_global_def.BATCH = True HAC_clustering(args[0], args[1], maskname, options.link, options.dist, options.dissimilar) sp_global_def.BATCH = False
def main(): arglist = [] for arg in sys.argv: arglist.append( arg ) progname = os.path.basename( arglist[0] ) usage = progname + " stack --params='parm1 parm2 parm3 ...' --zero --one --set=number --randomize --rand_alpha --import=file --export=file --print --backup --suffix --restore --delete" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--params", type="string", default=None, help="parameter list") parser.add_option("--zero", action="store_true", default=False, help="set parameter to zero") parser.add_option("--one", action="store_true", default=False, help="set parameter to one") parser.add_option("--set", type="float", default=0.0, help="set parameter to a value (different from 0.0)") parser.add_option("--randomize", action="store_true", default=False, help="set parameter to randomized value") parser.add_option("--rand_alpha", action="store_true", default=False, help="set all angles to randomized value") parser.add_option("--import", type="string", dest="fimport", default=None, help="import parameters from file") parser.add_option("--export", type="string", dest="fexport", default=None, help="export parameters to file") parser.add_option("--print", action="store_true", dest="fprint", default=False, help="print parameters") parser.add_option("--backup", action="store_true", default=False, help="backup parameters") parser.add_option("--suffix", type="string", default="_backup", help="suffix for xform name in backup") parser.add_option("--restore", action="store_true", default=False, help="restore parameters") parser.add_option("--delete", action="store_true", default=False, help="delete parameters") parser.add_option("--consecutive", action="store_true", default=False, help="set selected parameter to consecutive integers starting from 0") parser.add_option("--list", type="string", default=None, help="Indices list containing the same amount of rows as the import file") (options,args) = parser.parse_args( arglist[1:] ) if not options.fprint: sp_global_def.print_timestamp( "Start" ) sp_global_def.write_command() if len(args) != 1 : sxprint( "Usage: " + usage ) ERROR( "Invalid number of parameters provided. Please see usage information above." ) return if options.params == None: sxprint( "Usage: " + usage ) ERROR( "No parameters provided. Please see usage information above." ) return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import header header(args[0], options.params, options.zero, options.one, options.set, options.randomize, options.rand_alpha, options.fimport, options.fexport, \ options.fprint, options.backup, options.suffix, options.restore, options.delete, options.consecutive, options.list) if not options.fprint: sp_global_def.print_timestamp( "Finish" )
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " input_stack output_stack average --avg --CTF" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--avg", action="store_true", default=True, help= " Subtract averages computed within corners of individual images, default False" ) parser.add_option( "--CTF", action="store_true", default=False, help=" Consider CTF correction during the alignment, dafault False") (options, args) = parser.parse_args() if len(args) != 3: sxprint("Usage: " + usage) sxprint("Please run \'" + progname + " -h\' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import prepare_2d_forPCA sp_global_def.BATCH = True prepare_2d_forPCA(args[0], args[1], args[2], options.avg, options.CTF) sp_global_def.BATCH = False
def main(): arglist = [] for arg in sys.argv: arglist.append( arg ) progname = os.path.basename( arglist[0] ) usage = progname + " stack1 <stack2> <mask> --ccc --fsc file --inf --rad=r" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--ccc", action="store_true", default=False, help="print cross corelation coefficient" ) parser.add_option( "--fsc", type="string", default="", help="calculate resolution curve" ) parser.add_option( "--inf", action="store_true", default=False, help="print basic infomation of the img" ) parser.add_option( "--rad", type="int", default=-1, help="radius of operation" ) (options,args) = parser.parse_args( arglist[1:] ) if len(args)<1 or len(args)>3: sxprint("usage: " + usage) sxprint("Please run '" + progname + " -h' for detailed options") ERROR( "Invalid number of parameters. Please see usage information above" ) return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import imgstat sp_global_def.BATCH = True imgstat( args, options.ccc, options.fsc, options.inf, options.rad )
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " stack dendoname averages_name --K=number_of_groups" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--K", type="int", default=2, help="Number of classes (default 2)") (options, args) = parser.parse_args() if len(args) != 3: sxprint("Usage: " + usage) sxprint("Please run \'" + progname + " -h\' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import HAC_averages sp_global_def.BATCH = True HAC_averages(args[0], args[1], args[2], options.K) sp_global_def.BATCH = False
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " input_stack start end output_stack <mask> --rad=mask_radius" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--rad", type="int", default=-1, help="radius of mask") parser.add_option("--verbose", type="int", default=0, help="verbose level (0|1)") (options, args) = parser.parse_args() if len(args) < 4: sxprint("Usage: " + usage) sxprint("Please run \'" + progname + " -h\' for details") ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: from string import atoi input_stack = args[0] imgstart = atoi( args[1] ) imgend = atoi( args[2] ) +1 output_stack = args[3] if(len(args) == 5): mask = args[4] else: mask = None if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import varimax sp_global_def.BATCH = True varimax(input_stack, list(range(imgstart, imgend)), output_stack, mask, options.rad, options.verbose) sp_global_def.BATCH = False
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " input_filename output_filename --sym=Symmetry group --phi --theta --psi=The 3 Eulerian angles in degrees --r=Radius of mask --phirange --thetarange --psirange=A search scale for each angle --ftol --xtol = convergence criterion the function and angles values" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--sym", type="string", default="c1", help=" String that specifies the point group symmetry. default = 'c1'") parser.add_option("--phi", type='float',default=0.0, help=" phi angle, default = 0") parser.add_option("--theta", type='float',default=0.0, help=" theta angle, default=0") parser.add_option("--psi", type='float',default=0.0, help=" phi angle, default=0") parser.add_option("--r", type='float',default=None,help=" Input the radius of the mask. default=None") parser.add_option("--phirange", type='float',default=20.0,help=" The search scale for phi angle...default=20") parser.add_option("--thetarange", type='float',default=20.0,help=" The search scale for theta angle...default=20") parser.add_option("--psirange", type='float',default=20.0,help=" The search scale for psi angle...default=20") parser.add_option("--ftol", type='float',default=1.e-4,help=" convergence criterion on the function values...default = 1.e-4") parser.add_option("--xtol", type='float',default=1.e-4,help=" convergence criterion on the variable values...default = 1.e-4") (options, args) = parser.parse_args() if len(args) != 2: sxprint("Usage: " + usage) sxprint("Please run \'" + progname + " -h\' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import rot_sym sp_global_def.BATCH = True rot_sym(args[0],args[1],options.sym,options.r,options.phi,options.theta,options.psi,options.phirange,options.thetarange,options.psirange,options.ftol,options.xtol) sp_global_def.write_command('.') sp_global_def.BATCH = False
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " input_stack output_stack --subavg=average_image --rad=mask_radius --nvec=number_of_eigenvectors --incore --mask=maskfile --shuffle --usebuf --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--subavg", type="string", default="", help="subtract average") parser.add_option("--rad", type="int", default=-1, help="radius of mask") parser.add_option("--nvec", type="int", default=1, help="number of eigenvectors") parser.add_option("--mask", type="string", default="", help="mask file") parser.add_option("--genbuf", action="store_true", default=False, help="use existing buffer") parser.add_option("--shuffle", action="store_true", default=False, help="use shuffle") parser.add_option("--incore", action="store_true", default=False, help="no buffer on a disk") parser.add_option("--MPI", action="store_true", default=False, help="run mpi version") (options, args) = parser.parse_args() input_stacks = args[0:-1] output_stack = args[-1] if options.nvec is None: ERROR("Error: number of components is not given") return isRoot = True if options.MPI: isRoot = (mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD) == 0) if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import pca sp_global_def.BATCH = True vecs = [] vecs = pca(input_stacks, options.subavg, options.rad, options.nvec, options.incore, options.shuffle, not (options.genbuf), options.mask, options.MPI) if isRoot: for i in range(len(vecs)): vecs[i].write_image(output_stack, i) sp_global_def.BATCH = False
def get_gui_arg_img_sets(filenames): ''' returns the img_sets list required to intialized the GUI correctly ''' img_sets = [] if db_check_dict("bdb:e2ctf.parms"): db_parms = db_open_dict("bdb:e2ctf.parms", ro=True) else: return img_sets for file in filenames: name = base_name(file) if name not in db_parms: ERROR( "You must first run auto fit before running the gui - there are no parameters for \'" + name + "\'") return [] img_set = db_parms[name] ctf = EMAN2Ctf() ctf.from_string( img_set[0]) # convert to ctf object seeing as it's a string img_set[0] = ctf actual = [file] actual.extend(img_set) img_sets.append(actual) return img_sets
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + "2Dprojections plot_output Read projection angles from 2Dprojections file or from a text file and write a 2D image file containing their distribution on a hemisphere." parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--wnx", type="int", default=256, help="plot image size (default = 256)") (options, args) = parser.parse_args() if len(args) != 2: sxprint("Usage: " + usage) sxprint("Please run \'" + progname + " -h\' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import plot_projs_distrib sp_global_def.BATCH = True plot_projs_distrib(args[0], args[1], options.wnx) sp_global_def.BATCH = False
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " stack outdir <maskfile> --ou=outer_radius --br=brackets --center=center_type --eps=epsilon --maxit=max_iter --CTF --snr=SNR --function=user_function_name" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--ou", type="float", default=-1, help=" outer radius for a 2-D mask within which the alignment is performed") parser.add_option("--br", type="float", default=1.75, help=" brackets for the search of orientation parameters (each parameter will be checked +/- bracket (set to 1.75)") parser.add_option("--center", type="float", default=1, help=" 0 - if you do not want the average to be centered, 1 - center the average (default=1)") parser.add_option("--eps", type="float", default=0.001, help=" stopping criterion, program will terminate when the relative increase of the criterion is less than epsilon ") parser.add_option("--maxit", type="float", default=10, help=" maximum number of iterations (set to 10) ") parser.add_option("--CTF", action="store_true", default=False, help=" Consider CTF correction during the alignment ") parser.add_option("--snr", type="float", default=1.0, help=" Signal-to-Noise Ratio of the data") parser.add_option("--function", type="string", default="ref_ali2d", help=" name of the reference preparation function") (options, args) = parser.parse_args() if len(args) < 2 or len(args) >3: sxprint( "Usage: " + usage ) sxprint( "Please run '" + progname + " -h' for detailed options" ) ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: if len(args) == 2: mask = None else: mask = args[2] from sp_applications import local_ali2d if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() sp_global_def.BATCH = True local_ali2d(args[0], args[1], mask, options.ou, options.br, options.center, options.eps, options.maxit, options.CTF, options.snr, options.function) sp_global_def.BATCH = False
def main(): import sys arglist = [] for arg in sys.argv: arglist.append( arg ) progname = os.path.basename(arglist[0]) usage = progname + " prjstack outdir bufprefix --delta --d --nvol --nbufvol --seedbase --snr --npad --CTF --MPI --verbose" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--nvol", type="int", help="number of resample volumes to be generated") parser.add_option("--nbufvol", type="int", default=1, help="number of fftvols in the memory") parser.add_option("--delta", type="float", default=10.0, help="angular step for cones") parser.add_option("--d", type="float", default=0.1, help="fraction of projections to leave out") parser.add_option("--CTF", action="store_true", default=False, help="use CTF") parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio") parser.add_option("--npad", type="int", default=2, help="times of padding") parser.add_option("--seedbase", type="int", default=-1, help="random seed base") parser.add_option("--MPI", action="store_true", default=False, help="use MPI") parser.add_option("--verbose", type="int", default=0, help="verbose level: 0 no, 1 yes") (options, args) = parser.parse_args( arglist[1:] ) if( len(args) !=1 and len(args) != 3): sxprint("Usage: " + usage) ERROR( "Invalid number of parameters used. Please see usage information above." ) return prjfile = args[0] if options.MPI: myid = mpi.mpi_comm_rank( mpi.MPI_COMM_WORLD ) ncpu = mpi.mpi_comm_size( mpi.MPI_COMM_WORLD ) else: myid = 0 ncpu = 1 if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() outdir = args[1] bufprefix = args[2] resample( prjfile, outdir, bufprefix, options.nbufvol, options.nvol, options.seedbase,\ options.delta, options.d, options.snr, options.CTF, options.npad,\ options.MPI, myid, ncpu, options.verbose )
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " stack " parser = OptionParser(usage, version=sp_global_def.SPARXVERSION) (options, args) = parser.parse_args(sys.argv[1:]) if len(args) != 1: sxprint("Usage: " + usage) sxprint("Please run \'" + progname + " -h\' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import wrapper_params_3D_to_2D sp_global_def.BATCH = True wrapper_params_3D_to_2D(args[0]) sp_global_def.BATCH = False
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " prjstack bufprefix --npad --CTF --verbose" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--CTF", action="store_true", default=False, help="use CTF") parser.add_option("--npad", type="int", default=2, help="times of padding") parser.add_option("--verbose", type="int", default=0, help="verbose level: 0 no, 1 yes") (options, args) = parser.parse_args(arglist[1:]) if (len(args) != 2): sxprint("Usage: " + usage) sxprint("Please run \'" + progname + " -h\' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return prjfile = args[0] if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() bufprefix = args[1] nprj = EMUtil.get_image_count(prjfile) genbuf(prjfile, bufprefix, 0, nprj, options.CTF, options.npad, options.verbose)
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " stack_in stack_out" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--shift", action="store_true", default=False, help= "Apply only translation, disregard rotation, useful for centering of data (default False)" ) parser.add_option( "--ignore_mirror", action="store_true", default=False, help= "If centering data with CTF and astigmatism, use option ignore mirror (default False)" ) parser.add_option("--method", type="string", default="quadratic", help="Interpolation method (default quadratic)") (options, args) = parser.parse_args() if len(args) != 2: sxprint("Usage: " + usage) sxprint("Please run \'" + progname + " -h\' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() sp_global_def.BATCH = True transform2d(args[0], args[1], options.shift, options.ignore_mirror, options.method) sp_global_def.BATCH = False
def main(): import sys arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " eigvol EIG_prefix (output volumes multiplied by sqrt(eigval), if set" parser = OptionParser(usage, version=SPARXVERSION) (options, args) = parser.parse_args(arglist[1:]) if (len(args) != 2): sxprint("Usage: " + usage) ERROR( "Invalid number of parameters used. Please see usage information above." ) return from math import sqrt nimage = EMUtil.get_image_count(args[0]) for i in range(nimage): data = EMData() data.read_image(args[0], i) eigval = data.get_attr_default("eigval", 1.0) Util.mul_scalar(data, sqrt(eigval)) fname = args[1] + ("%04d_pos.hdf" % (i + 1)) data.write_image(fname) fname = args[1] + ("%04d_neg.hdf" % (i + 1)) Util.mul_scalar(data, -1) data.write_image(fname)
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " out_averages outdir --ou=outer_radius --xr=x_range --ts=translation_step --maxit=max_iteration --CTF --snr=SNR --function=user_function_name --Fourvar --th_err=threshold_cutoff --ali=kind_of_alignment --center=center_type" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--ou", type="int", default=-1, help="outer radius for rotational correlation < nx/2-1 (set to the radius of the particle)") parser.add_option("--xr", type="string", default="4 2", help="range for translation search in x direction, search is +/xr ") parser.add_option("--ts", type="string", default="2 1", help="step of translation search in both directions") parser.add_option("--maxit", type="float", default=0, help="maximum number of iterations (0 means the maximum iterations is 10, but it will automatically stop should the criterion falls") parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ") parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio of the data (set to 1.0)") parser.add_option("--Fourvar", action="store_true", default=False, help="compute Fourier variance") parser.add_option("--function", type="string", default="ref_ali2d", help="name of the reference preparation function") parser.add_option('--Ng', type='int', default=-1, help='Ng') parser.add_option('--K', type='int', default=-1, help='K') parser.add_option("--dst", type="float", default=0.0, help="") parser.add_option("--center", type="float", default=-1, help="-1.average center method; 0.not centered; 1.phase approximation; 2.cc with Gaussian function; 3.cc with donut-shaped image 4.cc with user-defined reference 5.cc with self-rotated average") parser.add_option("--CUDA", action="store_true", default=False, help=" whether to use CUDA ") parser.add_option("--GPUID", type="string", default="", help=" ID of GPUs to use") parser.add_option('--MPI', action='store_true', default=False, help='MPI') (options, args) = parser.parse_args() if len(args) != 3: sxprint( "Usage: " + usage ) sxprint( "Please run '" + progname + " -h' for detailed options" ) ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() sp_global_def.BATCH = True from sp_development import mref_alignment mref_alignment(args[0], args[1], args[2], options.ou, options.xr, options.ts, options.maxit, options.function, options.snr, options.CTF, options.Fourvar, options.Ng, options.K, options.dst, options.center, options.CUDA, options.GPUID, options.MPI) sp_global_def.BATCH = False
def helicalshiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1): nproc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD) myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD) main_node = 0 ftp = file_type(stack) if myid == main_node: print_begin_msg("helical-shiftali_MPI") max_iter = int(maxit) if (myid == main_node): infils = EMUtil.get_all_attributes(stack, "filament") ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord') filaments = ordersegments(infils, ptlcoords) total_nfils = len(filaments) inidl = [0] * total_nfils for i in range(total_nfils): inidl[i] = len(filaments[i]) linidl = sum(inidl) nima = linidl tfilaments = [] for i in range(total_nfils): tfilaments += filaments[i] del filaments else: total_nfils = 0 linidl = 0 total_nfils = bcast_number_to_all(total_nfils, source_node=main_node) if myid != main_node: inidl = [-1] * total_nfils inidl = bcast_list_to_all(inidl, myid, source_node=main_node) linidl = bcast_number_to_all(linidl, source_node=main_node) if myid != main_node: tfilaments = [-1] * linidl tfilaments = bcast_list_to_all(tfilaments, myid, source_node=main_node) filaments = [] iendi = 0 for i in range(total_nfils): isti = iendi iendi = isti + inidl[i] filaments.append(tfilaments[isti:iendi]) del tfilaments, inidl if myid == main_node: print_msg("total number of filaments: %d" % total_nfils) if total_nfils < nproc: ERROR( 'number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used' % (nproc, total_nfils), myid=myid) # balanced load temp = chunks_distribution([[len(filaments[i]), i] for i in range(len(filaments))], nproc)[myid:myid + 1][0] filaments = [filaments[temp[i][1]] for i in range(len(temp))] nfils = len(filaments) #filaments = [[0,1]] #print "filaments",filaments list_of_particles = [] indcs = [] k = 0 for i in range(nfils): list_of_particles += filaments[i] k1 = k + len(filaments[i]) indcs.append([k, k1]) k = k1 data = EMData.read_images(stack, list_of_particles) ldata = len(data) sxprint("ldata=", ldata) nx = data[0].get_xsize() ny = data[0].get_ysize() if maskfile == None: mrad = min(nx, ny) // 2 - 2 mask = pad(model_blank(2 * mrad + 1, ny, 1, 1.0), nx, ny, 1, 0.0) else: mask = get_im(maskfile) # apply initial xform.align2d parameters stored in header init_params = [] for im in range(ldata): t = data[im].get_attr('xform.align2d') init_params.append(t) p = t.get_params("2d") data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'], p['mirror'], p['scale']) if CTF: from sp_filter import filt_ctf from sp_morphology import ctf_img ctf_abs_sum = EMData(nx, ny, 1, False) ctf_2_sum = EMData(nx, ny, 1, False) else: ctf_2_sum = None ctf_abs_sum = None from sp_utilities import info for im in range(ldata): data[im].set_attr('ID', list_of_particles[im]) st = Util.infomask(data[im], mask, False) data[im] -= st[0] if CTF: ctf_params = data[im].get_attr("ctf") qctf = data[im].get_attr("ctf_applied") if qctf == 0: data[im] = filt_ctf(fft(data[im]), ctf_params) data[im].set_attr('ctf_applied', 1) elif qctf != 1: ERROR('Incorrectly set qctf flag', myid=myid) ctfimg = ctf_img(nx, ctf_params, ny=ny) Util.add_img2(ctf_2_sum, ctfimg) Util.add_img_abs(ctf_abs_sum, ctfimg) else: data[im] = fft(data[im]) del list_of_particles if CTF: reduce_EMData_to_root(ctf_2_sum, myid, main_node) reduce_EMData_to_root(ctf_abs_sum, myid, main_node) if CTF: if myid != main_node: del ctf_2_sum del ctf_abs_sum else: temp = EMData(nx, ny, 1, False) tsnr = 1. / snr for i in range(0, nx + 2, 2): for j in range(ny): temp.set_value_at(i, j, tsnr) temp.set_value_at(i + 1, j, 0.0) #info(ctf_2_sum) Util.add_img(ctf_2_sum, temp) #info(ctf_2_sum) del temp total_iter = 0 shift_x = [0.0] * ldata for Iter in range(max_iter): if myid == main_node: start_time = time() print_msg("Iteration #%4d\n" % (total_iter)) total_iter += 1 avg = EMData(nx, ny, 1, False) for im in range(ldata): Util.add_img(avg, fshift(data[im], shift_x[im])) reduce_EMData_to_root(avg, myid, main_node) if myid == main_node: if CTF: tavg = Util.divn_filter(avg, ctf_2_sum) else: tavg = Util.mult_scalar(avg, 1.0 / float(nima)) else: tavg = model_blank(nx, ny) if Fourvar: bcast_EMData_to_all(tavg, myid, main_node) vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF) if myid == main_node: if Fourvar: tavg = fft(Util.divn_img(fft(tavg), vav)) vav_r = Util.pack_complex_to_real(vav) # normalize and mask tavg in real space tavg = fft(tavg) stat = Util.infomask(tavg, mask, False) tavg -= stat[0] Util.mul_img(tavg, mask) tavg.write_image("tavg.hdf", Iter) # For testing purposes: shift tavg to some random place and see if the centering is still correct #tavg = rot_shift3D(tavg,sx=3,sy=-4) if Fourvar: del vav bcast_EMData_to_all(tavg, myid, main_node) tavg = fft(tavg) sx_sum = 0.0 nxc = nx // 2 for ifil in range(nfils): """ # Calculate filament average avg = EMData(nx, ny, 1, False) filnima = 0 for im in xrange(indcs[ifil][0], indcs[ifil][1]): Util.add_img(avg, data[im]) filnima += 1 tavg = Util.mult_scalar(avg, 1.0/float(filnima)) """ # Calculate 1D ccf between each segment and filament average nsegms = indcs[ifil][1] - indcs[ifil][0] ctx = [None] * nsegms pcoords = [None] * nsegms for im in range(indcs[ifil][0], indcs[ifil][1]): ctx[im - indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx, 1) pcoords[im - indcs[ifil][0]] = data[im].get_attr( 'ptcl_source_coord') #ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0]) #print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True) # search for best x-shift cents = nsegms // 2 dst = sqrt( max((pcoords[cents][0] - pcoords[0][0])**2 + (pcoords[cents][1] - pcoords[0][1])**2, (pcoords[cents][0] - pcoords[-1][0])**2 + (pcoords[cents][1] - pcoords[-1][1])**2)) maxincline = atan2(ny // 2 - 2 - float(search_rng), dst) kang = int(dst * tan(maxincline) + 0.5) #print " settings ",nsegms,cents,dst,search_rng,maxincline,kang # ## C code for alignment. @ming results = [0.0] * 3 results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline, kang, search_rng, nxc) sib = int(results[0]) bang = results[1] qm = results[2] #print qm, sib, bang # qm = -1.e23 # # for six in xrange(-search_rng, search_rng+1,1): # q0 = ctx[cents].get_value_at(six+nxc) # for incline in xrange(kang+1): # qt = q0 # qu = q0 # if(kang>0): tang = tan(maxincline/kang*incline) # else: tang = 0.0 # for kim in xrange(cents+1,nsegms): # dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) # xl = dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # #print " A ", ifil,six,incline,kim,xl,ixl,dxl # qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # xl = -dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # for kim in xrange(cents): # dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) # xl = -dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # xl = dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # if( qt > qm ): # qm = qt # sib = six # bang = tang # if( qu > qm ): # qm = qu # sib = six # bang = -tang #if incline == 0: print "incline = 0 ",six,tang,qt,qu #print qm,six,sib,bang #print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True) for im in range(indcs[ifil][0], indcs[ifil][1]): kim = im - indcs[ifil][0] dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) if (kim < cents): xl = -dst * bang + sib else: xl = dst * bang + sib shift_x[im] = xl # Average shift sx_sum += shift_x[indcs[ifil][0] + cents] # #print myid,sx_sum,total_nfils sx_sum = mpi.mpi_reduce(sx_sum, 1, mpi.MPI_FLOAT, mpi.MPI_SUM, main_node, mpi.MPI_COMM_WORLD) if myid == main_node: sx_sum = float(sx_sum[0]) / total_nfils print_msg("Average shift %6.2f\n" % (sx_sum)) else: sx_sum = 0.0 sx_sum = 0.0 sx_sum = bcast_number_to_all(sx_sum, source_node=main_node) for im in range(ldata): shift_x[im] -= sx_sum #print " %3d %6.3f"%(im,shift_x[im]) #exit() # combine shifts found with the original parameters for im in range(ldata): t1 = Transform() ##import random ##shix=random.randint(-10, 10) ##t1.set_params({"type":"2D","tx":shix}) t1.set_params({"type": "2D", "tx": shift_x[im]}) # combine t0 and t1 tt = t1 * init_params[im] data[im].set_attr("xform.align2d", tt) # write out headers and STOP, under MPI writing has to be done sequentially mpi.mpi_barrier(mpi.MPI_COMM_WORLD) par_str = ["xform.align2d", "ID"] if myid == main_node: from sp_utilities import file_type if (file_type(stack) == "bdb"): from sp_utilities import recv_attr_dict_bdb recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata, nproc) else: from sp_utilities import recv_attr_dict recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc) else: send_attr_dict(main_node, data, par_str, 0, ldata) if myid == main_node: print_end_msg("helical-shiftali_MPI")
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " stack ref_vol outdir --dp=rise --dphi=rotation --apix=pixel_size --phistep=phi_step --zstep=z_step --fract=helicising_fraction --rmax=maximum_radius --rmin=min_radius --CTF --sym=c1 --function=user_function --maxit=max_iter --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--dp", type="float", default=1.0, help="delta z - translation in Angstroms") parser.add_option("--dphi", type="float", default=1.0, help="delta phi - rotation in degrees") parser.add_option("--apix", type="float", default=1.84, help="pixel size in Angstroms") parser.add_option("--rmin", type="int", default=0, help="minimal radial extent of structure") parser.add_option("--rmax", type="int", default=70, help="maximal radial extent of structure") parser.add_option("--fract", type="float", default=0.66, help="fraction of the volume used for helical search") parser.add_option("--sym", type="string", default="c1", help="symmetry of the structure") parser.add_option("--function", type="string", default="helical", help="name of the reference preparation function") parser.add_option("--zstep", type="int", default=1, help="Step size for translational search along z") parser.add_option("--CTF", action="store_true", default=False, help="CTF correction") parser.add_option("--maxit", type="int", default=5, help="maximum number of iterations performed") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version") (options, args) = parser.parse_args(arglist[1:]) if len(args) != 3: sxprint("Usage: " + usage) sxprint("Please run \'" + progname + " -h\' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: if not options.MPI: ERROR( "There is only MPI version of sxfilrecons3d.py. See SPARX wiki page for downloading MyMPI details." ) return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_development import filrecons3D_MPI sp_global_def.BATCH = True filrecons3D_MPI(args[0], args[1], args[2], options.dp, options.dphi, options.apix, options.function, options.zstep, options.fract, options.rmax, options.rmin, options.CTF, options.maxit, options.sym) sp_global_def.BATCH = False
def shiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1, oneDx=False, search_rng_y=-1): number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD) myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD) main_node = 0 ftp = file_type(stack) if myid == main_node: print_begin_msg("shiftali_MPI") max_iter = int(maxit) if myid == main_node: if ftp == "bdb": from EMAN2db import db_open_dict dummy = db_open_dict(stack, True) nima = EMUtil.get_image_count(stack) else: nima = 0 nima = bcast_number_to_all(nima, source_node=main_node) list_of_particles = list(range(nima)) image_start, image_end = MPI_start_end(nima, number_of_proc, myid) list_of_particles = list_of_particles[image_start:image_end] # read nx and ctf_app (if CTF) and broadcast to all nodes if myid == main_node: ima = EMData() ima.read_image(stack, list_of_particles[0], True) nx = ima.get_xsize() ny = ima.get_ysize() if CTF: ctf_app = ima.get_attr_default('ctf_applied', 2) del ima else: nx = 0 ny = 0 if CTF: ctf_app = 0 nx = bcast_number_to_all(nx, source_node=main_node) ny = bcast_number_to_all(ny, source_node=main_node) if CTF: ctf_app = bcast_number_to_all(ctf_app, source_node=main_node) if ctf_app > 0: ERROR("data cannot be ctf-applied", myid=myid) if maskfile == None: mrad = min(nx, ny) mask = model_circle(mrad // 2 - 2, nx, ny) else: mask = get_im(maskfile) if CTF: from sp_filter import filt_ctf from sp_morphology import ctf_img ctf_abs_sum = EMData(nx, ny, 1, False) ctf_2_sum = EMData(nx, ny, 1, False) else: ctf_2_sum = None from sp_global_def import CACHE_DISABLE if CACHE_DISABLE: data = EMData.read_images(stack, list_of_particles) else: for i in range(number_of_proc): if myid == i: data = EMData.read_images(stack, list_of_particles) if ftp == "bdb": mpi.mpi_barrier(mpi.MPI_COMM_WORLD) for im in range(len(data)): data[im].set_attr('ID', list_of_particles[im]) st = Util.infomask(data[im], mask, False) data[im] -= st[0] if CTF: ctf_params = data[im].get_attr("ctf") ctfimg = ctf_img(nx, ctf_params, ny=ny) Util.add_img2(ctf_2_sum, ctfimg) Util.add_img_abs(ctf_abs_sum, ctfimg) if CTF: reduce_EMData_to_root(ctf_2_sum, myid, main_node) reduce_EMData_to_root(ctf_abs_sum, myid, main_node) else: ctf_2_sum = None if CTF: if myid != main_node: del ctf_2_sum del ctf_abs_sum else: temp = EMData(nx, ny, 1, False) for i in range(0, nx, 2): for j in range(ny): temp.set_value_at(i, j, snr) Util.add_img(ctf_2_sum, temp) del temp total_iter = 0 # apply initial xform.align2d parameters stored in header init_params = [] for im in range(len(data)): t = data[im].get_attr('xform.align2d') init_params.append(t) p = t.get_params("2d") data[im] = rot_shift2D(data[im], p['alpha'], sx=p['tx'], sy=p['ty'], mirror=p['mirror'], scale=p['scale']) # fourier transform all images, and apply ctf if CTF for im in range(len(data)): if CTF: ctf_params = data[im].get_attr("ctf") data[im] = filt_ctf(fft(data[im]), ctf_params) else: data[im] = fft(data[im]) sx_sum = 0 sy_sum = 0 sx_sum_total = 0 sy_sum_total = 0 shift_x = [0.0] * len(data) shift_y = [0.0] * len(data) ishift_x = [0.0] * len(data) ishift_y = [0.0] * len(data) for Iter in range(max_iter): if myid == main_node: start_time = time() print_msg("Iteration #%4d\n" % (total_iter)) total_iter += 1 avg = EMData(nx, ny, 1, False) for im in data: Util.add_img(avg, im) reduce_EMData_to_root(avg, myid, main_node) if myid == main_node: if CTF: tavg = Util.divn_filter(avg, ctf_2_sum) else: tavg = Util.mult_scalar(avg, 1.0 / float(nima)) else: tavg = EMData(nx, ny, 1, False) if Fourvar: bcast_EMData_to_all(tavg, myid, main_node) vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF) if myid == main_node: if Fourvar: tavg = fft(Util.divn_img(fft(tavg), vav)) vav_r = Util.pack_complex_to_real(vav) # normalize and mask tavg in real space tavg = fft(tavg) stat = Util.infomask(tavg, mask, False) tavg -= stat[0] Util.mul_img(tavg, mask) # For testing purposes: shift tavg to some random place and see if the centering is still correct #tavg = rot_shift3D(tavg,sx=3,sy=-4) tavg = fft(tavg) if Fourvar: del vav bcast_EMData_to_all(tavg, myid, main_node) sx_sum = 0 sy_sum = 0 if search_rng > 0: nwx = 2 * search_rng + 1 else: nwx = nx if search_rng_y > 0: nwy = 2 * search_rng_y + 1 else: nwy = ny not_zero = 0 for im in range(len(data)): if oneDx: ctx = Util.window(ccf(data[im], tavg), nwx, 1) p1 = peak_search(ctx) p1_x = -int(p1[0][3]) ishift_x[im] = p1_x sx_sum += p1_x else: p1 = peak_search(Util.window(ccf(data[im], tavg), nwx, nwy)) p1_x = -int(p1[0][4]) p1_y = -int(p1[0][5]) ishift_x[im] = p1_x ishift_y[im] = p1_y sx_sum += p1_x sy_sum += p1_y if not_zero == 0: if (not (ishift_x[im] == 0.0)) or (not (ishift_y[im] == 0.0)): not_zero = 1 sx_sum = mpi.mpi_reduce(sx_sum, 1, mpi.MPI_INT, mpi.MPI_SUM, main_node, mpi.MPI_COMM_WORLD) if not oneDx: sy_sum = mpi.mpi_reduce(sy_sum, 1, mpi.MPI_INT, mpi.MPI_SUM, main_node, mpi.MPI_COMM_WORLD) if myid == main_node: sx_sum_total = int(sx_sum[0]) if not oneDx: sy_sum_total = int(sy_sum[0]) else: sx_sum_total = 0 sy_sum_total = 0 sx_sum_total = bcast_number_to_all(sx_sum_total, source_node=main_node) if not oneDx: sy_sum_total = bcast_number_to_all(sy_sum_total, source_node=main_node) sx_ave = round(float(sx_sum_total) / nima) sy_ave = round(float(sy_sum_total) / nima) for im in range(len(data)): p1_x = ishift_x[im] - sx_ave p1_y = ishift_y[im] - sy_ave params2 = { "filter_type": Processor.fourier_filter_types.SHIFT, "x_shift": p1_x, "y_shift": p1_y, "z_shift": 0.0 } data[im] = Processor.EMFourierFilter(data[im], params2) shift_x[im] += p1_x shift_y[im] += p1_y # stop if all shifts are zero not_zero = mpi.mpi_reduce(not_zero, 1, mpi.MPI_INT, mpi.MPI_SUM, main_node, mpi.MPI_COMM_WORLD) if myid == main_node: not_zero_all = int(not_zero[0]) else: not_zero_all = 0 not_zero_all = bcast_number_to_all(not_zero_all, source_node=main_node) if myid == main_node: print_msg("Time of iteration = %12.2f\n" % (time() - start_time)) start_time = time() if not_zero_all == 0: break #for im in xrange(len(data)): data[im] = fft(data[im]) This should not be required as only header information is used # combine shifts found with the original parameters for im in range(len(data)): t0 = init_params[im] t1 = Transform() t1.set_params({ "type": "2D", "alpha": 0, "scale": t0.get_scale(), "mirror": 0, "tx": shift_x[im], "ty": shift_y[im] }) # combine t0 and t1 tt = t1 * t0 data[im].set_attr("xform.align2d", tt) # write out headers and STOP, under MPI writing has to be done sequentially mpi.mpi_barrier(mpi.MPI_COMM_WORLD) par_str = ["xform.align2d", "ID"] if myid == main_node: from sp_utilities import file_type if (file_type(stack) == "bdb"): from sp_utilities import recv_attr_dict_bdb recv_attr_dict_bdb(main_node, stack, data, par_str, image_start, image_end, number_of_proc) else: from sp_utilities import recv_attr_dict recv_attr_dict(main_node, stack, data, par_str, image_start, image_end, number_of_proc) else: send_attr_dict(main_node, data, par_str, image_start, image_end) if myid == main_node: print_end_msg("shiftali_MPI")
def main(): progname = os.path.basename(sys.argv[0]) usage = """%prog [options] input.pdb output.hdf Converts a pdb file into an electron density map. 0,0,0 in PDB space will map to the center of the volume.""" parser = OptionParser(usage=usage, version=EMANVERSION) parser.add_option("--apix", "-A", type="float", help="Angstrom/voxel", default=1.0) parser.add_option("--box", "-B", type="string", help="Box size in pixels, <xyz> or <x,y,z>") parser.add_option("--het", action="store_true", help="Include HET atoms in the map", default=False) parser.add_option( "--chains", type="string", help= "String list of chain identifiers to include, e.g. 'ABEFG'; default: include all chains", default='') parser.add_option( "--center", type="string", default="a", help= "center: c - coordinates; a (default) - center of gravity; <x,y,z> - vector (in Angstrom) to subtract from all coordinates; n - none" ) parser.add_option("--O", action="store_true", default=False, help="use O system of coordinates") parser.add_option("--quiet", action="store_true", default=False, help="Verbose is the default") parser.add_option("--tr0", type="string", default="none", help="Filename of initial 3x4 transformation matrix") parser.add_option("--set_apix_value", action="store_true", help="Set apix value in header of the ouput map", default=False) (options, args) = parser.parse_args() # if len(args) < 2: ERROR("Input and output files required") return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() chains = options.chains if chains == '': chains = None try: infile = open(args[0], "r") except: ERROR("Cannot open input file") return aavg = [0, 0, 0] # calculate atomic center asig = [0, 0, 0] # to compute radius of gyration natm = 0 atoms = [] # we store a list of atoms to process to avoid multiple passes nelec = 0 mass = 0 # read in initial-transformation file: if (options.tr0 != "none"): cols = read_text_file(options.tr0, -1) txlist = [] for i in range(3): txlist.append(cols[0][i]) txlist.append(cols[1][i]) txlist.append(cols[2][i]) txlist.append(cols[3][i]) tr0 = Transform(txlist) # parse the pdb file and pull out relevant atoms for line in infile: if (line[:4] == 'ATOM' or (line[:6] == 'HETATM' and options.het)): if chains and (line[21] not in chains): continue try: a = line[12:14].strip() aseq = int(line[6:11].strip()) res = int(line[22:26].strip()) if (options.O): x = float(line[38:46]) y = float(line[30:38]) z = -float(line[46:54]) else: x = float(line[30:38]) y = float(line[38:46]) z = float(line[46:54]) except: sxprint( "PDB Parse error:\n%s\n'%s','%s','%s' '%s','%s','%s'\n" % (line, line[12:14], line[6:11], line[22:26], line[30:38], line[38:46], line[46:54])) sxprint(a, aseq, res, x, y, z) try: nelec += atomdefs[a.upper()][0] mass += atomdefs[a.upper()][1] except: sxprint(("Unknown atom %s ignored at %d" % (a, aseq))) atoms.append([a, x, y, z]) natm += 1 if (options.center == "a"): aavg[0] += x * atomdefs[a.upper()][1] aavg[1] += y * atomdefs[a.upper()][1] aavg[2] += z * atomdefs[a.upper()][1] asig[0] += x**2 * atomdefs[a.upper()][1] asig[1] += y**2 * atomdefs[a.upper()][1] asig[2] += z**2 * atomdefs[a.upper()][1] else: aavg[0] += x aavg[1] += y aavg[2] += z asig[0] += x**2 asig[1] += y**2 asig[2] += z**2 infile.close() if (options.center == "a"): rad_gyr = sqrt((asig[0] + asig[1] + asig[2]) / mass - (aavg[0] / mass)**2 - (aavg[1] / mass)**2 - (aavg[2] / mass)**2) else: rad_gyr = sqrt((asig[0] + asig[1] + asig[2]) / natm - (aavg[0] / natm)**2 - (aavg[1] / natm)**2 - (aavg[2] / natm)**2) if not options.quiet: sxprint( "%d atoms; total charge = %d e-; mol mass = %.2f kDa; radius of gyration = %.2f A" % (natm, nelec, mass / 1000.0, rad_gyr)) # center PDB according to option: if (options.center == "a"): if not options.quiet: sxprint( "center of gravity at %1.1f,%1.1f,%1.1f (center of volume at 0,0,0)" % (aavg[0] / mass, aavg[1] / mass, aavg[2] / mass)) for i in range(len(atoms)): atoms[i][1] -= aavg[0] / mass atoms[i][2] -= aavg[1] / mass atoms[i][3] -= aavg[2] / mass if (options.center == "c"): if not options.quiet: sxprint( "atomic center at %1.1f,%1.1f,%1.1f (center of volume at 0,0,0)" % (aavg[0] / natm, aavg[1] / natm, aavg[2] / natm)) for i in range(len(atoms)): atoms[i][1] -= aavg[0] / natm atoms[i][2] -= aavg[1] / natm atoms[i][3] -= aavg[2] / natm spl = options.center.split(',') if len(spl) == 3: # substract the given vector from all coordinates if not options.quiet: sxprint( "vector to substract: %1.1f,%1.1f,%1.1f (center of volume at 0,0,0)" % (float(spl[0]), float(spl[1]), float(spl[2]))) for i in range(len(atoms)): atoms[i][1] -= float(spl[0]) atoms[i][2] -= float(spl[1]) atoms[i][3] -= float(spl[2]) # apply given initial transformation (this used to be done before centering, # thereby loosing the translation. This is the right place to apply tr0): if (options.tr0 != "none"): if not options.quiet: sxprint("Applying initial transformation to PDB coordinates... ") for i in range(len(atoms)): atom_coords = Vec3f(atoms[i][1], atoms[i][2], atoms[i][3]) new_atom_coords = tr0 * atom_coords atoms[i][1] = new_atom_coords[0] atoms[i][2] = new_atom_coords[1] atoms[i][3] = new_atom_coords[2] if not options.quiet: sxprint("done.\n") # bounding box: amin = [atoms[0][1], atoms[0][2], atoms[0][3]] amax = [atoms[0][1], atoms[0][2], atoms[0][3]] for i in range(1, len(atoms)): for k in range(3): amin[k] = min(atoms[i][k + 1], amin[k]) amax[k] = max(atoms[i][k + 1], amax[k]) if not options.quiet: sxprint("Range of coordinates [A]: x: %7.2f - %7.2f" % (amin[0], amax[0])) sxprint(" y: %7.2f - %7.2f" % (amin[1], amax[1])) sxprint(" z: %7.2f - %7.2f" % (amin[2], amax[2])) # find the output box size, either user specified or from bounding box box = [0, 0, 0] try: spl = options.box.split(',') if len(spl) == 1: box[0] = box[1] = box[2] = int(spl[0]) else: box[0] = int(spl[0]) box[1] = int(spl[1]) box[2] = int(spl[2]) except: for i in range(3): box[i] = int(2 * max(fabs(amax[i]), fabs(amin[i])) / options.apix) # Increase the box size by 1/4. box[i] += box[i] // 4 if not options.quiet: sxprint("Bounding box [pixels]: x: %5d " % box[0]) sxprint(" y: %5d " % box[1]) sxprint(" z: %5d " % box[2]) # figure oversampled box size #bigb = max(box[0],box[1],box[2]) fcbig = 1 """Multiline Comment0""" #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 if not options.quiet: sxprint("Box size: %d x %d x %d" % (box[0], box[1], box[2]), ", oversampling ", fcbig) # Calculate working dimensions pixelbig = options.apix / fcbig bigbox = [] for i in range(3): bigbox.append(box[i] * fcbig) # initialize the final output volume outmap = EMData(bigbox[0], bigbox[1], bigbox[2], True) nc = [] for i in range(3): nc.append(bigbox[i] // 2) # fill in the atoms for i in range(len(atoms)): #print "Adding %d '%s'"%(i,atoms[i][0]) if not options.quiet and i % 1000 == 0: sxprint('\r %d' % i, end=' ') sys.stdout.flush() try: elec = atomdefs[atoms[i][0].upper()][0] #outmap[int(atoms[i][1]/pixelbig+bigbox[0]//2),int(atoms[i][2]/pixelbig+bigbox[1]//2),int(atoms[i][3]/pixelbig+bigbox[2]//2)] += elec for k in range(2): pz = atoms[i][3] / pixelbig + nc[2] dz = pz - int(pz) uz = ((1 - k) + (2 * k - 1) * dz) * elec for l in range(2): py = atoms[i][2] / pixelbig + nc[1] dy = py - int(py) uy = ((1 - l) + (2 * l - 1) * dy) * uz for m in range(2): px = atoms[i][1] / pixelbig + nc[0] dx = px - int(px) outmap[int(px) + m, int(py) + l, int(pz) + k] += ((1 - m) + (2 * m - 1) * dx) * uy except: sxprint("Skipping %d '%s'" % (i, atoms[i][0])) if not options.quiet: sxprint('\r %d\nConversion complete.' % len(atoms)) #," Now shape atoms." """Multiline Comment1""" #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 (filename_path, filextension) = os.path.splitext(args[1]) if filextension == ".hdf": if options.set_apix_value: outmap.set_attr("apix_x", options.apix) outmap.set_attr("apix_y", options.apix) outmap.set_attr("apix_z", options.apix) outmap.set_attr("pixel_size", options.apix) else: sxprint("Pixel_size is not set in the header!") outmap.write_image(args[1], 0, EMUtil.ImageType.IMAGE_HDF) elif filextension == ".spi": outmap.write_image(args[1], 0, EMUtil.ImageType.IMAGE_SINGLE_SPIDER) else: ERROR("Unknown image type") return
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " stack outdir --phase=1 --ou=outer_radius|sxconsistency.py --phase=3 newlocal/main000 --ou=133 --thresherr=3.0 --params=paramsa outgrouparms" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--phase", type="int", default=1, help= "Phase =1 prepares resampled stacks, =2 analyzes consistency of orientation parameters" ) parser.add_option("--ou", type="int", default=-1, help="outer radius for calculation of pixel error") parser.add_option("--sym", type="string", default="c1", help="symmetry of the refined structure") parser.add_option( "--thresherr", type="float", default=1.0, help="Threshold for accpetable orientation errors (in pixels)") parser.add_option( "--ndigits", type="int", default=1, help="Accuracy for checking whether parameters are identical") parser.add_option("--chunk", type="string", default="", help="Root of of four chunk files with indeces") parser.add_option( "--params", type="string", default="", help="Root of of six parameter file names with refinement results") (options, args) = parser.parse_args(arglist[1:]) sp_global_def.BATCH = True if options.phase == 1 and len(args) == 2: inputbdb = args[0] outdir = args[1] nn = EMUtil.get_image_count(inputbdb) t = list(range(nn)) shuffle(t) chunks = [] for i in range(4): # I use the MPI function here just to easily get the balanced load j, k = MPI_start_end(nn, 4, i) chunks.append(t[j:k]) chunks[i].sort() write_text_file(chunks[i], os.path.join(outdir, 'chunk%01d.txt' % i)) del t write_text_file([len(chunks[i]) for i in range(4)], os.path.join(outdir, 'chunklengths.txt')) """ pt = [[None]]*6 ll=0 for i in xrange(3): for j in xrange(i+1,4): pt[ll] = chunks[i]+chunks[j] ll+=1 for i in xrange(6): listfile = os.path.join(outdir,'lili%01d.txt'%i) write_text_file(pt[i],listfile) outbdb = "bdb:"+ os.path.join(outdir,"X%01d"%i) cmd = '{} {} {} {}'.format('e2bdb.py', inputbdb, '--makevstack='+outbdb, '--list='+listfile) subprocess.call(cmd, shell=True) """ # Run 6 programs elif options.phase == 2 and len(args) == 2: outdir = args[0] howmanythesame = args[1] ndigits = options.ndigits #for chc5 1, for ribo 4#4.0 prms = [] for i in range(6): prms.append( read_text_row( os.path.join(outdir, options.params + "%01d.txt" % i))) for j in range(len(prms[-1])): for k in range(5): prms[-1][j][k] = round(prms[-1][j][k], ndigits) nn = 2 * len(prms[0]) n4 = nn // 4 qt = [[[-1.0]] * nn for i in range(6)] ll = 0 for i in range(3): for j in range(i + 1, 4): qt[ll][i * n4:(i + 1) * n4] = prms[ll][:n4] qt[ll][j * n4:(j + 1) * n4] = prms[ll][n4:] ll += 1 thesame = 0 for ll in range(len(qt[0])): rw = [] for j in range(6): if (len(qt[j][ll]) > 1): rw.append(qt[j][ll]) isame = True for j in range(3): if (rw[0][j] != rw[1][j]): isame = False #print ll,rw[0][j], rw[1][j] break if (rw[0][j] != rw[2][j]): isame = False #print ll,rw[0][j], rw[2][j] break if isame: thesame += 1 qt = float(thesame) / nn sxprint("Proportion of the same orientations ", qt) write_text_file([qt], os.path.join(outdir, howmanythesame)) ######### PHASE 3 elif options.phase == 3 and len(args) == 2: outdir = args[0] outgrouparms = args[1] radius = options.ou thresherr = options.thresherr #for chc5 1, for ribo 4#4.0 sym = int(options.sym[1:]) qsym = 360.0 / sym blocks = ['A', 'B', 'C', 'D'] params = {} ll = 0 for i in range(3): for j in range(i + 1, 4): params[chr(65 + i) + chr(48 + ll)] = [] params[chr(65 + j) + chr(48 + ll)] = [] ll += 1 chunks = {} chunklengths = {} for i in range(4): chunks[chr(65 + i)] = list( map( int, read_text_file( os.path.join(outdir, options.chunk + "%01d.txt" % i)))) chunklengths[chr(65 + i)] = len(chunks[chr(65 + i)]) for i in range(6): prms = read_text_row( os.path.join(outdir, options.params + "%01d.txt" % i)) for q in blocks: if q + chr(48 + i) in params: params[q + chr(48 + i)] = prms[:chunklengths[q]] del prms[:chunklengths[q]] pairs = [["A0", "A1"], ["B3", "B4"], ["C1", "C5"], ["D2", "D4"]] lefts = ["A2", "B0", "C3", "D5"] # Compute average projection params and pixel errors avgtrans = {} pixer = {} for i, q in enumerate(pairs): avgtrans[q[0][0]] = [0.0] * chunklengths[q[0][0]] pixer[q[0][0]] = [0.0] * chunklengths[q[0][0]] for j in range(chunklengths[q[0][0]]): fifi = [params[q[0]][j], params[q[1]][j]] nas = [0.0, 0.0, 0.0] if (sym == 1): pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in range(2): n1, n2, n3 = getfvec(fifi[i][0], fifi[i][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1, m2, m3 = getfvec(fifi[0][0], fifi[0][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 #if(k == 2): # print "XXXX" # print fifi[0],nas for i in range(1, 2): qnom = -1.e10 for j in range(-1, 2, 1): t1, t2, t3 = getfvec(fifi[i][0] + j * qsym, fifi[i][1]) nom = t1 * m1 + t2 * m2 + t3 * m3 if (nom > qnom): qnom = nom n1 = t1 n2 = t2 n3 = t3 #if(k == 2): # print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 sxprint(qnom, n1, n2, n3, nas) # To get the correct pixer phi angle has to be taken from the above!! pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if (nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2] / nom)) % 360.0 if (sym > 1 and ntheta > 90.0): nphi = (degrees(atan2(nas[1], nas[0])) - 180.0) % qsym + 180.0 else: nphi = degrees(atan2(nas[1], nas[0])) % qsym #print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform([fifi[0][2:], fifi[1][2:]]) avgtrans[q[0][0]][j] = [ nphi, ntheta, twod[0], twod[1], twod[2] ] perr = {} tgood = 0 for q in blocks: perr[q] = [True] * chunklengths[q] for k in range(chunklengths[q]): if (pixer[q][k] > thresherr): perr[q][k] = False if perr[q][k]: tgood += 1 if (tgood < 4): ERROR( "No good images within the pixel error threshold specified" ) return sxprint(" tgood ", tgood) hi = hist_list( [pixer[q][k] for q in blocks for k in range(chunklengths[q])], 16) for i in range(len(hi[0])): sxprint("%4d %12.3f %12.0f " % (i, hi[0][i], hi[1][i])) # Finished, store average orientation params and table of good images # store lists of good images for each group # blocks is indexed by first letter good = [[] for i in range(4)] bad = [[] for i in range(4)] for i, q in enumerate(blocks): for k in range(chunklengths[q]): #deprt = max_3D_pixel_error(params[lefts[i]][k],avgtrans[q][k],r=radius) if perr[q][k]: good[i].append(chunks[q][k]) #[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k] ]) else: bad[i].append(chunks[q][k]) #[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k]]) write_text_file(good[i], os.path.join(outdir, "newgood%01d.txt" % i)) write_text_file(bad[i], os.path.join(outdir, "newbad%01d.txt" % i)) # write out parameters, for those in pairs write out average, for leftouts leave them as they were # These parameters refer to the original X files. ll = 0 for m in range(6): if q + chr(48 + m) in params: prmsgood = [] prmsbad = [] try: j = lefts.index(q + chr(48 + m)) for k in range(chunklengths[q]): if perr[q][k]: prmsgood.append(params[q + chr(48 + m)][k]) else: prmsbad.append(params[q + chr(48 + m)][k]) except: for k in range(chunklengths[q]): if perr[q][k]: prmsgood.append(avgtrans[q][k]) else: prmsbad.append(avgtrans[q][k]) write_text_row( prmsgood, os.path.join(outdir, "params-newgood%01d%01d.txt" % (i, ll))) write_text_row( prmsbad, os.path.join(outdir, "params-newbad%01d%01d.txt" % (i, ll))) ll += 1 # Generate newlili files from newgood, these contain original numbering of the total single file ll = 0 for i in range(3): for j in range(i + 1, 4): write_text_file(good[i] + good[j], os.path.join(outdir, "newlili%01d.txt" % ll)) ll += 1 # write out parameters, for those in pairs write out average, for leftouts leave them as they were # These parameters refer to the original X files. for i in range(6): prms = [] for q in blocks: if q + chr(48 + i) in params: try: j = lefts.index(q + chr(48 + i)) prms += params[q + chr(48 + i)] except: prms += avgtrans[q] write_text_row(prms, os.path.join(outdir, outgrouparms + "%01d.txt" % i)) # Write chunklengths chunklengths = [len(good[i]) for i in range(4)] write_text_file(chunklengths, os.path.join(outdir, "chunklengths.txt")) """ # We do not use consecutive numbering anymore # Generate newx files from newgood, these contain consecutive (with gaps) numbering that allows to generate truncated X files from the previous X files ll = 0 for i in xrange(3): firstblock = [] l = len(perr[blocks[i]]) for k in xrange(l): if perr[blocks[i]][k]: firstblock.append(k) for j in xrange(i+1,4): secondblock = [] for k in xrange(len(perr[blocks[j]])): if perr[blocks[j]][k]: secondblock.append(k+l) write_text_file( firstblock + secondblock, os.path.join(outdir,"goodX%01d.txt"%ll) ) ll += 1 del good,bad,firstblock,secondblock,perr """ """ 0 1 2 3 4 5 A A A = = = B = = B B = = C = C = C = = D = D D """ elif options.phase == 4 and len(args) == 1: outdir = args[0] bp = 'badparams' #outgrouparms= args[1] radius = options.ou thresherr = options.thresherr sym = int(options.sym[1:]) qsym = 360.0 / sym #params = [[None for i in xrange(3)] for j in xrange(4)] ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general. for jj in range(4): params = [None for ii in range(ll)] newbad = list( map(int, read_text_file(options.params + "%01d.txt" % jj))) nn = len(newbad) for ii in range(ll): params[ii] = read_text_row( os.path.join(outdir, bp + "%01d%01d.txt" % (jj, ii))) assert (nn == len(params[ii])) # Compute average projection params and pixel errors avgtrans = [None] * nn pixer = [0.0] * nn for j in range(nn): nas = [0.0, 0.0, 0.0] if (sym == 1): #pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in range(ll): n1, n2, n3 = getfvec(params[i][j][0], params[i][j][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1, m2, m3 = getfvec(params[0][j][0], params[0][j][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 for i in range(1, ll): qnom = -1.e10 for j in range(-1, 2, 1): t1, t2, t3 = getfvec(params[i][j][0] + j * qsym, params[i][j][1]) nom = t1 * m1 + t2 * m2 + t3 * m3 if (nom > qnom): qnom = nom n1 = t1 n2 = t2 n3 = t3 #if(k == 2): # sxprint ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 sxprint(qnom, n1, n2, n3, nas) # To get the correct pixer phi angle has to be taken from the above!! nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if (nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2] / nom)) % 360.0 if (sym > 1 and ntheta > 90.0): nphi = (degrees(atan2(nas[1], nas[0])) - 180.0) % qsym + 180.0 else: nphi = degrees(atan2(nas[1], nas[0])) % qsym #sxprint "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform( [params[ii][j][2:] for ii in range(ll)]) avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]] perr = errors_per_image(params, avgtrans, thresherr, radius) rescued = [] rejects = [] for j in range(nn): #sxprint chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)] if (perr[j][0] <= thresherr): rescued.append([newbad[j], j]) else: rejects.append([newbad[j], j]) if (len(rescued) == 0): write_text_row([-1, -1], os.path.join(outdir, "rescued%01d.txt" % jj)) else: write_text_row(rescued, os.path.join(outdir, "rescued%01d.txt" % jj)) # We also have to write params. if (len(rescued) != 0): for ii in range(ll): write_text_row( [ params[ii][rescued[k][1]] for k in range(len(rescued)) ], os.path.join(outdir, "params-rescued%01d%01d.txt" % (jj, ii))) if (len(rejects) == 0): write_text_row([-1, -1], os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) else: write_text_row( rejects, os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) if (len(rejects) != 0): for ii in range(ll): write_text_row( [ params[ii][rejects[k][1]] for k in range(len(rejects)) ], os.path.join(outdir, "params-rejects%01d%01d.txt" % (jj, ii))) hi = hist_list([perr[j][0] for j in range(nn)], 16) sxprint("Pixel errors for BAD GROUP ", chr(65 + jj)) for ii in range(len(hi[0])): sxprint("%4d %12.3f %12.0f " % (ii, hi[0][ii], hi[1][ii])) elif options.phase == 5 and len(args) == 1: # This version is for meridien refinement. There are simply three full sets of params. outdir = args[0] bp = 'badparams' #outgrouparms= args[1] radius = options.ou thresherr = options.thresherr sym = int(options.sym[1:]) qsym = 360.0 / sym #params = [[None for i in xrange(3)] for j in xrange(4)] ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general. for jj in range(1): params = [None for ii in range(ll)] #newbad = map(int, read_text_file(options.params+"%01d.txt"%jj) ) #nn = len(newbad) for ii in range(ll): params[ii] = read_text_row( os.path.join(outdir, "params%01d.txt" % (ii))) #assert(nn == len(params[ii]) ) nn = len(params[0]) newbad = list(range(nn)) # Compute average projection params and pixel errors avgtrans = [None] * nn pixer = [0.0] * nn for j in range(nn): nas = [0.0, 0.0, 0.0] if (sym == 1): #pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in range(ll): n1, n2, n3 = getfvec(params[i][j][0], params[i][j][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1, m2, m3 = getfvec(params[0][j][0], params[0][j][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 for i in range(1, ll): qnom = -1.e10 for j in range(-1, 2, 1): t1, t2, t3 = getfvec(params[i][j][0] + j * qsym, params[i][j][1]) nom = t1 * m1 + t2 * m2 + t3 * m3 if (nom > qnom): qnom = nom n1 = t1 n2 = t2 n3 = t3 #if(k == 2): # sxprint ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 sxprint(qnom, n1, n2, n3, nas) # To get the correct pixer phi angle has to be taken from the above!! nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if (nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2] / nom)) % 360.0 if (sym > 1 and ntheta > 90.0): nphi = (degrees(atan2(nas[1], nas[0])) - 180.0) % qsym + 180.0 else: nphi = degrees(atan2(nas[1], nas[0])) % qsym #sxprint "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform( [params[ii][j][2:] for ii in range(ll)]) avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]] write_text_row(avgtrans, os.path.join(outdir, "avgtrans.txt")) perr = errors_per_image(params, avgtrans, thresherr, radius) rescued = [] rejects = [] for j in range(nn): #sxprint chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)] if (perr[j][0] <= thresherr): rescued.append([newbad[j], j]) else: rejects.append([newbad[j], j]) if (len(rescued) == 0): write_text_row([-1, -1], os.path.join(outdir, "rescued%01d.txt" % jj)) else: write_text_row(rescued, os.path.join(outdir, "rescued%01d.txt" % jj)) # We also have to write params. if (len(rescued) != 0): for ii in range(ll): write_text_row( [ params[ii][rescued[k][1]] for k in range(len(rescued)) ], os.path.join(outdir, "params-rescued%01d%01d.txt" % (jj, ii))) if (len(rejects) == 0): write_text_row([-1, -1], os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) else: write_text_row( rejects, os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) if (len(rejects) != 0): for ii in range(ll): write_text_row( [ params[ii][rejects[k][1]] for k in range(len(rejects)) ], os.path.join(outdir, "params-rejects%01d%01d.txt" % (jj, ii))) hi = hist_list([perr[j][0] for j in range(nn)], 16) sxprint("Pixel errors for BAD GROUP ", chr(65 + jj)) for ii in range(len(hi[0])): sxprint("%4d %12.3f %12.0f " % (ii, hi[0][ii], hi[1][ii])) else: sxprint("Usage: ") sxprint(""" Phase 1: sxconsistency.py --phase=1 bdb:data outdir output files are: in directory outdir: lili0.txt to lili5.txt contain indices of images in resampled six groups bdb:dataX0 to bdb:dataX5 are metafiles containing six resampled groups of images derived from bdb:data Phase 2 sxconsistency.py --phase=2 outdir --ndigits=1 --params=paramsb howmanythesame.txt outdir - directory containing files lili0.txt to lili5.txt produced in phase 1 --params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be master/main/params0.txt to master/main/params5.txt output files: howmanythesame.txt - contains one number, a ratio of number of images that did not change orientations to the total number of images Phase 3: sxconsistency.py --phase=3 outdir --ou=133 --thresherr=3.0 --params=paramsa outgrouparms input files: outdir - directory containing files lili0.txt to lili5.txt produced in phase 1 --params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be master/main/params0.txt to master/main/params5.txt output files: outgrouparms*.txt - Root of of six parameters files with average 3D orientation parameters computed from six runs, can be imported into bdb:data The next two files contain the original image numbers refering to the top bdb. outdir/newgood.txt - Text file with indices of images whose orientation parameters arrors are below specified thresherr (to be kept) outdir/bad.txt - Text file with indices of images whose orientation parameters arrors are above specified thresherr (to be rejected) Phase 4: sxconsistency.py --phase=4 outdir --ou=133 --thresherr=3.0 --params=master/main001/newbad input files: outdir - directory containing files badparamsij.txt i= 0,3, j=0,2, which resulted from three alignments of four badchunk images --params= - Root of four files with original indices of bad images, they will be read and a subset corresponding to rescued ones will be outputed output files: rescued*.txt - four files with indices of accepted images from badchunk. rejects*.txt - four files with indices of rejected images from badchunk. There are two columns: [number in original bdb:chunk, number in bdb:newbad] """) sxprint("Please run '" + progname + " -h' for detailed options") sp_global_def.BATCH = False
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " stack outdir <maskfile> --K=10 --trials=2 --debug --maxit=100 --rand_seed=10 --crit='all' --F=0.9 --T0=2.0 --init_method='rnd' --normalize --CTF --MPI --CUDA" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--K", type="int", default=2, help="Number of classes (default 2)") parser.add_option("--trials", type="int", default=1, help="Number of trials of K-means (default 1)") parser.add_option("--maxit", type="int", default=100, help="Maximum number of iterations within K-means") parser.add_option("--CTF", action="store_true", default=False, help="Perform classification using CTF information") parser.add_option("--rand_seed", type="int", default=-1, help="Random seed of initial (default random)") parser.add_option( "--crit", type="string", default="D", help= "Criterions: Coleman [C], Harabasz[H], Davies-Bouldin[D], All [all]") #parser.add_option("--F", type="float", default=0.0, help="Cooling in simulated annealing, ex.: 0.9") #parser.add_option("--T0", type="float", default=0.0, help="Initial temperature in simulated annealing, ex: 100") parser.add_option("--MPI", action="store_true", default=False, help="Use MPI version") parser.add_option("--debug", action="store_true", default=False, help="") parser.add_option("--normalize", action="store_true", default=False, help="Normalize images under the mask") parser.add_option( '--init_method', type='string', default='rnd', help= 'Method used to initialize partition: "rnd" randomize or "d2w" for d2 weighting initialization (default is rnd)' ) (options, args) = parser.parse_args() if len(args) < 2 or len(args) > 3: sxprint("usage: " + usage) sxprint("Please run '" + progname + " -h' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return elif options.trials < 1: ERROR("Number of trials should be at least 1") return else: if len(args) == 2: mask = None else: mask = args[2] if options.K < 2: ERROR("K must be > 1 group") return if options.CTF: ERROR("CTF option not implemented") return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import k_means_main sp_global_def.BATCH = True k_means_main(args[0], args[1], mask, "SSE", options.K, options.rand_seed, options.maxit, options.trials, options.crit, options.CTF, 0.0, 0.0, options.MPI, False, options.debug, options.normalize, options.init_method) sp_global_def.BATCH = False
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " proj_stack output_averages --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--img_per_group", type="int", default=100, help="number of images per group") parser.add_option("--radius", type="int", default=-1, help="radius for alignment") parser.add_option( "--xr", type="string", default="2 1", help="range for translation search in x direction, search is +/xr") parser.add_option( "--yr", type="string", default="-1", help= "range for translation search in y direction, search is +/yr (default = same as xr)" ) parser.add_option( "--ts", type="string", default="1 0.5", help= "step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional" ) parser.add_option( "--iter", type="int", default=30, help="number of iterations within alignment (default = 30)") parser.add_option( "--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)") parser.add_option("--thld_err", type="float", default=1.0, help="threshold of pixel error (default = 1.732)") parser.add_option( "--grouping", type="string", default="GRP", help= "do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size" ) parser.add_option( "--delta", type="float", default=-1.0, help="angular step for reference projections (required for GEV method)" ) parser.add_option( "--fl", type="float", default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter") parser.add_option( "--aa", type="float", default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter") parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version") (options, args) = parser.parse_args() myid = mpi.mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi.mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 2: stack = args[0] outdir = args[1] else: sp_global_def.ERROR("Incomplete list of arguments", "sxproj_stability.main", 1, myid=myid) return if not options.MPI: sp_global_def.ERROR("Non-MPI not supported!", "sxproj_stability.main", 1, myid=myid) return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() sp_global_def.BATCH = True img_per_grp = options.img_per_group radius = options.radius ite = options.iter num_ali = options.num_ali thld_err = options.thld_err xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else: yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) if radius == -1: radius = nx / 2 - 2 mask = model_circle(radius, nx, nx) st = time() if options.grouping == "GRP": if myid == main_node: sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") proj_params = [] for i in range(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) # Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta, # So I will briefly explain it here # proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers # except for the last one. Depending on the number of particles left, they will either form a # group or append themselves to the last group # angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi, # theta) is the projection angle of the center of the group, delta is the range of this group # mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates # whether it should take mirror position. # In this program angle_list and mirror list are not of interest. proj_list_all, angle_list, mirror_list = group_proj_by_phitheta( proj_params, img_per_grp=img_per_grp) del proj_params sxprint(" B number of groups ", myid, " ", len(proj_list_all), time() - st) mpi_barrier(MPI_COMM_WORLD) # Number of groups, actually there could be one or two more groups, since the size of the remaining group varies # we will simply assign them to main node. n_grp = nima / img_per_grp - 1 # Divide proj_list_all equally to all nodes, and becomes proj_list proj_list = [] for i in range(n_grp): proc_to_stay = i % number_of_proc if proc_to_stay == main_node: if myid == main_node: proj_list.append(proj_list_all[i]) elif myid == main_node: mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) elif myid == proc_to_stay: img_per_grp = mpi_recv(1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) img_per_grp = int(img_per_grp[0]) temp = mpi_recv(img_per_grp, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) proj_list.append(list(map(int, temp))) del temp mpi_barrier(MPI_COMM_WORLD) sxprint(" C ", myid, " ", time() - st) if myid == main_node: # Assign the remaining groups to main_node for i in range(n_grp, len(proj_list_all)): proj_list.append(proj_list_all[i]) del proj_list_all, angle_list, mirror_list # Compute stability per projection projection direction, equal number assigned, thus overlaps elif options.grouping == "GEV": if options.delta == -1.0: ERROR( "Angular step for reference projections is required for GEV method" ) return from sp_utilities import even_angles, nearestk_to_refdir, getvec refproj = even_angles(options.delta) img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid) # Now each processor keeps its own share of reference projections refprojdir = refproj[img_begin:img_end] del refproj ref_ang = [0.0] * (len(refprojdir) * 2) for i in range(len(refprojdir)): ref_ang[i * 2] = refprojdir[0][0] ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1 sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") # the solution below is very slow, do not use it unless there is a problem with the i/O """ for i in xrange(number_of_proc): if myid == i: proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") mpi_barrier(MPI_COMM_WORLD) """ sxprint(" B ", myid, " ", time() - st) proj_ang = [0.0] * (nima * 2) for i in range(nima): dp = proj_attr[i].get_params("spider") proj_ang[i * 2] = dp["phi"] proj_ang[i * 2 + 1] = dp["theta"] sxprint(" C ", myid, " ", time() - st) asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp) del proj_ang, ref_ang proj_list = [] for i in range(len(refprojdir)): proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp]) del asi sxprint(" D ", myid, " ", time() - st) #from sys import exit #exit() # Compute stability per projection elif options.grouping == "PPR": sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") sxprint(" B ", myid, " ", time() - st) proj_params = [] for i in range(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) sxprint(" C ", myid, " ", time() - st) from sp_utilities import nearest_proj proj_list, mirror_list = nearest_proj( proj_params, img_per_grp, list(range(img_begin, img_begin + 1))) #range(img_begin, img_end)) refprojdir = proj_params[img_begin:img_end] del proj_params, mirror_list sxprint(" D ", myid, " ", time() - st) else: ERROR("Incorrect projection grouping option") return ########################################################################################################### # Begin stability test from sp_utilities import get_params_proj, read_text_file #if myid == 0: # from utilities import read_text_file # proj_list[0] = map(int, read_text_file("lggrpp0.txt")) from sp_utilities import model_blank aveList = [model_blank(nx, ny)] * len(proj_list) if options.grouping == "GRP": refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list) for i in range(len(proj_list)): sxprint(" E ", myid, " ", time() - st) class_data = EMData.read_images(stack, proj_list[i]) #print " R ",myid," ",time()-st if options.CTF: from sp_filter import filt_ctf for im in range(len(class_data)): # MEM LEAK!! atemp = class_data[im].copy() btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1) class_data[im] = btemp #class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: try: t = im.get_attr( "xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) #print " F ",myid," ",time()-st # Here, we perform realignment num_ali times all_ali_params = [] for j in range(num_ali): if (xrng[0] == 0.0 and yrng[0] == 0.0): avet = ali2d_ras(class_data, randomize=True, ir=1, ou=radius, rs=1, step=1.0, dst=90.0, maxit=ite, check_mirror=True, FH=options.fl, FF=options.aa) else: avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa) ali_params = [] for im in range(len(class_data)): alpha, sx, sy, mirror, scale = get_params2D(class_data[im]) ali_params.extend([alpha, sx, sy, mirror]) all_ali_params.append(ali_params) #aveList[i] = avet #print " G ",myid," ",time()-st del ali_params # We determine the stability of this group here. # stable_set contains all particles deemed stable, it is a list of list # each list has two elements, the first is the pixel error, the second is the image number # stable_set is sorted based on pixel error #from utilities import write_text_file #write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid) stable_set, mir_stab_rate, average_pix_err = multi_align_stability( all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1) #print " H ",myid," ",time()-st if (len(stable_set) > 5): stable_set_id = [] members = [] pix_err = [] # First put the stable members into attr 'members' and 'pix_err' for s in stable_set: # s[1] - number in this subset stable_set_id.append(s[1]) # the original image number members.append(proj_list[i][s[1]]) pix_err.append(s[0]) # Then put the unstable members into attr 'members' and 'pix_err' from sp_fundamentals import rot_shift2D avet.to_zero() if options.grouping == "GRP": aphi = 0.0 atht = 0.0 vphi = 0.0 vtht = 0.0 l = -1 for j in range(len(proj_list[i])): # Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses if j in stable_set_id: l += 1 avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3]) if options.grouping == "GRP": phi, theta, psi, sxs, sy_s = get_params_proj( class_data[j]) if (theta > 90.0): phi = (phi + 540.0) % 360.0 theta = 180.0 - theta aphi += phi atht += theta vphi += phi * phi vtht += theta * theta else: members.append(proj_list[i][j]) pix_err.append(99999.99) aveList[i] = avet.copy() if l > 1: l += 1 aveList[i] /= l if options.grouping == "GRP": aphi /= l atht /= l vphi = (vphi - l * aphi * aphi) / l vtht = (vtht - l * atht * atht) / l from math import sqrt refprojdir[i] = [ aphi, atht, (sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0 ] # Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION aveList[i].set_attr('members', members) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', pix_err) else: sxprint(" empty group ", i, refprojdir[i]) aveList[i].set_attr('members', [-1]) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', [99999.]) del class_data if myid == main_node: km = 0 for i in range(number_of_proc): if i == main_node: for im in range(len(aveList)): aveList[im].write_image(args[1], km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in range(nl): ave = recv_EMData(i, im + i + 70000) nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('members', list(map(int, members))) members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('pixerr', list(map(float, members))) members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('refprojdir', list(map(float, members))) ave.write_image(args[1], km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in range(len(aveList)): send_EMData(aveList[im], main_node, im + myid + 70000) members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) members = aveList[im].get_attr('pixerr') mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) except: mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) sp_global_def.BATCH = False mpi_barrier(MPI_COMM_WORLD)
def main(): arglist = [] for arg in sys.argv: arglist.append( arg ) progname = os.path.basename( arglist[0] ) usage = progname + " prj_stack volume [begin end step] --CTF --npad=ntimes_padding --list=file --group=ID --snr=SNR --sym=symmetry --verbose=(0|1) --xysize --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--CTF", action="store_true", default=False, help="apply CTF correction") parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio" ) parser.add_option("--sym", type="string", default="c1", help="symmetry" ) parser.add_option("--list", type="string", help="file with list of images to be used in the first column" ) parser.add_option("--group", type="int", default=-1, help="perform reconstruction using images for a given group number (group is attribute in the header)" ) parser.add_option("--MPI", action="store_true", default=False, help="use MPI version ") parser.add_option("--npad", type="int", default=2, help="number of times padding (default 2)" ) parser.add_option("--verbose", type="int", default=0, help="verbose level: 0 no verbose, 1 verbose" ) parser.add_option("--xysize", type="int", default=-1, help="user expected size at xy direction" ) parser.add_option("--zsize", type="int", default=-1, help="user expected size at z direction" ) parser.add_option("--smearstep", type="float", default=0.0, help="Rotational smear step (default 0.0, no smear)" ) parser.add_option("--interpolation_method", type ="string", default="4nn", help="4nn, or tril: nearest neighbor, or trilinear interpolation" ) parser.add_option("--niter", type="int", default=10, help="number of iterations for iterative reconstruction" ) parser.add_option("--upweighted", action="store_true", default=False, help="apply background noise") parser.add_option("--compensate", action="store_true", default=False, help="compensate in reconstruction") parser.add_option("--chunk_id", type="int", default=-1, help="reconstruct both odd and even groups of particles") parser.add_option("--target_window_size", type="int", default=-1, help=" size of the targeted reconstruction ") (options,args) = parser.parse_args(arglist[1:]) if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() if len(args) == 2: prj_stack = args[0] vol_stack = args[1] nimage = EMUtil.get_image_count( prj_stack ) pid_list = list(range(0, nimage)) elif len(args) == 5: prj_stack = args[0] vol_stack = args[1] begin = atoi( args[2] ) end = atoi( args[3] ) step = atoi( args[4] ) pid_list = list(range(begin, end, step)) else: ERROR( "Incomplete list of arguments" ) return if(options.list and options.group > -1): ERROR( "options group and list cannot be used together" ) return from sp_applications import recons3d_n, recons3d_trl_MPI sp_global_def.BATCH = True if options.interpolation_method == "4nn": recons3d_n(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\ options.sym, options.list, options.group, options.verbose, options.MPI,options.xysize, options.zsize, options.smearstep, options.upweighted, options.compensate,options.chunk_id) elif options.interpolation_method == "tril": if options.MPI is False: ERROR( "Trilinear interpolation reconstruction has MPI version only!" ) return recons3d_trl_MPI(prj_stack, pid_list, vol_stack, options.CTF, options.snr, 1, options.npad,\ options.sym, options.verbose, options.niter, options.compensate, options.target_window_size) else: ERROR( "Wrong interpolation method. The current options are 4nn, and tril. 4nn is the default one." ) return sp_global_def.BATCH = False
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " stack ref_vol outdir <maskfile> --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --ynumber=y_numbers --txs=translational_search_stepx --delta=angular_step --an=angular_neighborhood --center=1 --maxit=max_iter --CTF --snr=1.0 --ref_a=S --sym=c1 --datasym=symdoc --new" parser = OptionParser(usage, version=SPARXVERSION) #parser.add_option("--ir", type="float", default= -1, help="inner radius for rotational correlation > 0 (set to 1) (Angstroms)") parser.add_option( "--ou", type="float", default=-1, help= "outer radius for rotational 2D correlation < int(nx/2)-1 (set to the radius of the particle) (Angstroms)" ) parser.add_option( "--rs", type="int", default=1, help="step between rings in rotational correlation >0 (set to 1)") parser.add_option( "--xr", type="string", default=" 4 2 1 1 1", help= "range for translation search in x direction, search is +/-xr (Angstroms) " ) parser.add_option( "--txs", type="string", default="1 1 1 0.5 0.25", help= "step size of the translation search in x directions, search is -xr, -xr+ts, 0, xr-ts, xr (Angstroms)" ) parser.add_option( "--y_restrict", type="string", default="-1 -1 -1 -1 -1", help= "range for translational search in y-direction, search is +/-y_restrict in Angstroms. This only applies to local search, i.e., when an is not -1. If y_restrict < 0, then for ihrsrlocalcons (option --localcons local search with consistency), the y search range is set such that it is the same ratio to dp as angular search range is to dphi. For regular ihrsr, y search range is the full range when y_restrict< 0. Default is -1." ) parser.add_option( "--ynumber", type="string", default="4 8 16 32 32", help= "even number of the translation search in y direction, search is (-dpp/2,-dpp/2+dpp/ny,,..,0,..,dpp/2-dpp/ny dpp/2]" ) parser.add_option("--delta", type="string", default=" 10 6 4 3 2", help="angular step of reference projections") parser.add_option( "--an", type="string", default="-1", help= "angular neighborhood for local searches (default -1, meaning do exhaustive search)" ) parser.add_option( "--maxit", type="int", default=30, help= "maximum number of iterations performed for each angular step (default 30) " ) parser.add_option("--CTF", action="store_true", default=False, help="CTF correction") parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio of the data (default 1)") parser.add_option("--MPI", action="store_true", default=True, help="use MPI version") #parser.add_option("--fourvar", action="store_true", default=False, help="compute Fourier variance") parser.add_option("--apix", type="float", default=-1.0, help="pixel size in Angstroms") parser.add_option("--dp", type="float", default=-1.0, help="delta z - translation in Angstroms") parser.add_option("--dphi", type="float", default=-1.0, help="delta phi - rotation in degrees") parser.add_option( "--ndp", type="int", default=12, help= "In symmetrization search, number of delta z steps equals to 2*ndp+1") parser.add_option( "--ndphi", type="int", default=12, help="In symmetrization search,number of dphi steps equas to 2*ndphi+1" ) parser.add_option("--dp_step", type="float", default=0.1, help="delta z (Angstroms) step for symmetrization") parser.add_option("--dphi_step", type="float", default=0.1, help="dphi step for symmetrization") parser.add_option( "--psi_max", type="float", default=10.0, help= "maximum psi - how far rotation in plane can can deviate from 90 or 270 degrees (default 10)" ) parser.add_option("--rmin", type="float", default=0.0, help="minimal radius for hsearch (Angstroms)") parser.add_option("--rmax", type="float", default=80.0, help="maximal radius for hsearch (Angstroms)") parser.add_option("--fract", type="float", default=0.7, help="fraction of the volume used for helical search") parser.add_option("--sym", type="string", default="c1", help="symmetry of the structure") parser.add_option("--function", type="string", default="helical", help="name of the reference preparation function") parser.add_option("--datasym", type="string", default="datasym.txt", help="symdoc") parser.add_option( "--nise", type="int", default=200, help="start symmetrization after nise steps (default 200)") parser.add_option("--npad", type="int", default=2, help="padding size for 3D reconstruction, (default 2)") parser.add_option("--debug", action="store_true", default=False, help="debug") parser.add_option("--new", action="store_true", default=False, help="use rectangular recon and projection version") parser.add_option( "--initial_theta", type="float", default=90.0, help="intial theta for reference projection (default 90)") parser.add_option( "--delta_theta", type="float", default=1.0, help="delta theta for reference projection (default 1.0)") parser.add_option("--WRAP", type="int", default=1, help="do helical wrapping (default 1, meaning yes)") (options, args) = parser.parse_args(arglist[1:]) if len(args) < 1 or len(args) > 5: sxprint("usage: " + usage + "\n") sxprint("Please run '" + progname + " -h' for detailed options") ERROR( "Invalid number of parameters used. please see usage information above." ) return else: # Convert input arguments in the units/format as expected by ihrsr_MPI in applications. if options.apix < 0: ERROR("Please enter pixel size") return rminp = int((float(options.rmin) / options.apix) + 0.5) rmaxp = int((float(options.rmax) / options.apix) + 0.5) from sp_utilities import get_input_from_string, get_im xr = get_input_from_string(options.xr) txs = get_input_from_string(options.txs) y_restrict = get_input_from_string(options.y_restrict) irp = 1 if options.ou < 0: oup = -1 else: oup = int((options.ou / options.apix) + 0.5) xrp = '' txsp = '' y_restrict2 = '' for i in range(len(xr)): xrp += " " + str(float(xr[i]) / options.apix) for i in range(len(txs)): txsp += " " + str(float(txs[i]) / options.apix) # now y_restrict has the same format as x search range .... has to change ihrsr accordingly for i in range(len(y_restrict)): y_restrict2 += " " + str(float(y_restrict[i]) / options.apix) if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import ihrsr sp_global_def.BATCH = True if len(args) < 4: mask = None else: mask = args[3] ihrsr(args[0], args[1], args[2], mask, irp, oup, options.rs, xrp, options.ynumber, txsp, options.delta, options.initial_theta, options.delta_theta, options.an, options.maxit, options.CTF, options.snr, options.dp, options.ndp, options.dp_step, options.dphi, options.ndphi, options.dphi_step, options.psi_max, rminp, rmaxp, options.fract, options.nise, options.npad, options.sym, options.function, options.datasym, options.apix, options.debug, options.MPI, options.WRAP, y_restrict2) sp_global_def.BATCH = False
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + """ Input Output [options] Generate three micrographs, each micrograph contains one projection of a long filament. Input: Reference Volume, output directory Output: Three micrographs stored in output directory sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84 Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200 sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35 Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60 Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams Generate run through example script for helicon sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35 """ parser = OptionParser(usage, version=SPARXVERSION) # helicise the Atom coordinates # generate micrographs of helical filament parser.add_option( "--generate_micrograph", action="store_true", default=False, help= "Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory" ) parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction") parser.add_option("--apix", type="float", default=-1, help="pixel size in Angstroms") parser.add_option( "--rand_seed", type="int", default=14567, help= "the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs." ) parser.add_option("--Cs", type="float", default=2.0, help="Microscope Cs (spherical aberation)") parser.add_option("--voltage", type="float", default=200.0, help="Microscope voltage in KV") parser.add_option("--ac", type="float", default=10.0, help="Amplitude contrast (percentage, default=10)") parser.add_option("--nonoise", action="store_true", default=False, help="Do not add noise to the micrograph.") # generate initial volume parser.add_option("--generate_noisycyl", action="store_true", default=False, help="Generate initial volume of noisy cylinder.") parser.add_option( "--boxsize", type="string", default="100,100,200", help= "String containing x , y, z dimensions (separated by comma) in pixels") parser.add_option("--rad", type="int", default=35, help="Radius of initial volume in pixels") # generate 2D mask parser.add_option("--generate_mask", action="store_true", default=False, help="Generate 2D rectangular mask.") parser.add_option( "--masksize", type="string", default="200,200", help= "String containing x and y dimensions (separated by comma) in pixels") parser.add_option("--maskwidth", type="int", default=60, help="Width of rectangular mask") # Apply 2D alignment parameters to input stack and output new images to output stack parser.add_option( "--applyparams", action="store_true", default=False, help= "Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack" ) # Generate run script parser.add_option("--generate_script", action="store_true", default=False, help="Generate script for helicon run through example") parser.add_option("--filename", type="string", default="runhelicon", help="Name of run script to generate") parser.add_option("--seg_ny", type="int", default=180, help="y-dimension of segment used for refinement") parser.add_option( "--ptcl_dist", type="int", default=15, help= "Distance in pixels between adjacent segments windowed from same filament" ) parser.add_option( "--fract", type="float", default=0.35, help="Fraction of the volume used for applying helical symmetry.") (options, args) = parser.parse_args() if len(args) > 3: sxprint("usage: " + usage) sxprint("Please run '" + progname + " -h' for detailed options") ERROR( "Invalid number of parameters. Please see usage information above." ) return else: if options.generate_script: generate_runscript(options.filename, options.seg_ny, options.ptcl_dist, options.fract) if options.generate_micrograph: if options.apix <= 0: ERROR("Please enter pixel size.") return generate_helimic(args[0], args[1], options.apix, options.CTF, options.Cs, options.voltage, options.ac, options.nonoise, options.rand_seed) if options.generate_noisycyl: from sp_utilities import model_cylinder, model_gauss_noise outvol = args[0] boxdims = options.boxsize.split(',') if len(boxdims) < 1 or len(boxdims) > 3: ERROR( "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize=\'100,100,200\'" ) return nx = int(boxdims[0]) if len(boxdims) == 1: ny = nx nz = nx else: ny = int(boxdims[1]) if len(boxdims) == 3: nz = int(boxdims[2]) (model_cylinder(options.rad, nx, ny, nz) * model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol) if options.generate_mask: from sp_utilities import model_blank, pad outvol = args[0] maskdims = options.masksize.split(',') if len(maskdims) < 1 or len(maskdims) > 2: ERROR( "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize=\'200,200\'" ) return nx = int(maskdims[0]) if len(maskdims) == 1: ny = nx else: ny = int(maskdims[1]) mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1, 0.0) mask.write_image(outvol) if options.applyparams: from sp_utilities import get_im, get_params2D, set_params2D from sp_fundamentals import cyclic_shift stack = args[0] newstack = args[1] mask = get_im(args[2]) nima = EMUtil.get_image_count(stack) for im in range(nima): prj = get_im(stack, im) alpha, sx, sy, mirror, scale = get_params2D(prj) prj = cyclic_shift(prj, int(sx)) set_params2D(prj, [0.0, 0., 0.0, 0, 1]) stat = Util.infomask(prj, mask, False) prj = (prj - stat[0]) / stat[1] ctf_params = prj.get_attr("ctf") prj.set_attr('ctf_applied', 0) prj.write_image(newstack, im)
def generate_runscript(filename, seg_ny, ptcl_dst, fract): if ptcl_dst < 15: ERROR( "Distance in pixels between adjacent segments should be at least one rise!" ) return sxprint("Generating run script with the following parameters: \n") sxprint("y-dimension of segment used for refinement: %d" % seg_ny) sxprint("Distance in pixels between adjacent segments: %d" % ptcl_dst) sxprint("Fraction of structure used for applying helical symmetry: %.2f" % fract) if os.path.exists(filename): ERROR("File " + filename + " already exists. Please either remove or rename the file") return f = open(filename, 'w') f.write('#!/bin/csh\n') f.write('\n') f.write('set echo on\n') f.write('\n') f.write('#clean the previous outputs\n') f.write('rm *.hdf *.txt *.bck rm *.pdb\n') f.write('rm log*\n') #f.write('rm -rf outsymsearch \n') f.write('rm -rf mic result_* \n') f.write('rm -rf EMAN*\n') f.write('\n') f.write('# get the previous runned results\n') f.write('tar -zxvf answer.tar.gz\n') f.write('\n') f.write('#generate volume from pdb\n') f.write('tar -zxvf 3MFP_1SU.tar.gz\n') f.write('\n') f.write('# Helicise the Atom coordinates\n') f.write( '# Input: pdb file containing atom coordinates to helicise (3MFP_1SU.pdb)\n' ) f.write('# Output: pdb file containing helicised coordinates (rnew.pdb)\n') f.write( 'sxhelical_demo.py 3MFP_1SU.pdb rnew.pdb --heli --dp=27.6 --dphi=166.715\n' ) f.write('\n') f.write('#generate the density map\n') f.write('sxpdb2em.py rnew.pdb tmp.hdf --apix=1.84 --center=c \n') f.write('\n') f.write( '# Generate three micrographs, where each micrograph contains one projection of a long filament.\n' ) f.write( '# Input: Reference volume from which projections are calculated (tmp.hdf)\n' ) f.write( '# Output: Output directory in which micrographs will be written (mic)\n' ) f.write( 'sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84\n' ) f.write('\n') f.write('# generate initial volume for later helical refinement\n') f.write( '# Output: A noisy cylinder written to the output file name (ini.hdf).\n' ) f.write( 'sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35\n' ) f.write('\n') f.write( '# Estimate defocus value for each micrograph. This can be done either by GUI or pure command-line \n' ) f.write('# 1. To estimate defocus by GUI:\n') f.write('cd mic\n') f.write('sxhelixboxer.py mic0.hdf --gui &\n') f.write('\n') f.write( "# When the GUI starts up, there will be a checkbox at the bottom labelled 'CTF Estimation using CTER'. Check this box, and additional fields will appear in the GUI.\n" ) f.write( "# Under 'Parameters of CTF estimation,' enter 256 for 'Window size,' 2.0 for 'Cs, 10.0 for 'Amplitude Contrast,' 200 for 'Voltage', and 1.84 for Pixel size.\n" ) f.write( '# Click "Estimate CTF using CTER" button, and the estimated defocus will show up in the "Estimated defocus" box.\n' ) f.write( '# Now you can either estimate defocus for the remaining micrographs one by one using the GUI, or you can estimate defocus for the remaining micrographs in batch mode using \n' ) f.write('# the parameters you entered in the GUI for mic0.hdf. \n') f.write( '# Make sure to remove any output directories (pwrot and partres followed by underscore and \n' ) f.write( '# then name of the micrograph, e.g., pwrot_mic0 and partres_mic0) generated in the previous run of CTF estimation using CTER.\n' ) f.write('rm -r pwrot_mic0;rm -r partres_mic0\n') f.write('sxhelixboxer.py mic*.hdf --gui &\n') f.write('cd ..\n') f.write('\n') f.write('# 2. To estimate defocus by command-line\n') f.write('cd mic\n') f.write( 'sxhelixboxer.py pwrot partres --cter --indir=. --nameroot=mic --nx=200 --Cs=2.0 --voltage=200 --kboot=16 --apix=1.84 --ac=10.0 \n' ) f.write('cd ..\n') f.write('\n') f.write('cd mic\n') f.write('# have to open boxer\n') f.write( '# if want to save time, just close the window immediately and use saved coordinate\n' ) f.write('# otherwise draw the box carefully to get the coordinate \n') f.write('sxhelixboxer.py *.hdf --gui --helix-width=200 --ptcl-width=200\n') f.write('cd ..\n') f.write('\n') f.write('# if use saved coordinate, please use below commands\n') f.write('tar -zxvf saved_pos.tar.gz\n') f.write('cp saved_db.tar.gz mic/.\n') f.write('cp -r saved_pos/*box* mic/.\n') f.write('rm -rf saved_pos\n') f.write('cd mic\n') f.write('tar -zxvf saved_db.tar.gz\n') f.write('cd ..\n') f.write('\n') f.write('# Window segments from boxed helices\n') f.write( '# For more information on the windowing utility, see http://sparx-em.org/sparxwiki/windowallmic\n' ) f.write('# distance between segments is 1 rise exactly\n') f.write( 'sxhelixboxer.py houtdir --window --dirid=mic --micid=mic --micsuffix=hdf --dp=27.6 --apix=1.84 --boxsize=200 --outstacknameall=bdb:hadata --hcoords_suffix=_boxes.txt --ptcl-dst=%d --rmax=64\n' % ptcl_dst) f.write('\n') f.write('# Generate 2D mask for centering EM images\n') f.write( '# Output: 2D mask written to output file name provided (mask2d.hdf)\n' ) f.write( 'sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=70\n' ) f.write('\n') f.write('# center all the EM images\n') f.write( '# centering parameters will be saved in header, the input images will not be changed.\n' ) # horatio active_refactoring Jy51i1EwmLD4tWZ9_00000_1 # f.write('sxheader.py bdb:hadata --params=active --one\n') f.write('sxheader.py bdb:hadata --params=xform.align2d --zero\n') f.write( 'mpirun -np 2 sxshiftali.py bdb:hadata mask2d.hdf --oneDx --search_rng=10 --maxit=20 --CTF --MPI\n' ) f.write('\n') f.write( '# Apply the centering parameters stored in the header of each image in bdb:hadata to the image, \n' ) f.write( '# normalize using average and standard deviation outside the mask, and save the centered and normalized image to bdb:hdata\n' ) f.write( '# Input: Input stack containing 2D alignment parameters in the header (bdb:hadata)\n' ) f.write('# Name of 2D mask to use for normalization\n') f.write( '# Output: Stack of images (bdb:hdata) after applying 2D alignment parameters to the images in input stack. \n' ) f.write( 'sxhelical_demo.py bdb:hadata bdb:hdata mask2d.hdf --applyparams\n') f.write('\n') f.write('#Exhaustive search \n') f.write('sxheader.py bdb:hdata --params=xform.projection --zero\n') # horatio active_refactoring Jy51i1EwmLD4tWZ9_00000_1 # f.write('sxheader.py bdb:hdata --params=active --one\n') f.write( 'mpirun -np 3 sxhelicon.py bdb:hdata ini.hdf result_helicon --CTF --seg_ny=%d --fract=%.2f --psi_max=2.0 --delta=1.0 --maxit=7 --function=[.,nofunc,helical3c] --searchxshift=3.68 --xwobble=1.84 --ywobble=0 --dp=27.6 --dphi=166.715 --apix=1.84 --rmin=0.0 --rmax=64 --MPI\n' % (seg_ny, fract)) f.write('\n') f.write('#Local search \n') f.write( 'sxheader.py bdb:hdata --params=xform.projection --import=result_helicon/parameters0007.txt\n' ) f.write( 'mpirun -np 3 sxlocalhelicon.py bdb:hdata result_helicon/volf007.hdf result_local --CTF --seg_ny=%d --fract=%.2f --psi_max=2.0 --delta=1.0 --maxit=11 --function=[.,nofunc,helical3c] --boundaryavg --MA --MA_WRAP=0 --xr=3.68 --txs=1.84 --an=20 --ynumber=16 --dp=27.6 --dphi=166.715 --apix=1.84 --rmin=0.0 --rmax=64 --MPI\n' % (seg_ny, fract)) f.write('\n')
def generate_helimic(refvol, outdir, pixel, CTF=False, Cs=2.0, voltage=200.0, ampcont=10.0, nonoise=False, rand_seed=14567): from sp_utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im from random import random from sp_projection import prgs, prep_vol from sp_filter import filt_gaussl, filt_ctf from EMAN2 import EMAN2Ctf if os.path.exists(outdir): ERROR( "Output directory exists, please change the name and restart the program" ) return os.mkdir(outdir) seed(rand_seed) Util.set_randnum_seed(rand_seed) angles = [] for i in range(3): angles.append([0.0 + 60.0 * i, 90.0 - i * 5, 0.0, 0.0, 0.0]) nangle = len(angles) volfts = get_im(refvol) nx = volfts.get_xsize() ny = volfts.get_ysize() nz = volfts.get_zsize() volfts, kbx, kby, kbz = prep_vol(volfts) iprj = 0 width = 500 xstart = 0 ystart = 0 for idef in range(3, 6): mic = model_blank(2048, 2048) #defocus = idef*0.2 defocus = idef * 0.6 ##@ming if CTF: #ctf = EMAN2Ctf() #ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} ) from sp_utilities import generate_ctf ctf = generate_ctf( [defocus, 2, 200, 1.84, 0.0, ampcont, defocus * 0.2, 80] ) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice. i = idef - 4 for k in range(1): psi = 90 + 10 * i proj = prgs( volfts, kbz, [angles[idef - 3][0], angles[idef - 3][1], psi, 0.0, 0.0], kbx, kby) proj = Util.window(proj, 320, nz) mic += pad(proj, 2048, 2048, 1, 0.0, 750 * i, 20 * i, 0) if not nonoise: mic += model_gauss_noise(30.0, 2048, 2048) if CTF: #apply CTF mic = filt_ctf(mic, ctf) if not nonoise: mic += filt_gaussl(model_gauss_noise(17.5, 2048, 2048), 0.3) mic.write_image("%s/mic%1d.hdf" % (outdir, idef - 3), 0)