def main(): usage = """e2tomopreproc.py <imgs> <options> . This program takes a tiltseries ('.st' or '.ali' file from IMOD) and applies preprocessing operations to them, such as lowpass, highpass, masking, etc. The options should be supplied in "--option=value" format, replacing "option" for a valid option name, and "value" for an acceptable value for that option. """ parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_argument("--path", type=str, default='', help="""Directory to store results in. The default is a numbered series of directories containing the prefix 'tomopreproc'; for example, tomopreproc_02 will be the directory by default if 'tomopreproc_01' already exists.""") parser.add_pos_argument(name="stack_files", default="", help="Stacks or images to process.") parser.add_argument( "--input", type=str, default='', help= """"tiltseries to process. redundant with --tiltseries, or with providing images as arguments (separated by a space: e2tomopreproc.py stack1.hdf stack2.hdf), but --input takes precedence.""" ) parser.add_argument( "--tiltseries", type=str, default='', help=""""tiltseries to process. redundant with --input""") parser.add_argument( "--tltfile", type=str, default='', help="""".tlt file containing the tilt angles for --tiltseries""") parser.add_argument( "--outmode", type=str, default='', help= """All EMAN2 programs write images with 4-byte floating point values when possible by default. This allows specifying an alternate format when supported: float, int8, int16, int32, uint8, uint16, uint32. Values are rescaled to fill MIN-MAX range.""" ) parser.add_argument( "--dontcleanup", action='store_true', default=False, help="""If specified, intermediate files will be kept.""") parser.add_argument( "--clip", type=str, default='', help= """Default=None. This resizes the 2-D images in the tilt series. If one number is provided, then x and y dimensions will be made the same. To specify both dimensions, supply two numbers, --clip=x,y. Clipping will be about the center of the image.""" ) #parser.add_argument("--apix",type=float,default=0.0,help="""True apix of images to be written on final stack.""") parser.add_argument( "--shrink", type=float, default=0.0, help= """Default=0.0 (no shrinking). Can use decimal numbers, larger than 1.0. Optionally shrink the images by this factor. Uses processor math.fft.resample.""" ) parser.add_argument( "--threshold", type=str, default='', help="""Default=None. A threshold processor applied to each image.""") parser.add_argument( "--erasegold", action='store_true', default='', help="""Default=False. Runs erase_gold.py on the stack.""") parser.add_argument( "--mask", type=str, default='', help="""Default=None. Masking processor applied to each image.""") parser.add_argument( "--maskbyangle", action='store_true', default=False, help= """Default=False. Requires --tltfile. This will mask out from tilted images the info that isn't present at the 0 tilt angle. It uses the tomo.tiltedgemask processor (type 'e2help.py processors' at the commandline to read a description of the processor and its parameters). Provide --maskbyanglefalloff and --maskbyanglesigma to modify the default parameters.""" ) parser.add_argument( "--maskbyanglefalloff", type=int, default=4, help= """Default=4. Number of pixels over which --maskbyangle will fall off to zero.""" ) parser.add_argument( "--maskbyanglesigma", type=float, default=2.0, help= """Default=2.0. Number of sigmas for the width of the gaussian fall off in --maskbyangle and --maskbyanglefalloff""" ) parser.add_argument( "--normproc", type=str, default='', help= """Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to each image.""" ) parser.add_argument( "--normalizeimod", action='store_true', default=False, help= """Default=False. This will apply 'newstack -float 2' to the input stack. Requires IMOD.""" ) parser.add_argument( "--preprocess", type=str, default='', help= """Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each image.""" ) parser.add_argument( "--lowpassfrac", type=float, default=0.0, help= """Default=0.0 (not used). Fraction of Nyquist to lowpass at. The processor used is filter.lowpass.tanh""" ) parser.add_argument( "--highpasspix", type=int, default=0, help= """Default=0 (not used). Number of Fourier pixels to apply highpass filter at. The processor used is filter.highpass.gauss.""" ) parser.add_argument( "--parallel", type=str, default="thread:1", help= """default=thread:1. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel""" ) parser.add_argument( "--prenadminite", type=int, default=0, help= """Default=0. Requires IMOD to be installed. Used to apply prenad filtering to a tiltseries. This is the --minite parameter in IMOD's preNAD program (minimum number of iterations).""" ) parser.add_argument( "--prenadmaxite", type=int, default=0, help= """Default=0. Requires IMOD to be installed. Used to apply prenad filtering to a tiltseries. This is the --maxite parameter in IMOD's preNAD program (maximum number of iterations).""" ) parser.add_argument( "--prenadsigma", type=int, default=0, help= """Default=0. Requires IMOD to be installed. Used to apply prenad filtering to a tiltseries. This is the --sigma parameter in IMOD's preNAD program (initial sigma for 'smoothing structure tensor').""" ) parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help= "verbose level [0-9], higner number means higher level of verboseness." ) parser.add_argument( "--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-1) (options, args) = parser.parse_args() logger = E2init(sys.argv, options.ppid) print "\n(e2tomopreproc)(main) started log" from e2spt_classaverage import sptmakepath options = sptmakepath(options, 'tomopreproc') #print "args are",args infiles = [] if not options.input: #try: # infiles.append( sys.argv[1] ) #except: if options.tiltseries: infiles.append(options.tiltseries) else: if args: print "copying args to infiles" infiles = list(args) print "infiles are", infiles else: print "\n(e2tomopreproc)(main) ERROR: must provide input files as arguments or via the --input or --tiltseries parameters." elif options.input: infiles.append(options.input) if infiles: print "\n(e2tomopreproc)(main) identified --input", options.input #print " .ali in options.input[:-4]", '.ali' in options.input[-4:] #print "options.input[-4] is", options.input[-4:] for infile in infiles: if '.ali' in infile[-4:] or '.st' in infile[ -3:] or '.mrc' in infile[-4:] or '.mrcs' in infile[ -5:] or '.hdf' in infile[-4:]: pass else: print "\n(e2tomopreproc)(main) ERROR: invalid image extension %s for image %s. Extension must be .st, .ali, .hdf, .mrc or .mrcs" % ( options.input.split('.')[-1], infile) sys.exit(1) else: print "\n(e2tomopreproc)(main) ERROR: no images found/provided" sys.exit(1) originalextension = infiles[0].split('.')[-1] angles = {} if options.maskbyangle or (options.prenadminite and options.prenadmaxite and options.prenadsigma): if not options.tltfile: print "\n(e2tomopreproc)(main) ERROR: --maskbyangle and --prenad parameters require --tltfile" sys.exit(1) else: f = open(options.tltfile, 'r') lines = f.readlines() print "\nnumber of lines read from --tltfile", len(lines) f.close() #print "lines in tlt file are", lines k = 0 for line in lines: line = line.replace('\t', '').replace('\n', '') if line: angle = float(line) angles.update({k: angle}) if options.verbose: print "appending angle", angle k += 1 if len(angles) < 2: print "\nERROR: something went terribly wrong with parsing the --tltlfile. This program does not work on single images" sys.exit() if len(angles) < 2: print "\nERROR: (second angle check) something went terribly wrong with parsing the --tltlfile. This program does not work on single images" sys.exit() print "\n(e2spt_preproc)(main) - INITIALIZING PARALLELISM!\n" from EMAN2PAR import EMTaskCustomer etc = EMTaskCustomer(options.parallel) pclist = [options.input] etc.precache(pclist) print "\n(e2spt_preproc)(main) - precaching --input" tasks = [] results = [] mrcstacks = [] print "there are these many infiles to loop over", len(infiles) if options.lowpassfrac: hdr = EMData(infiles[0], 0, True) apix = hdr['apix_x'] print "\n(e2spt_preproc)(main) apix is", apix nyquist = 2.0 * apix print "\n(e2spt_preproc)(main) therefore nyquist resolution is", nyquist print lowpassres = nyquist / options.lowpassfrac options.lowpassfrac = 1.0 / (lowpassres) if float(options.shrink) > 1.0: options.lowpassfrac /= float(options.shrink) print "there's shrinking", options.shrink lowpassres = nyquist / options.lowpassfrac print "\n(e2spt_preproc)(main) and final lowpass frequency is", options.lowpassfrac print "corresponding to lowpassres of", lowpassres for infile in infiles: mrcstack = options.path + '/' + infile print "infile is", infile print "infile[-5:] is ", infile[-5:] if '.hdf' in infile[-5:]: print "replacing .hdf extension" mrcstack = options.path + '/' + infile.replace('.hdf', '.mrc') if '.mrcs' in infile[-5:]: print "replacing .mrcs extension" mrcstack = options.path + '/' + infile.replace('.mrcs', '.mrc') if '.st' in infile[-5:]: print "replacing .st extension" mrcstack = options.path + '/' + infile.replace('.st', '.mrc') if '.ali' in infile[-5:]: print "replacing .ali extension" mrcstack = options.path + '/' + infile.replace('.ali', '.mrc') if '.tif' in infile[-5:]: print "replacing .ali extension" mrcstack = options.path + '/' + infile.replace('.tif', '.mrc') #go = 0 #if go: print "mrcstack is", mrcstack #outname = outname.replace('.mrc','.mrcs') mrcstacks.append(mrcstack) go = 0 if options.maskbyangle: outname = mrcstack.replace('.mrc', '_UNSTACKED.mrc') print "therefore, outname is", outname cmd = 'e2proc2d.py ' + infile + ' ' + outname + ' --unstacking --threed2twod' #from shutil import copyfile #copyfile(options.input, outname) #print "copied input to", outname if options.outmode: cmd += ' --outmode=' + options.outmode if options.verbose: cmd += ' --verbose=' + str(options.verbose) print "\ncommand to unstack original input tiltseries is", cmd print "\n(e2tomopreproc)(main) unstacking command is", cmd p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) #p = subprocess.Popen( cmd , shell=True, stdout=subprocess.PIPE) text = p.communicate() #p.stdout.close() p.wait() if p.returncode == 0: go = 1 else: go = 1 if go: imgs = [] if options.maskbyangle: c = os.getcwd() + '/' + options.path findir = os.listdir(os.getcwd() + '/' + options.path) print "\n(e2tomopreproc)(main) directory to look for images is", c for f in findir: #if '.mrcs' in f: if "_UNSTACKED" in f: imgs.append(options.path + '/' + f) kk = 0 imgs.sort() print "\n(e2spt_preproc)(main) found these many images", len( imgs) for img in imgs: #task=None #if options.maskbyangle: outimage = img.replace('.mrc', '_preproc.mrc') task = TomoPreproc2DTask(img, options, angles[kk], outimage) tasks.append(task) kk += 1 else: outimage = options.path + '/' + infile.replace( '.mrc', '_preproc.mrcs') task = TomoPreproc2DTask(infile, options, 0, outimage) tasks.append(task) #else: # newmrcs = mrcstack.replace('.mrc','.mrcs') # print "copying file %s to %s" %(infile,newmrcs) # copyfile( infile, newmrcs ) # imgs.append( newmrcs ) #print "and the final lowpass frequency will be", options.lowpassfrac tids = etc.send_tasks(tasks) if options.verbose: print "\n(e2spt_preproc)(main) preprocessing %d tasks queued" % ( len(tids)) results = get_results(etc, tids, options) print "\n(e2tomopreproc)(main) these many images have been processsed", len( results) imgspreproc = [] findir = os.listdir(os.getcwd() + '/' + options.path) #for mrcstack in mrcstacks: for f in findir: if "_preproc.mrc" in f: print "found preprocessed image", f imgspreproc.append(options.path + '/' + f) else: print "this file is NOT a preprocessed image", f imgspreproc.sort() print "\n(e2tomopreproc)(main) these many preprocessed images loaded", len( imgspreproc) finalfiles = [] if options.maskbyangle: outfile = mrcstack.replace('.mrc', '.mrcs') print "for RESTACKING" print "\n\n\noutfile is", outfile for f in imgspreproc: print "appending image %s to outfile %s" % (f, outfile) cmd = 'e2proc2d.py ' + f + ' ' + outfile if options.outmode: cmd += ' --outmode=' + options.outmode if options.verbose: cmd += ' --verbose ' + str(options.verbose) print "\ncmd is with .mrcs outputformat is", cmd print "becauase outfile is", outfile p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) text = p.communicate() p.stdout.close() finaloutput = outfile.replace('.mrcs', '.' + originalextension) os.rename(outfile, finaloutput) finalfiles.append(finaloutput) else: finalfiles = list(imgspreproc) for finalf in finalfiles: if not options.tltfile: break if options.normalizeimod: try: cmd = 'newstack ' + finalf + ' ' + finalf + ' --float 2' print "normalizeimod cmd is", cmd p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) text = p.communicate() p.wait() except: print "\nERROR: --normalizeimod skipped. Doesn't seem like IMOD is installed on this machine" if not options.dontcleanup and options.maskbyangle: purge(options.path, '_preproc.mrc') purge(options.path, '_UNSTACKED') purge(options.path, '~') if options.tltfile: if options.prenadminite or options.prenadmaxite or options.prenadsigma: if options.prenadminite and options.prenadmaxite and options.prenadsigma: cmd = 'preNAD -input ' + finalf + ' -output ' + finalf.replace( '.' + originalextension, '_prenad.' + originalextension) + ' -minite ' + str( options.prenadminite) + ' -maxite ' + str( options.prenadmaxite) + ' -sigma ' + str( options.prenadsigma ) + ' -angles ' + options.tltfile if options.verbose: print "\n(e2tomopreproc)(main) prenad cmd to run is", cmd try: p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) text = p.communicate() p.wait() except: print "\nERROR: check that a version of IMOD containing the preNAD program is correctly installed on this machine" else: if options.prenadminite: if not options.prenadmaxite: print "\nERROR: --prenadmaxite required with --prenadminite" if not options.prenadsigma: print "\nERROR: --prenadsigma required with --prenadminite" if options.prenadmaxite: if not options.prenadminite: print "\nERROR: --prenadminite required with --prenadmaxite" if not options.prenadsigma: print "\nERROR: --prenadsigma required with --prenadmaxite" if options.prenadsigma: if not options.prenadminite: print "\nERROR: --prenadminite required with --prenadsigma" if not options.prenadmaxite: print "\nERROR: --prenadmaxite required with --prenadsigma" E2end(logger) return ()
def main(): progname = os.path.basename(sys.argv[0]) usage = """prog <output> [options] This program is used to preprocess subtomograms before aligning them. The same can be accomplished with e2proc3d, except that this program is parallelized and thus should be substantially faster for large subtomograms. """ parser = EMArgumentParser(usage=usage,version=EMANVERSION) parser.add_argument("--input", type=str, default='',help="""Default=None. The name of the input volume stack. MUST be HDF since volume stack support is required.""") parser.add_argument("--output", type=str, default='',help="""Default=None. Specific name of HDF file to write processed particles to.""") parser.add_argument("--parallel",type=str, default='', help="""default=None. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel""") parser.add_argument("--ppid", type=int, help="""Default=-1. Set the PID of the parent process, used for cross platform PPID""",default=-1) parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="""Default=0. Verbose level [0-9], higner number means higher level of verboseness""") parser.add_argument("--subset",type=int,default=0,help="""Default=0 (not used). Refine only this substet of particles from the stack provided through --input""") parser.add_argument("--apix",type=float,default=0.0,help="""Default=0.0 (not used). Use this apix value where relevant instead of whatever is in the header of the reference and the particles. Will overwrite particle header as well.""") parser.add_argument("--shrink", type=int,default=0,help="""Default=0 (no shrinking). Optionally shrink the input volumes by an integer amount for coarse alignment.""") parser.add_argument("--threshold",type=str,default='',help="""Default=None. A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""") parser.add_argument("--mask",type=str,default='', help="""Default=None. Masking processor applied to particles before alignment. IF using --clip, make sure to express outer mask radii as negative pixels from the edge.""") parser.add_argument("--maskfile",type=str,default='',help="""Default=None. Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None.""") parser.add_argument("--normproc",type=str, default='',help="""Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to particles before alignment. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'""") parser.add_argument("--preprocess",type=str,default='',help="""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""") parser.add_argument("--lowpass",type=str,default='',help="""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""") parser.add_argument("--highpass",type=str,default='',help="""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""") parser.add_argument("--clip",type=int,default=0,help="""Default=0 (which means it's not used). Boxsize to clip particles. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary.""") parser.add_argument("--nopath",action='store_true',default=False,help="""If supplied, this option will save results in the directory where the command is run. A directory to store the results will not be made.""") parser.add_argument("--path",type=str,default='sptpreproc',help="""Default=spt. Directory to store results in. The default is a numbered series of directories containing the prefix 'sptpreproc'; for example, sptpreproc_02 will be the directory by default if 'sptpreproc_01' already exists.""") (options, args) = parser.parse_args() logger = E2init(sys.argv, options.ppid) print "\n(e2spt_preproc)(main) started log" from e2spt_classaverage import sptmakepath if options.path and not options.nopath: options = sptmakepath(options,'sptpreproc') if options.parallel=='None' or options.parallel=='none': options.parallel=None if not options.input: try: options.input = sys.argv[1] except: print "\n(e2spt_preproc)(main) ERROR: invalid input file" if options.mask or options.maskfile or options.threshold or options.clip or options.threshold or options.normproc or options.preprocess or options.lowpass or options.highpass or int(options.shrink) > 1: preprocstack = str(os.path.basename(options.input).replace('.hdf','_preproc.hdf')) if options.path and not options.nopath: preprocstack = options.path + '/' + preprocstack if options.output: if '.hdf' in options.output[-4:]: preprocstack = options.output else: print "\n(e2spt_preproc)(main) ERROR: '.hdf' must be the last four characters of the output filename." print "\n(e2spt_preproc)(main) output stack will be %s" %( preprocstack) n = 0 try: n = EMUtil.get_image_count( options.input ) except: print "\n(e2spt_preproc)(main) ERROR: --input stack seems to be invalid" sys.exit() print "\n(e2spt_preproc)(main) number of particles is %d" %( n) c = os.getcwd() findir = os.listdir( c ) if preprocstack not in findir: dimg = EMData(8,8,8) dimg.to_one() for i in range(n): dimg.write_image( preprocstack, i ) else: print "\n(e2spt_preproc)(main) WARNING: a file with the name of the output stack %s is already in the current directory and will be DELETED" %( preprocstack ) os.remove( preprocstack ) dimg = EMData(8,8,8) dimg.to_one() for i in range(n): dimg.write_image( preprocstack, i ) finalbox = EMData(options.input,0,True)['nx'] if options.clip: finalbox=options.clip #dimglarge=EMData(finalbox,finalbox,finalbox) #dimglarge.to_one() #dimglarge.write_image(preprocstack,0) #n=EMUtil.get_image_count(options.input) #if options.subset: # n=options.subset #dimglarge.write_image(preprocstack,n-1) if options.verbose: print "\n(e2spt_preproc)(main) wrote dummy ptcls to %s" %( preprocstack) print "\n(e2spt_preproc)(main) - INITIALIZING PARALLELISM!\n" if options.parallel: from EMAN2PAR import EMTaskCustomer etc=EMTaskCustomer(options.parallel) pclist=[options.input] etc.precache(pclist) print "\n(e2spt_preproc)(main) - precaching --input" tasks=[] results=[] from e2spt_classaverage import sptOptionsParser options = sptOptionsParser( options ) for j in range(n): #print "processing particle", j img = EMData( options.input, j ) if options.parallel: #task = Preproc3DTask( ["cache",options.input,j], options, j, preprocstack ) task = Preproc3DTask( img, options, j, preprocstack ) tasks.append(task) else: img = EMData( options.input, j ) pimg = preprocfunc( img, options, j, preprocstack) if options.parallel and tasks: tids = etc.send_tasks(tasks) if options.verbose: print "\n(e2spt_preproc)(main) preprocessing %d tasks queued" % (len(tids)) results = get_results( etc, tids, options ) #print "\n(e2spt_preproc)(main) preprocessing results are", results #print "\n(e2spt_preproc)(main) input changing to preprocstack" #options.input = preprocstack #cache needs to be reloaded with the new options.input else: print "\n(e2spt_preproc)(main) Nothing to do. No preprocessing parameters specified." E2end(logger) return
def main(): progname = os.path.basename(sys.argv[0]) usage = """prog <output> [options] Program to build an initial subtomogram average by averaging pairs from the largest subset in --input that is a power of 2. For example, if you supply an input stack with 100 subtomograms, this program will build an initial reference using 64, since 64 is the largest power of 2 contained in 100. In the first iteration, particle 1 will be averaged with 2, 3 with 4, 5 with 6... etc. 32 new averages (each an average of 2 subtomograms) will be used for the second iteration. Again, 1 will be averaged with 2, 3 with 4, etc... yielding 16 new averages. The algorithm continues until the entire subset (64) has been merged into 1 average. This program imports 'preprocfunc' from e2spt_preproc.py and 'alignment' from e2spt_classaverage.py --mask=mask.sharp:outer_radius=<safe radius> --preprocess=filter.lowpass.gauss:cutoff_freq=<1/resolution in A> """ parser = EMArgumentParser(usage=usage,version=EMANVERSION) parser.add_header(name="sptbtheader", help="""Options below this label are specific to sptbinarytree""", title="### sptbinarytree options ###", row=6, col=0, rowspan=1, colspan=3,mode="align") parser.add_header(name="caheader", help="""Options below this label are specific to sptclassaverage""", title="### sptclassaverage options ###", row=3, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument("--path",type=str,default='spt',help="""Default=spt. Directory to store results in. The default is a numbered series of directories containing the prefix 'spt'; for example, spt_02 will be the directory by default if 'spt_01' already exists.""") parser.add_argument("--input", type=str, default='',help="""Default=None. The name of the input volume stack. MUST be HDF since volume stack support is required.""", guitype='filebox', browser='EMSubTomosTable(withmodal=True,multiselect=False)', row=0, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument("--npeakstorefine", type=int, help="""Default=1. The number of best coarse alignments to refine in search of the best final alignment. Default=1.""", default=4, guitype='intbox', row=9, col=0, rowspan=1, colspan=1, nosharedb=True, mode='alignment,breaksym[1]') parser.add_argument("--parallel",default="thread:1",help="""default=thread:1. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel""", guitype='strbox', row=19, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument("--ppid", type=int, help="""Default=-1. Set the PID of the parent process, used for cross platform PPID""",default=-1) parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="""Default=0. Verbose level [0-9], higner number means higher level of verboseness""") #parser.add_argument("--resume",type=str,default='',help="""(Not working currently). tomo_fxorms.json file that contains alignment information for the particles in the set. If the information is incomplete (i.e., there are less elements in the file than particles in the stack), on the first iteration the program will complete the file by working ONLY on particle indexes that are missing. For subsequent iterations, all the particles will be used.""") parser.add_argument("--plots", action='store_true', default=False,help="""Default=False. Turn this option on to generatea plot of the ccc scores during each iteration. Running on a cluster or via ssh remotely might not support plotting.""") parser.add_argument("--subset",type=int,default=0,help="""Default=0 (not used). Refine only this substet of particles from the stack provided through --input""") parser.add_argument("--preavgproc1",type=str,default='',help="""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)""") parser.add_argument("--preavgproc2",type=str,default='',help="""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)""") parser.add_argument("--weighbytiltaxis",type=str,default='',help="""Default=None. A,B, where A is an integer number and B a decimal. A represents the location of the tilt axis in the tomogram in pixels (eg.g, for a 4096x4096xZ tomogram, this value should be 2048), and B is the weight of the particles furthest from the tomogram. For example, --weighbytiltaxis=2048,0.5 means that praticles at the tilt axis (with an x coordinate of 2048) will have a weight of 1.0 during averaging, while the distance in the x coordinates of particles not-on the tilt axis will be used to weigh their contribution to the average, with particles at the edge(0+radius or 4096-radius) weighing 0.5, as specified by the value provided for B.""") parser.add_argument("--weighbyscore",action='store_true',default=False,help="""Default=False. This option will weigh the contribution of each subtomogram to the average by score/bestscore.""") parser.add_argument("--align",type=str,default="rotate_translate_3d:search=8:delta=12:dphi=12",help="""This is the aligner used to align particles to the previous class average. Default is rotate_translate_3d:search=8:delta=12:dphi=12, specify 'None' (with capital N) to disable.""", returnNone=True,guitype='comboparambox', choicelist='re_filter_list(dump_aligners_list(),\'3d\')', row=12, col=0, rowspan=1, colspan=3, nosharedb=True, mode="alignment,breaksym['rotate_symmetry_3d']") parser.add_argument("--aligncmp",type=str,default="ccc.tomo.thresh",help="""Default=ccc.tomo.thresh. The comparator used for the --align aligner. Do not specify unless you need to use anotherspecific aligner.""",guitype='comboparambox',choicelist='re_filter_list(dump_cmps_list(),\'tomo\')', row=13, col=0, rowspan=1, colspan=3,mode="alignment,breaksym") #parser.add_argument("--output", type=str, default='avg.hdf', help="""Default=avg.hdf. The name of the output class-average stack. MUST be HDF since volume stack support is required.""", guitype='strbox', row=2, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') #parser.add_argument("--classmx", type=str, default='', help="""Default=None. The name of the classification matrix specifying how particles in 'input' should be grouped. If omitted, all particles will be averaged.""") #parser.add_argument("--ref", type=str, default='', help="""Default=None. Reference image(s). Used as an initial alignment reference and for final orientation adjustment if present. This is typically the projections that were used for classification.""", guitype='filebox', browser='EMBrowserWidget(withmodal=True,multiselect=True)', filecheck=False, row=1, col=0, rowspan=1, colspan=3, mode='alignment') #parser.add_argument("--refpreprocess",action="store_true",default=False,help="""Default=False. This will preprocess the reference identically to the particles. It is off by default, but it is internally turned on when no reference is supplied.""") #parser.add_argument("--resultmx",type=str,default=None,help="""Default=Npone. Specify an output image to store the result matrix. This is in the same format as the classification matrix. http://blake.bcm.edu/emanwiki/EMAN2/ClassmxFiles""") #parser.add_argument("--refinemultireftag", type=str, default='', help="""Default=''. DO NOT USE THIS PARAMETER. It is passed on from e2spt_refinemulti.py if needed.""") ''' ADVANCED parameters ''' parser.add_argument("--averager",type=str,default="mean.tomo",help="""Default=mean.tomo. The type of averager used to produce the class average. Default=mean.tomo.""") ''' PRE-FFT processing parameters ''' #parser.add_argument("--nopreprocprefft",action="store_true",default=False,help="""Turns off all preprocessing that happens only once before alignment (--normproc, --mask, --maskfile, --clipali, --threshold; i.e., all preprocessing excepting filters --highpass, --lowpass, --preprocess, and --shrink.""") parser.add_argument("--shrink", type=int,default=1,help="""Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount for coarse alignment.""", guitype='shrinkbox', row=5, col=1, rowspan=1, colspan=1, mode='alignment,breaksym') parser.add_argument("--shrinkfine", type=int,default=1,help="""Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount for refine alignment.""", guitype='intbox', row=5, col=2, rowspan=1, colspan=1, mode='alignment') parser.add_argument("--threshold",type=str,default='',help="""Default=None. A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=10, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument("--mask",type=str,default='', help="""Default=None. Masking processor applied to particles before alignment. IF using --clipali, make sure to express outer mask radii as negative pixels from the edge.""", returnNone=True, guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'mask\')', row=11, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument("--maskfile",type=str,default='',help="""Default=None. Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None.""") parser.add_argument("--normproc",type=str, default='',help="""Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to particles before alignment. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'""") parser.add_argument("--clipali",type=int,default=0,help="""Default=0 (which means it's not used). Boxsize to clip particles as part of preprocessing to speed up alignment. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary; still, there are some benefits from 'oversampling' the data during averaging; so you might still want an average of size 2x, but perhaps particles in a box of 1.5x are sufficiently good for alignment. In this case, you would supply --clipali=75""") ''' POST-FFT filtering parameters ''' parser.add_argument("--preprocess",type=str,default='',help="""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=10, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument("--preprocessfine",type=str,default='',help="""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging.""") parser.add_argument("--lowpass",type=str,default='',help="""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=17, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument("--lowpassfine",type=str,default='',help="""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging.""") parser.add_argument("--highpass",type=str,default='',help="""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=18, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument("--highpassfine",type=str,default='',help="""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging.""") parser.add_argument("--matchimgs",action='store_true',default=False,help="""Default=False. Applies filter.matchto to one image so that it matches the other's spectral profile during preprocessing for pair-wise alignment purposes.""") parser.add_argument("--filterbyfsc",action='store_true',default=False,help="""Default=False. If on, this parameter will use dynamic FSC filtering. --lowpass will be used to build initial references if no --ref supplied, then, the FSC between the even and odd initial references will be used to filter the data during preprocessing. If --ref is supplied, --lowpass will be used during the first iteration to align the particles against the reference. Thereafter, the FSC between the most current particle average and the original reference (--ref) will be used in the next iteration.""") ''' OTHER ADVANCED parameters ''' parser.add_argument("--radius", type=float, default=0, help="""Default=0 (which means it's not used by default). Hydrodynamic radius of the particle in Angstroms. This will be used to automatically calculate the angular steps to use in search of the best alignment. Make sure the apix is correct on the particles' headers, sine the radius will be converted from Angstroms to pixels. Then, the fine angular step is equal to 360/(2*pi*radius), and the coarse angular step 4 times that.""") parser.add_argument("--precision",type=float,default=1.0,help="""Default=1.0. Precision in pixels to use when figuring out alignment parameters automatically using --radius. Precision would be the number of pixels that the the edge of the specimen is moved (rotationally) during the finest sampling, --falign. If precision is 1, then the precision of alignment will be that of the sampling (apix of your images) times the --shrinkfine factor specified.""") parser.add_argument("--search", type=int,default=8,help=""""Default=8. During COARSE alignment translational search in X, Y and Z, in pixels. Default=8. This WILL overwrite any search: provided through --align, EXCEPT if you provide --search=8, which is the default. In general, just avoid providing search twice (through here and through the aligner, --align). If you do, just be careful to make them consistent to minimize misinterpretation and error.""") parser.add_argument("--searchfine", type=int,default=2,help=""""Default=2. During FINE alignment translational search in X, Y and Z, in pixels. Default=2. This WILL overwrite any search: provided through --falign, EXCEPT if you provide --searchfine=2, which is the default. In general, just avoid providing search twice (through here and through the fine aligner --falign). If you do, just be careful to make them consistent to minimize misinterpretation and error.""") #parser.add_argument("--donotaverage",action="store_true", help="""If e2spt_refinemulti.py is calling e2spt_classaverage.py, the latter need not average any particles, but rather only yield the alignment results.""", default=False) parser.add_argument("--iterstop", type=int, default=0, help="""Default=0. (Not used). The program is called to convergence by default (all particles merge into one final average). To stop at an intermediate iteration, provide this parameter. For example, --iterstop=1, will only allow the algorithm to complete 1 iteration; --iterstop=2 will allow it to go through 2, etc.""") parser.add_argument("--savesteps",action="store_true", default=False, help="""Default=False. If set, will save the average after each iteration to class_#.hdf. Each class in a separate file. Appends to existing files.""", guitype='boolbox', row=4, col=0, rowspan=1, colspan=1, mode='alignment,breaksym') parser.add_argument("--saveali",action="store_true", default=False, help="""Default=False. If set, will save the aligned particle volumes in class_ptcl.hdf. Overwrites existing file.""", guitype='boolbox', row=4, col=1, rowspan=1, colspan=1, mode='alignment,breaksym') parser.add_argument("--saveallalign",action="store_true", default=False, help="""Default=False. If set, will save the alignment parameters after each iteration""", guitype='boolbox', row=4, col=2, rowspan=1, colspan=1, mode='alignment,breaksym') parser.add_argument("--sym", dest = "sym", default='', help = """Default=None (equivalent to c1). Symmetry to impose -choices are: c<n>, d<n>, h<n>, tet, oct, icos""", guitype='symbox', row=9, col=1, rowspan=1, colspan=2, mode='alignment,breaksym') parser.add_argument("--postprocess",type=str,default='',help="""A processor to be applied to the FINAL volume after averaging the raw volumes in their FINAL orientations, after all iterations are done.""",guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=16, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument("--procfinelikecoarse",action='store_true',default=False,help="""If you supply this parameters, particles for fine alignment will be preprocessed identically to particles for coarse alignment by default. If you supply this, but want specific parameters for preprocessing particles for also supply: fine alignment, nd supply fine alignment parameters, such as --lowpassfine, --highpassfine, etc; to preprocess the particles for FINE alignment differently than for COARSE alignment.""") parser.add_argument("--falign",type=str,default='',help="""Default=None. This is the second stage aligner used to fine-tune the first alignment.""", returnNone=True, guitype='comboparambox', choicelist='re_filter_list(dump_aligners_list(),\'refine.*3d\')', row=14, col=0, rowspan=1, colspan=3, nosharedb=True, mode='alignment,breaksym[None]') parser.add_argument("--faligncmp",type=str,default="ccc.tomo.thresh",help="""Default=ccc.tomo.thresh. The comparator used by the second stage aligner.""", guitype='comboparambox', choicelist='re_filter_list(dump_cmps_list(),\'tomo\')', row=15, col=0, rowspan=1, colspan=3,mode="alignment,breaksym") #parser.add_argument("--nopreprocprefft",action="store_true",default=False,help="""Turns off all preprocessing that happens only once before alignment (--normproc, --mask, --maskfile, --clipali, --threshold; i.e., all preprocessing excepting filters --highpass, --lowpass, --preprocess, and --shrink.""") #parser.add_argument("--keep",type=float,default=1.0,help="""Default=1.0 (all particles kept). The fraction of particles to keep in each class.""", guitype='floatbox', row=6, col=0, rowspan=1, colspan=1, mode='alignment,breaksym') #parser.add_argument("--keepsig", action="store_true", default=False,help="""Default=False. Causes the keep argument to be interpreted in standard deviations.""", guitype='boolbox', row=6, col=1, rowspan=1, colspan=1, mode='alignment,breaksym') #parser.add_argument("--inixforms",type=str,default="",help="""Default=None. .json file containing a dict of transforms to apply to 'pre-align' the particles.""", guitype='dirbox', dirbasename='spt_|sptsym_', row=7, col=0,rowspan=1, colspan=2, nosharedb=True, mode='breaksym') parser.add_argument("--breaksym",action="store_true", default=False,help="""Default=False. Break symmetry. Do not apply symmetrization after averaging, even if searching the asymmetric unit provided through --sym only for alignment. Default=False""", guitype='boolbox', row=7, col=2, rowspan=1, colspan=1, nosharedb=True, mode=',breaksym[True]') #parser.add_argument("--groups",type=int,default=0,help="""Default=0 (not used; data not split). This parameter will split the data into a user defined number of groups. For purposes of gold-standard FSC computation later, select --group=2.""") parser.add_argument("--randomizewedge",action="store_true", default=False,help="""Default=False. This parameter is EXPERIMENTAL. It randomizes the position of the particles BEFORE alignment, to minimize missing wedge bias and artifacts during symmetric alignment where only a fraction of space is scanned""") #parser.add_argument("--savepreproc",action="store_true", default=False,help="""Default=False. Will save stacks of preprocessed particles (one for coarse alignment and one for fine alignment if preprocessing options are different).""") parser.add_argument("--autocenter",type=str, default='',help="""Default=None. Autocenters each averaged pair during initial average generation with --btref and --hacref. Will also autocenter the average of all particles after each iteration of iterative refinement. Options are --autocenter=xform.centerofmass (self descriptive), or --autocenter=xform.centeracf, which applies auto-convolution on the average.""") parser.add_argument("--autocentermask",type=str, default='',help="""Default=None. Masking processor to apply before autocentering. See 'e2help.py processors -v 10' at the command line.""") parser.add_argument("--autocenterpreprocess",action='store_true', default=False,help="""Default=False. This will apply a highpass filter at a frequency of half the box size times the apix, shrink by 2, and apply a low pass filter at half nyquist frequency to any computed average for autocentering purposes if --autocenter is provided. Default=False.""") parser.add_argument("--tweak",action='store_true',default=False,help="""WARNING: BUGGY. This will perform a final alignment with no downsampling [without using --shrink or --shrinkfine] if --shrinkfine > 1.""") ''' BT SPECIFIC PARAMETERS ''' parser.add_argument("--nseedlimit",type=int,default=0,help="""Maximum number of particles to use. For example, if you supply a stack with 150 subtomograms, the program will automatically select 128 as the limit to use because it's the largest power of 2 that is smaller than 150. But if you provide, say --nseedlimit=100, then the number of particles used will be 64, because it's the largest power of 2 that is still smaller than 100.""") (options, args) = parser.parse_args() options.nopreprocprefft = False if options.shrink < options.shrinkfine: options.shrink = options.shrinkfine print "\n(e2spt_binarytree)(main) it makes no sense for shrinkfine to be larger than shrink; therefore, shrink will be made to match shrinkfine" from e2spt_classaverage import checksaneimagesize checksaneimagesize( options, options.input ) ''' Make the directory where to create the database where the results will be stored ''' from e2spt_classaverage import sptmakepath options = sptmakepath(options,'spt_bt') rootpath = os.getcwd() if rootpath not in options.path: options.path = rootpath + '/' + options.path if not options.input: parser.print_help() exit(0) elif options.subset: subsetStack = options.path + '/subset' + str( options.subset ).zfill( len( str( options.subset))) + '.hdf' print "\nSubset to be written to", subsetStack subsetcmd = 'e2proc3d.py ' + options.input + ' ' + subsetStack + ' --first=0 --last=' + str(options.subset-1) print "Subset cmd is", subsetcmd p=subprocess.Popen( subsetcmd, shell=True,stdout=subprocess.PIPE, stderr=subprocess.PIPE ) text=p.communicate() p.stdout.close() options.input = subsetStack from e2spt_classaverage import sptParseAligner options = sptParseAligner( options ) ''' If --radius of the particle is provided, we calculate the optimal alignment steps for coarse and fine alignment rounds using --shrink and --shrinkfine options and apix info ''' if options.radius: from e2spt_classaverage import calcAliStep options = calcAliStep(options) ''' Parse parameters such that "None" or "none" are adequately interpreted to turn of an option ''' from e2spt_classaverage import sptOptionsParser options = sptOptionsParser( options ) from e2spt_classaverage import writeParameters writeParameters(options,'e2spt_binarytree.py', 'bt') hdr = EMData(options.input,0,True) nx = hdr["nx"] ny = hdr["ny"] nz = hdr["nz"] if nx!=ny or ny!=nz : print "ERROR, input volumes are not cubes" sys.exit(1) logger = E2init(sys.argv, options.ppid) ''' Initialize parallelism if being used ''' if options.parallel : if options.parallel == 'none' or options.parallel == 'None' or options.parallel == 'NONE': options.parallel = '' etc = '' else: print "\n\n(e2spt_classaverage.py) INITIALIZING PARALLELISM!" print "\n\n" from EMAN2PAR import EMTaskCustomer etc=EMTaskCustomer(options.parallel) pclist=[options.input] etc.precache(pclist) else: etc='' options.raw = options.input """ if 'tree' in options.align: options.falign = None options.mask = None options.lowpass = None options.highpass = None options.normproc = None options.lowpassfine = None options.highpassfine = None options.preprocess = None options.preprocessfine = None else: from e2spt_classaverage import cmdpreproc cmpreproc( options.input, options, False ) """ nptcl=EMUtil.get_image_count(options.input) if nptcl < 1: print "ERROR : at least 2 particles required in input stack" sys.exit(1) ptclnums=range(nptcl) nptclForRef = len(ptclnums) nseed=2**int(floor(log(len(ptclnums),2))) # we stick with powers of 2 for this to make the tree easier to collapse if options.nseedlimit: nseed=2**int(floor(log( options.nseedlimit , 2))) #binaryTreeRef(options,nptclForRef,nseed,-1,etc) binaryTreeRef(options,nptclForRef,nseed,etc) print "Will end logger" E2end(logger) print "logger ended" sys.stdout.flush() return
def main(): progname = os.path.basename(sys.argv[0]) usage = """prog [options] This program will take a set of reference-free class-averages (or other projections) and generate a set of possible 3-D initial models. It does this by heavily downsampling the data, then running a number of very fast, full iterative refinements, each seeded with a random starting model. The results are sorted in order of apparent agreement with the data, such that at the end, the first numbered model should be the best result. Ideally the top few answers will all qualtitatively agree on the overall structure. If they do not, the results should be thoroughly assessed manually to insure a sensible result. By default this routine will generate 10 initial models, but this may be fewer or more than is strictly necessary depending on a number of factors. If the data is highly structurally heterogeneous, particularly if combined with a strongly preferred orientation, a correct solution using this technique may not be possible, but for most situations it will work well. For other situations, single particle tomography presents a good alternative for generating initial models.""" parser = EMArgumentParser(usage=usage,version=EMANVERSION) parser.add_header(name="initialmodelheader", help='Options below this label are specific to e2initialmodel', title="### e2initialmodel options ###", row=1, col=0, rowspan=1, colspan=3) parser.add_argument("--input", dest="input", default=None,type=str, help="This file should contain good class-averages to use in constructing the initial model", browser='EMBrowserWidget(withmodal=True,multiselect=False)', guitype='filebox', row=0, col=0, rowspan=1, colspan=3) parser.add_argument("--iter", type = int, default=8, help = "The total number of refinement iterations to perform, typically 5-10", guitype='intbox', row=2, col=0, rowspan=1, colspan=1) parser.add_argument("--tries", type=int, default=10, help="The number of different initial models to generate in search of a good one", guitype='intbox', row=2, col=1, rowspan=1, colspan=1) parser.add_argument("--shrink", dest="shrink", type = int, default=0, help="Optionally shrink the input particles by an integer factor prior to reconstruction. Default=0, no shrinking", guitype='shrinkbox', row=2, col=2, rowspan=1, colspan=1) parser.add_argument("--sym", dest = "sym", help = "Specify symmetry - choices are: c<n>, d<n>, h<n>, tet, oct, icos",default="c1", guitype='symbox', row=4, col=0, rowspan=1, colspan=2) parser.add_argument("--randorient",action="store_true",help="Instead of seeding with a random volume, seeds by randomizing input orientations",default=False, guitype='boolbox', row=4, col=2, rowspan=1, colspan=1) parser.add_argument("--maskproc", default=None, type=str,help="Default=none. If specified, this mask will be performed after the built-in automask, eg - mask.soft to remove the core of a virus", ) # parser.add_argument("--savemore",action="store_true",help="Will cause intermediate results to be written to flat files",default=False, guitype='boolbox', expert=True, row=5, col=0, rowspan=1, colspan=1) parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness") parser.add_argument("--orientgen",type=str, default="eman:delta=9.0:inc_mirror=0:perturb=1",help="The type of orientation generator. Default is eman:delta=9.0:inc_mirror=0:perturb=1. See e2help.py orientgens", guitype='strbox', expert=True, row=4, col=2, rowspan=1, colspan=1) parser.add_argument("--parallel","-P",type=str,help="Run in parallel, specify type:<option>=<value>:<option>=<value>. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel",default="thread:1", guitype='strbox', row=6, col=0, rowspan=1, colspan=2) parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1) # Database Metadata storage #parser.add_argument("--dbls",type=str,default=None,help="data base list storage, used by the workflow. You can ignore this argument.") (options, args) = parser.parse_args() verbose=options.verbose try: ptcls=EMData.read_images(options.input) except: print "Error: bad input file" exit(1) apix=ptcls[0]["apix_x"] if options.shrink>1 : apix*=options.shrink for i in range(len(ptcls)): ptcls[i].process_inplace("normalize.edgemean",{}) if options.shrink>1 : ptcls[i]=ptcls[i].process("math.meanshrink",{"n":options.shrink}) if ptcls[0]["nx"]>160 : print "WARNING: using a large box size may be slow. Suggest trying --shrink=" if not ptcls or len(ptcls)==0 : parser.error("Bad input file") boxsize=ptcls[0].get_xsize() if verbose>0 : print "%d particles %dx%d"%(len(ptcls),boxsize,boxsize) print "Models will be %1.3f A/pix"%apix [og_name,og_args] = parsemodopt(options.orientgen) try: sfcurve=XYData() sfcurve.read_file("strucfac.txt") sfcurve.update() except : sfcurve=None if options.maskproc!=None : mask2=EMData(boxsize,boxsize,boxsize) mask2.to_one() parms=parsemodopt(options.maskproc) if parms[0]=="mask.auto3d": print "Error, maskproc may not be mask.auto3d, it must be a processor that does not rely on the input map density to function" sys.exit(1) mask2.process_inplace(parms[0],parms[1]) else: mask2=None # angles to use for refinement sym_object = parsesym(options.sym) orts = sym_object.gen_orientations(og_name,og_args) logid=E2init(sys.argv,options.ppid) results=[] try: os.mkdir("initial_models") except: pass iters=[int(i[10:12]) for i in os.listdir("initial_models") if i[:10]=="particles_"] try : newiter=max(iters)+1 except : newiter=0 results_name="initial_models/model_%02d"%newiter particles_name="initial_models/particles_%02d.hdf"%newiter # we write the pre-processed "particles" (usually class-averages) to disk, both as a record and to prevent collisions for i,p in enumerate(ptcls): p.write_image(particles_name,i) # parallelism from EMAN2PAR import EMTaskCustomer # we need to put this here to avoid a circular reference etc=EMTaskCustomer(options.parallel) pclist=[particles_name] etc.precache(pclist) # make sure the input particles are precached on the compute nodes tasks=[] for t in xrange(options.tries): tasks.append(InitMdlTask(particles_name,len(ptcls),orts,t,sfcurve,options.iter,options.sym,mask2,options.randorient,options.verbose)) taskids=etc.send_tasks(tasks) alltaskids=taskids[:] # we keep a copy for monitoring progress # This loop runs until all subtasks are complete (via the parallelism system ltime=0 while len(taskids)>0 : time.sleep(0.1) curstat=etc.check_task(taskids) # a list of the progress on each task if options.verbose>1 : if time.time()-ltime>1 : print "progress: ",curstat ltime=time.time() for i,j in enumerate(curstat): if j==100 : rslt=etc.get_results(taskids[i]) # read the results back from a completed task as a one item dict results.append(rslt[1]["result"]) if options.verbose==1 : print "Task {} ({}) complete".format(i,taskids[i]) # filter out completed tasks. We can't do this until after the previous loop completes taskids=[taskids[i] for i in xrange(len(taskids)) if curstat[i]!=100] # Write out the final results results.sort() for i,j in enumerate(results): out_name = results_name+"_%02d.hdf"%(i+1) j[1].write_image(out_name,0) j[4].write_image(results_name+"_%02d_init.hdf"%(i+1),0) print out_name,j[1]["quality"],j[0],j[1]["apix_x"] for k,l in enumerate(j[3]): l[0].write_image(results_name+"_%02d_proj.hdf"%(i+1),k) # set of projection images for k,l in enumerate(j[2]): l.process("normalize").write_image(results_name+"_%02d_aptcl.hdf"%(i+1),k*2) # set of aligned particles j[3][l["match_n"]][0].process("normalize").write_image(results_name+"_%02d_aptcl.hdf"%(i+1),k*2+1) # set of projections matching aligned particles E2end(logid)
class EMParallelSimMX(object): def __init__(self, options, args, logger=None): ''' @param options the options produced by (options, args) = parser.parse_args() @param args the options produced by (options, args) = parser.parse_args() @param logger and EMAN2 logger, i.e. logger=E2init(sys.argv) assumes you have already called the check function. ''' self.options = options self.args = args self.logger = logger from EMAN2PAR import EMTaskCustomer self.etc = EMTaskCustomer(options.parallel, module="e2simmx.EMSimTaskDC") if options.colmasks != None: self.etc.precache([args[0], args[1], options.colmasks]) else: self.etc.precache([args[0], args[1]]) self.num_cpus = self.etc.cpu_est() if self.num_cpus < 32: # lower limit self.num_cpus = 32 self.__task_options = None def __get_task_options(self, options): ''' Get the options required by each task as a dict @param options is always self.options - the initialization argument. Could be changed. ''' if self.__task_options == None: d = {} d["align"] = parsemodopt(options.align) d["aligncmp"] = parsemodopt(options.aligncmp) d["cmp"] = parsemodopt(options.cmp) if hasattr(options, "ralign") and options.ralign != None: d["ralign"] = parsemodopt(options.ralign) d["raligncmp"] = parsemodopt( options.raligncmp ) # raligncmp must be specified if using ralign else: d["ralign"] = None d["raligncmp"] = None d["prefilt"] = options.prefilt if hasattr(options, "shrink") and options.shrink != None: d["shrink"] = options.shrink else: d["shrink"] = None self.__task_options = d return self.__task_options def __init_memory(self, options): ''' @param options is always self.options - the initialization argument. Could be changed. Establishes several important attributes they are: ---- self.clen - the number of images in the image defined by args[0], the number of columns in the similarity matrix self.rlen - the number of images in the image defined by args[1], the number of rows in the similarity matrix ---- Also, since we adopted region output writing as our preferred approach, this function makes sure the output image(s) exists on disk and has the correct dimensions - seeing as this is the way region writing works (the image has to exist on disk and have its full dimensions) ''' self.clen = EMUtil.get_image_count(self.args[0]) self.rlen = EMUtil.get_image_count(self.args[1]) output = self.args[2] if file_exists(output) and not options.fillzero: remove_file(output) e = EMData(self.clen, self.rlen) e.to_zero() e.set_attr(PROJ_FILE_ATTR, self.args[0]) e.set_attr(PART_FILE_ATTR, self.args[1]) n = 1 if self.options.saveali: n = 6 # the total number of images written to disk if not options.fillzero: e.write_image(output, 0) for i in range(1, n): e.write_image(output, i) def __get_blocks(self): ''' Gets the blocks that will be processed in parallel, these are essentially ranges ''' steve_factor = 3 # increase number of jobs a bit for better distribution total_jobs = steve_factor * self.num_cpus [col_div, row_div] = opt_rectangular_subdivision(self.clen, self.rlen, total_jobs) block_c = old_div(self.clen, col_div) block_r = old_div(self.rlen, row_div) residual_c = self.clen - block_c * col_div # residual left over by integer division blocks = [] current_c = 0 for c in range(0, col_div): last_c = current_c + block_c if residual_c > 0: last_c += 1 residual_c -= 1 current_r = 0 residual_r = self.rlen - block_r * row_div # residual left over by integer division for r in range(0, row_div): last_r = current_r + block_r if residual_r > 0: last_r += 1 residual_r -= 1 blocks.append([current_c, last_c, current_r, last_r]) current_r = last_r current_c = last_c # print col_div,row_div,col_div*row_div # print self.clen,self.rlen,residual_c,residual_r return blocks def execute(self): ''' The main function to be called ''' if len(self.options.parallel) > 1: self.__init_memory(self.options) blocks = self.__get_blocks() # print blocks # self.check_blocks(blocks) # testing function can be removed at some point tasks = [] for bn, block in enumerate(blocks): data = {} data["references"] = ("cache", self.args[0], block[0], block[1]) data["particles"] = ("cache", self.args[1], block[2], block[3]) if self.options.colmasks != None: data["colmasks"] = ("cache", self.options.colmasks, block[0], block[1]) if self.options.mask != None: data["mask"] = ("cache", self.options.mask, 0, 1) if self.options.fillzero: # for each particle check to see which portion of the matrix we need to fill if (bn % 10 == 0): print("%d/%d \r" % (bn, len(blocks)), end=' ') sys.stdout.flush() rng = [] for i in range(block[2], block[3]): c = EMData() c.read_image( self.args[2], 0, False, Region(block[0], i, block[1] - block[0] + 1, 1)) inr = 0 st = 0 for j in range(c["nx"]): if c[j] == 0 and not inr: st = j inr = 1 if c[j] != 0 and inr: rng.append( (i, st + block[0], j - 1 + block[0])) inr = 0 if inr: rng.append((i, st + block[0], j + block[0])) data["partial"] = rng # print "%d) %s\t"%(bn,str(block)),rng if self.options.fillzero and len(data["partial"]) == 0: continue # nothing to compute in this block, skip it completely else: task = EMSimTaskDC(data=data, options=self.__get_task_options( self.options)) #print "Est %d CPUs"%etc.cpu_est() tasks.append(task) # This just verifies that all particles have at least one class #a=set() #for i in tasks: #for k in i.data["partial"] : a.add(k[0]) #b=set(range(self.rlen)) #b-=a #print b print("%d/%d " % (bn, len(blocks))) self.tids = self.etc.send_tasks(tasks) print(len(self.tids), " tasks submitted") # while 1: if len(self.tids) == 0: break print(len(self.tids), "simmx tasks left in main loop \r", end=' ') sys.stdout.flush() st_vals = self.etc.check_task(self.tids) for i in range(len(self.tids) - 1, -1, -1): st = st_vals[i] if st == 100: tid = self.tids[i] try: rslts = self.etc.get_results(tid) # display(rslts[1]["rslt_data"][0]) self.__store_output_data(rslts[1]) except: traceback.print_exc() print( "ERROR storing results for task %d. Rerunning." % tid) self.etc.rerun_task(tid) continue if self.logger != None: E2progress( self.logger, 1.0 - old_div(len(self.tids), float(len(blocks)))) if self.options.verbose > 0: print("%d/%d\r" % (len(self.tids), len(blocks))) sys.stdout.flush() self.tids.pop(i) print(len(self.tids), "simmx tasks left in main loop \r", end=' ') sys.stdout.flush() time.sleep(10) print("\nAll simmx tasks complete ") # if using fillzero, we must fix the -1.0e38 values placed into empty cells if self.options.fillzero: l = EMData(self.args[2], 0, True) rlen = l["ny"] clen = l["nx"] # launch_childprocess("e2proc2d.py %s %s"%(self.args[2],self.args[2]+"_x")) print( "Filling noncomputed regions in similarity matrix (%dx%d)" % (clen, rlen)) l = EMData() for r in range(rlen): l.read_image(self.args[2], 0, False, Region(0, r, clen, 1)) fill = l["maximum"] + .0001 l.process_inplace("threshold.belowtominval", { "minval": -1.0e37, "newval": fill }) l.write_image(self.args[2], 0, EMUtil.ImageType.IMAGE_UNKNOWN, False, Region(0, r, clen, 1)) print("Filling complete") else: raise NotImplementedError( "The parallelism option you specified (%s) is not supported" % self.options.parallel) def __store_output_data(self, rslts): ''' Store output data to internal images (matrices) @param a dictionary return by the EMSimTaskDC ''' result_data = rslts["rslt_data"] output = self.args[2] insertion_c = rslts["min_ref_idx"] insertion_r = rslts["min_ptcl_idx"] result_mx = result_data[0] r = Region(insertion_c, insertion_r, result_mx.get_xsize(), result_mx.get_ysize()) # Note this is region io - the init_memory function made sure the images exist and are the right dimensions (on disk) for i, mxout in enumerate(result_data): mxout.write_image(output, i, EMUtil.ImageType.IMAGE_UNKNOWN, False, r)
def main(): progname = os.path.basename(sys.argv[0]) usage = """prog [options] This program will take a set of reference-free class-averages (or other projections) and generate a set of possible 3-D initial models. It does this by heavily downsampling the data, then running a number of very fast, full iterative refinements, each seeded with a random starting model. The results are sorted in order of apparent agreement with the data, such that at the end, the first numbered model should be the best result. Ideally the top few answers will all qualtitatively agree on the overall structure. If they do not, the results should be thoroughly assessed manually to insure a sensible result. By default this routine will generate 10 initial models, but this may be fewer or more than is strictly necessary depending on a number of factors. If the data is highly structurally heterogeneous, particularly if combined with a strongly preferred orientation, a correct solution using this technique may not be possible, but for most situations it will work well. For other situations, single particle tomography presents a good alternative for generating initial models.""" parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_header( name="initialmodelheader", help='Options below this label are specific to e2initialmodel', title="### e2initialmodel options ###", row=1, col=0, rowspan=1, colspan=3) parser.add_argument( "--input", dest="input", default=None, type=str, help= "This file should contain good class-averages to use in constructing the initial model", browser='EMBrowserWidget(withmodal=True,multiselect=False)', guitype='filebox', row=0, col=0, rowspan=1, colspan=3) parser.add_argument( "--iter", type=int, default=8, help= "The total number of refinement iterations to perform, typically 5-10", guitype='intbox', row=2, col=0, rowspan=1, colspan=1) parser.add_argument( "--tries", type=int, default=10, help= "The number of different initial models to generate in search of a good one", guitype='intbox', row=2, col=1, rowspan=1, colspan=1) parser.add_argument( "--shrink", dest="shrink", type=int, default=0, help= "Optionally shrink the input particles by an integer factor prior to reconstruction. Default=0, no shrinking", guitype='shrinkbox', row=2, col=2, rowspan=1, colspan=1) parser.add_argument( "--sym", dest="sym", help="Specify symmetry - choices are: c<n>, d<n>, h<n>, tet, oct, icos", default="c1", guitype='symbox', row=4, col=0, rowspan=1, colspan=2) parser.add_argument( "--automaskexpand", default=-1, type=int, help= "Number of voxels of post-threshold expansion in the mask, for use when peripheral features are truncated. (default=shrunk boxsize/20)", guitype='intbox', row=6, col=2, rowspan=1, colspan=1) parser.add_argument( "--randorient", action="store_true", help= "Instead of seeding with a random volume, seeds by randomizing input orientations", default=False, guitype='boolbox', row=4, col=2, rowspan=1, colspan=1) parser.add_argument( "--maskproc", default=None, type=str, help= "Default=none. If specified, this mask will be performed after the built-in automask, eg - mask.soft to remove the core of a virus", ) # parser.add_argument("--savemore",action="store_true",help="Will cause intermediate results to be written to flat files",default=False, guitype='boolbox', expert=True, row=5, col=0, rowspan=1, colspan=1) parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help= "verbose level [0-9], higher number means higher level of verboseness") parser.add_argument( "--orientgen", type=str, default="eman:delta=9.0:inc_mirror=0:perturb=1", help= "The type of orientation generator. Default is eman:delta=9.0:inc_mirror=0:perturb=1. See e2help.py orientgens", guitype='strbox', expert=True, row=4, col=2, rowspan=1, colspan=1) parser.add_argument( "--parallel", "-P", type=str, help= "Run in parallel, specify type:<option>=<value>:<option>=<value>. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel", default="thread:1", guitype='strbox', row=6, col=0, rowspan=1, colspan=2) parser.add_argument( "--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-1) # Database Metadata storage #parser.add_argument("--dbls",type=str,default=None,help="data base list storage, used by the workflow. You can ignore this argument.") (options, args) = parser.parse_args() verbose = options.verbose try: ptcls = EMData.read_images(options.input) except: print("Error: bad input file") exit(1) apix = ptcls[0]["apix_x"] if options.shrink > 1: apix *= options.shrink if options.tries < 10: print( "Warning: suggest using --tries >=10. The first 8 starting maps are generated deterministically, and it is good to have several random seeds as well to increase the liklihood of a good outcome." ) for i in range(len(ptcls)): ptcls[i].process_inplace("normalize.edgemean", {}) if options.shrink > 1: ptcls[i] = ptcls[i].process("math.meanshrink", {"n": options.shrink}) if ptcls[0]["nx"] > 160: print( "WARNING: using a large box size may be slow. Suggest trying --shrink=" ) if not ptcls or len(ptcls) == 0: parser.error("Bad input file") boxsize = ptcls[0].get_xsize() if verbose > 0: print("%d particles %dx%d" % (len(ptcls), boxsize, boxsize)) print("Models will be %1.3f A/pix" % apix) [og_name, og_args] = parsemodopt(options.orientgen) try: sfcurve = XYData() sfcurve.read_file("strucfac.txt") sfcurve.update() except: sfcurve = None if options.maskproc != None: mask2 = EMData(boxsize, boxsize, boxsize) mask2.to_one() parms = parsemodopt(options.maskproc) if parms[0] == "mask.auto3d": print( "Error, maskproc may not be mask.auto3d, it must be a processor that does not rely on the input map density to function" ) sys.exit(1) mask2.process_inplace(parms[0], parms[1]) else: mask2 = None # angles to use for refinement sym_object = parsesym(options.sym) orts = sym_object.gen_orientations(og_name, og_args) logid = E2init(sys.argv, options.ppid) results = [] try: os.mkdir("initial_models") except: pass iters = [ int(i[10:12]) for i in os.listdir("initial_models") if i[:10] == "particles_" ] try: newiter = max(iters) + 1 except: newiter = 0 results_name = "initial_models/model_%02d" % newiter particles_name = "initial_models/particles_%02d.hdf" % newiter # we write the pre-processed "particles" (usually class-averages) to disk, both as a record and to prevent collisions for i, p in enumerate(ptcls): p.write_image(particles_name, i) # parallelism from EMAN2PAR import EMTaskCustomer # we need to put this here to avoid a circular reference etc = EMTaskCustomer(options.parallel, module="e2initialmodel.InitMdlTask") pclist = [particles_name] etc.precache( pclist ) # make sure the input particles are precached on the compute nodes tasks = [] for t in range(options.tries): tasks.append( InitMdlTask(particles_name, len(ptcls), orts, t, sfcurve, options.iter, options.sym, mask2, options.randorient, options.automaskexpand, options.verbose)) taskids = etc.send_tasks(tasks) alltaskids = taskids[:] # we keep a copy for monitoring progress # This loop runs until all subtasks are complete (via the parallelism system ltime = 0 while len(taskids) > 0: time.sleep(0.1) curstat = etc.check_task( taskids) # a list of the progress on each task if options.verbose > 1: if time.time() - ltime > 1: print("progress: ", curstat) ltime = time.time() for i, j in enumerate(curstat): if j == 100: rslt = etc.get_results( taskids[i] ) # read the results back from a completed task as a one item dict results.append(rslt[1]["result"]) if options.verbose == 1: print("Task {} ({}) complete".format(i, taskids[i])) # filter out completed tasks. We can't do this until after the previous loop completes taskids = [ taskids[i] for i in range(len(taskids)) if curstat[i] != 100 ] # Write out the final results results.sort() for i, j in enumerate(results): out_name = results_name + "_%02d.hdf" % (i + 1) j[1].write_image(out_name, 0) j[4].write_image(results_name + "_%02d_init.hdf" % (i + 1), 0) print(out_name, j[1]["quality"], j[0], j[1]["apix_x"]) for k, l in enumerate(j[3]): l[0].write_image(results_name + "_%02d_proj.hdf" % (i + 1), k) # set of projection images for k, l in enumerate(j[2]): l.process("normalize").write_image( results_name + "_%02d_aptcl.hdf" % (i + 1), k * 2) # set of aligned particles j[3][l["match_n"]][0].process("normalize").write_image( results_name + "_%02d_aptcl.hdf" % (i + 1), k * 2 + 1) # set of projections matching aligned particles E2end(logid)
class EMParallelProject3D: def __init__(self, options, fsp, sym, start, modeln=0, logger=None): ''' @param options the options produced by (options, args) = parser.parse_args() @param args the options produced by (options, args) = parser.parse_args() @param logger and EMAN2 logger, i.e. logger=E2init(sys.argv) assumes you have already called the check function. ''' self.options = options self.args = fsp self.sym = sym self.logger = logger self.start = start self.modeln = modeln from EMAN2PAR import EMTaskCustomer self.etc = EMTaskCustomer(options.parallel) print "Precache ", fsp self.etc.precache([fsp]) self.num_cpus = self.etc.cpu_est() print self.num_cpus, " total CPUs available" if self.num_cpus > 64: # upper limit self.num_cpus = 64 self.__task_options = None def __init_memory(self, options): ''' ''' sym_object = parsesym(self.sym) [og_name, og_args] = parsemodopt(options.orientgen) self.eulers = sym_object.gen_orientations(og_name, og_args) def __get_task_options(self, options): if self.__task_options == None: d = {} d["projector"] = parsemodopt(options.projector) d["prethreshold"] = options.prethreshold self.__task_options = d return self.__task_options def execute(self): # from EMAN2PAR import EMTaskCustomer if len(self.options.parallel) > 1: self.__init_memory(self.options) num_tasks = self.num_cpus # In the worst case we can only spawn as many tasks as there are eulers if self.num_cpus > len(self.eulers): num_tasks = len(self.eulers) eulers_per_task = len(self.eulers) / num_tasks resid_eulers = len( self.eulers ) - eulers_per_task * num_tasks # we can distribute the residual evenly first = 0 task_customers = [] tids = [] # self.etc=EMTaskCustomer(self.options.parallel) for i in xrange(0, num_tasks): last = first + eulers_per_task if resid_eulers > 0: last += 1 resid_eulers -= 1 tmp_eulers = self.eulers[first:last] indices = range(first, last) data = {} data["input"] = ("cache", self.args, 0) data["eulers"] = tmp_eulers data["indices"] = indices task = EMProject3DTaskDC(data=data, options=self.__get_task_options( self.options)) #print "Est %d CPUs"%etc.cpu_est() tid = self.etc.send_task(task) #print "Task submitted tid=",tid tids.append(tid) first = last print "Task ids are", tids while 1: print len(tids), "projection tasks left in main loop" st_vals = self.etc.check_task(tids) for i in xrange(len(tids) - 1, -1, -1): st = st_vals[i] if st == 100: tid = tids[i] rslts = self.etc.get_results(tid) if not self.__write_output_data(rslts[1]): print "There was a problem with the task of id", tid if self.logger != None: E2progress(self.logger, 1.0 - len(tids) / float(num_tasks)) if self.options.verbose > 0: print "%d/%d\r" % (num_tasks - len(tids), num_tasks) sys.stdout.flush() print "Task", tids.pop(i), "completed" print "These tasks are remaining:", tids if len(tids) == 0: break time.sleep(5) return len(self.eulers) else: raise NotImplementedError( "The parallelism option you specified (%s) is not suppored" % self.options.parallel) def __write_output_data(self, rslts): for idx, image in rslts.items(): if not isinstance(image, EMData): continue # this is here because we get the dimensions of the database as a key (e.g. '40x40x1'). image["model_id"] = self.modeln if self.options.append: image.write_image(self.options.outfile, -1) else: image.write_image(self.options.outfile, idx + self.start) return True
def main(): progname = os.path.basename(sys.argv[0]) usage = """prog <output> [options] This program is used to preprocess subtomograms before aligning them. The same can be accomplished with e2proc3d, except that this program is parallelized and thus should be substantially faster for large subtomograms. """ parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_argument( "--input", type=str, default='', help= """Default=None. The name of the input volume stack. MUST be HDF since volume stack support is required.""" ) parser.add_argument( "--output", type=str, default='', help= """Default=None. Specific name of HDF file to write processed particles to.""" ) parser.add_argument( "--parallel", type=str, default='', help= """default=None. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel""" ) parser.add_argument( "--ppid", type=int, help= """Default=-1. Set the PID of the parent process, used for cross platform PPID""", default=-1) parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help= """Default=0. Verbose level [0-9], higner number means higher level of verboseness""" ) parser.add_argument( "--subset", type=int, default=0, help= """Default=0 (not used). Refine only this substet of particles from the stack provided through --input""" ) parser.add_argument( "--apix", type=float, default=0.0, help= """Default=0.0 (not used). Use this apix value where relevant instead of whatever is in the header of the reference and the particles. Will overwrite particle header as well.""" ) parser.add_argument( "--shrink", type=int, default=0, help= """Default=0 (no shrinking). Optionally shrink the input volumes by an integer amount for coarse alignment.""" ) parser.add_argument( "--threshold", type=str, default='', help= """Default=None. A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""" ) parser.add_argument( "--mask", type=str, default='', help= """Default=None. Masking processor applied to particles before alignment. IF using --clip, make sure to express outer mask radii as negative pixels from the edge.""" ) parser.add_argument( "--maskfile", type=str, default='', help= """Default=None. Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None.""" ) parser.add_argument( "--normproc", type=str, default='', help= """Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to particles before alignment. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'""" ) parser.add_argument( "--preprocess", type=str, default='', help= """Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""" ) parser.add_argument( "--lowpass", type=str, default='', help= """Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""" ) parser.add_argument( "--highpass", type=str, default='', help= """Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""" ) parser.add_argument( "--clip", type=int, default=0, help= """Default=0 (which means it's not used). Boxsize to clip particles. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary.""" ) parser.add_argument( "--nopath", action='store_true', default=False, help= """If supplied, this option will save results in the directory where the command is run. A directory to store the results will not be made.""" ) parser.add_argument( "--path", type=str, default='sptpreproc', help= """Default=spt. Directory to store results in. The default is a numbered series of directories containing the prefix 'sptpreproc'; for example, sptpreproc_02 will be the directory by default if 'sptpreproc_01' already exists.""" ) (options, args) = parser.parse_args() logger = E2init(sys.argv, options.ppid) print "\n(e2spt_preproc)(main) started log" from e2spt_classaverage import sptmakepath if options.path and not options.nopath: options = sptmakepath(options, 'sptpreproc') if options.parallel == 'None' or options.parallel == 'none': options.parallel = None if not options.input: try: options.input = sys.argv[1] except: print "\n(e2spt_preproc)(main) ERROR: invalid input file" if options.mask or options.maskfile or options.threshold or options.clip or options.threshold or options.normproc or options.preprocess or options.lowpass or options.highpass or int( options.shrink) > 1: preprocstack = str( os.path.basename(options.input).replace('.hdf', '_preproc.hdf')) if options.path and not options.nopath: preprocstack = options.path + '/' + preprocstack if options.output: if '.hdf' in options.output[-4:]: preprocstack = options.output else: print "\n(e2spt_preproc)(main) ERROR: '.hdf' must be the last four characters of the output filename." print "\n(e2spt_preproc)(main) output stack will be %s" % ( preprocstack) n = 0 try: n = EMUtil.get_image_count(options.input) except: print "\n(e2spt_preproc)(main) ERROR: --input stack seems to be invalid" sys.exit() print "\n(e2spt_preproc)(main) number of particles is %d" % (n) c = os.getcwd() findir = os.listdir(c) if preprocstack not in findir: dimg = EMData(8, 8, 8) dimg.to_one() for i in range(n): dimg.write_image(preprocstack, i) else: print "\n(e2spt_preproc)(main) WARNING: a file with the name of the output stack %s is already in the current directory and will be DELETED" % ( preprocstack) os.remove(preprocstack) dimg = EMData(8, 8, 8) dimg.to_one() for i in range(n): dimg.write_image(preprocstack, i) finalbox = EMData(options.input, 0, True)['nx'] if options.clip: finalbox = options.clip #dimglarge=EMData(finalbox,finalbox,finalbox) #dimglarge.to_one() #dimglarge.write_image(preprocstack,0) #n=EMUtil.get_image_count(options.input) #if options.subset: # n=options.subset #dimglarge.write_image(preprocstack,n-1) if options.verbose: print "\n(e2spt_preproc)(main) wrote dummy ptcls to %s" % ( preprocstack) print "\n(e2spt_preproc)(main) - INITIALIZING PARALLELISM!\n" if options.parallel: from EMAN2PAR import EMTaskCustomer etc = EMTaskCustomer(options.parallel) pclist = [options.input] etc.precache(pclist) print "\n(e2spt_preproc)(main) - precaching --input" tasks = [] results = [] from e2spt_classaverage import sptOptionsParser options = sptOptionsParser(options) for j in range(n): #print "processing particle", j img = EMData(options.input, j) if options.parallel: #task = Preproc3DTask( ["cache",options.input,j], options, j, preprocstack ) task = Preproc3DTask(img, options, j, preprocstack) tasks.append(task) else: img = EMData(options.input, j) pimg = preprocfunc(img, options, j, preprocstack) if options.parallel and tasks: tids = etc.send_tasks(tasks) if options.verbose: print "\n(e2spt_preproc)(main) preprocessing %d tasks queued" % ( len(tids)) results = get_results(etc, tids, options) #print "\n(e2spt_preproc)(main) preprocessing results are", results #print "\n(e2spt_preproc)(main) input changing to preprocstack" #options.input = preprocstack #cache needs to be reloaded with the new options.input else: print "\n(e2spt_preproc)(main) Nothing to do. No preprocessing parameters specified." E2end(logger) return
def main(): progname = os.path.basename(sys.argv[0]) usage = """prog <output> [options] Program to build an initial subtomogram average by averaging pairs from the largest subset in --input that is a power of 2. For example, if you supply an input stack with 100 subtomograms, this program will build an initial reference using 64, since 64 is the largest power of 2 contained in 100. In the first iteration, particle 1 will be averaged with 2, 3 with 4, 5 with 6... etc. 32 new averages (each an average of 2 subtomograms) will be used for the second iteration. Again, 1 will be averaged with 2, 3 with 4, etc... yielding 16 new averages. The algorithm continues until the entire subset (64) has been merged into 1 average. This program imports 'preprocfunc' from e2spt_preproc.py and 'alignment' from e2spt_classaverage.py --mask=mask.sharp:outer_radius=<safe radius> --preprocess=filter.lowpass.gauss:cutoff_freq=<1/resolution in A> """ parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_header(name="sptbtheader", help="""Options below this label are specific to sptbinarytree""", title="### sptbinarytree options ###", row=6, col=0, rowspan=1, colspan=3, mode="align") parser.add_header( name="caheader", help="""Options below this label are specific to sptclassaverage""", title="### sptclassaverage options ###", row=3, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument( "--path", type=str, default='spt_bt', help= """Default=spt. Directory to store results in. The default is a numbered series of directories containing the prefix 'spt'; for example, spt_02 will be the directory by default if 'spt_01' already exists.""" ) parser.add_argument( "--input", type=str, default='', help= """Default=None. The name of the input volume stack. MUST be HDF since volume stack support is required.""", guitype='filebox', browser='EMSubTomosTable(withmodal=True,multiselect=False)', row=0, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument( "--npeakstorefine", type=int, help= """Default=1. The number of best coarse alignments to refine in search of the best final alignment. Default=1.""", default=4, guitype='intbox', row=9, col=0, rowspan=1, colspan=1, nosharedb=True, mode='alignment,breaksym[1]') parser.add_argument( "--parallel", default="thread:1", help= """default=thread:1. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel""", guitype='strbox', row=19, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument( "--ppid", type=int, help= """Default=-1. Set the PID of the parent process, used for cross platform PPID""", default=-1) parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help= """Default=0. Verbose level [0-9], higner number means higher level of verboseness; 10-11 will trigger many messages that might make little sense since this level of verboseness corresponds to 'debugging mode'""" ) #parser.add_argument("--resume",type=str,default='',help="""(Not working currently). tomo_fxorms.json file that contains alignment information for the particles in the set. If the information is incomplete (i.e., there are less elements in the file than particles in the stack), on the first iteration the program will complete the file by working ONLY on particle indexes that are missing. For subsequent iterations, all the particles will be used.""") parser.add_argument( "--plots", action='store_true', default=False, help= """Default=False. Turn this option on to generatea plot of the ccc scores during each iteration. Running on a cluster or via ssh remotely might not support plotting.""" ) parser.add_argument( "--subset", type=int, default=0, help= """Default=0 (not used). Refine only this substet of particles from the stack provided through --input""" ) parser.add_argument( "--preavgproc1", type=str, default='', help= """Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)""" ) parser.add_argument( "--preavgproc2", type=str, default='', help= """Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)""" ) parser.add_argument( "--weighbytiltaxis", type=str, default='', help= """Default=None. A,B, where A is an integer number and B a decimal. A represents the location of the tilt axis in the tomogram in pixels (eg.g, for a 4096x4096xZ tomogram, this value should be 2048), and B is the weight of the particles furthest from the tomogram. For example, --weighbytiltaxis=2048,0.5 means that praticles at the tilt axis (with an x coordinate of 2048) will have a weight of 1.0 during averaging, while the distance in the x coordinates of particles not-on the tilt axis will be used to weigh their contribution to the average, with particles at the edge(0+radius or 4096-radius) weighing 0.5, as specified by the value provided for B.""" ) parser.add_argument( "--weighbyscore", action='store_true', default=False, help= """Default=False. This option will weigh the contribution of each subtomogram to the average by score/bestscore.""" ) #parser.add_argument("--align",type=str,default="rotate_translate_3d:search=8:delta=12:dphi=12",help="""This is the aligner used to align particles to the previous class average. Default is rotate_translate_3d:search=8:delta=12:dphi=12, specify 'None' (with capital N) to disable.""", returnNone=True,guitype='comboparambox', choicelist='re_filter_list(dump_aligners_list(),\'3d\')', row=12, col=0, rowspan=1, colspan=3, nosharedb=True, mode="alignment,breaksym['rotate_symmetry_3d']") parser.add_argument( "--align", type=str, default="rotate_translate_3d_tree", help= """Default is rotate_translate_3d_tree. See e2help.py aligners to see the list of parameters the aligner takes (for example, if there's symmetry, supply --align rotate_translate_3d_tree:sym=icos). This is the aligner used to align particles to the previous class average. Specify 'None' (with capital N) to disable.""" ) parser.add_argument( "--aligncmp", type=str, default="ccc.tomo.thresh", help= """Default=ccc.tomo.thresh. The comparator used for the --align aligner. Do not specify unless you need to use anotherspecific aligner.""", guitype='comboparambox', choicelist='re_filter_list(dump_cmps_list(),\'tomo\')', row=13, col=0, rowspan=1, colspan=3, mode="alignment,breaksym") #parser.add_argument("--output", type=str, default='avg.hdf', help="""Default=avg.hdf. The name of the output class-average stack. MUST be HDF since volume stack support is required.""", guitype='strbox', row=2, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') #parser.add_argument("--classmx", type=str, default='', help="""Default=None. The name of the classification matrix specifying how particles in 'input' should be grouped. If omitted, all particles will be averaged.""") #parser.add_argument("--ref", type=str, default='', help="""Default=None. Reference image(s). Used as an initial alignment reference and for final orientation adjustment if present. This is typically the projections that were used for classification.""", guitype='filebox', browser='EMBrowserWidget(withmodal=True,multiselect=True)', filecheck=False, row=1, col=0, rowspan=1, colspan=3, mode='alignment') #parser.add_argument("--refpreprocess",action="store_true",default=False,help="""Default=False. This will preprocess the reference identically to the particles. It is off by default, but it is internally turned on when no reference is supplied.""") #parser.add_argument("--resultmx",type=str,default=None,help="""Default=Npone. Specify an output image to store the result matrix. This is in the same format as the classification matrix. http://blake.bcm.edu/emanwiki/EMAN2/ClassmxFiles""") #parser.add_argument("--refinemultireftag", type=str, default='', help="""Default=''. DO NOT USE THIS PARAMETER. It is passed on from e2spt_refinemulti.py if needed.""") ''' ADVANCED parameters ''' parser.add_argument( "--averager", type=str, default="mean.tomo", help= """Default=mean.tomo. The type of averager used to produce the class average. Default=mean.tomo.""" ) ''' PRE-FFT processing parameters ''' #parser.add_argument("--nopreprocprefft",action="store_true",default=False,help="""Turns off all preprocessing that happens only once before alignment (--normproc, --mask, --maskfile, --clip, --threshold; i.e., all preprocessing excepting filters --highpass, --lowpass, --preprocess, and --shrink.""") parser.add_argument( "--shrink", type=int, default=1, help= """Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount for coarse alignment.""", guitype='shrinkbox', row=5, col=1, rowspan=1, colspan=1, mode='alignment,breaksym') parser.add_argument( "--shrinkfine", type=int, default=1, help= """Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount for refine alignment.""", guitype='intbox', row=5, col=2, rowspan=1, colspan=1, mode='alignment') parser.add_argument( "--threshold", type=str, default='', help= """Default=None. A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=10, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument( "--mask", type=str, default='', help= """Default=None. Masking processor applied to particles before alignment. IF using --clip, make sure to express outer mask radii as negative pixels from the edge.""", returnNone=True, guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'mask\')', row=11, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument( "--maskfile", type=str, default='', help= """Default=None. Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None.""" ) parser.add_argument( "--normproc", type=str, default='', help= """Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to particles before alignment. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'""" ) parser.add_argument( "--clip", type=int, default=0, help= """Default=0 (which means it's not used). Boxsize to clip particles as part of preprocessing to speed up alignment. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary; still, there are some benefits from 'oversampling' the data during averaging; so you might still want an average of size 2x, but perhaps particles in a box of 1.5x are sufficiently good for alignment. In this case, you would supply --clip=75""" ) ''' POST-FFT filtering parameters ''' parser.add_argument( "--preprocess", type=str, default='', help= """Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=10, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument( "--preprocessfine", type=str, default='', help= """Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging.""" ) parser.add_argument( "--lowpass", type=str, default='', help= """Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=17, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument( "--lowpassfine", type=str, default='', help= """Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging.""" ) parser.add_argument( "--highpass", type=str, default='', help= """Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=18, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument( "--highpassfine", type=str, default='', help= """Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging.""" ) parser.add_argument( "--matchimgs", action='store_true', default=False, help= """Default=False. Applies filter.matchto to one image so that it matches the other's spectral profile during preprocessing for pair-wise alignment purposes.""" ) parser.add_argument( "--filterbyfsc", action='store_true', default=False, help= """Default=False. If on, this parameter will use dynamic FSC filtering. --lowpass will be used to build initial references if no --ref supplied, then, the FSC between the even and odd initial references will be used to filter the data during preprocessing. If --ref is supplied, --lowpass will be used during the first iteration to align the particles against the reference. Thereafter, the FSC between the most current particle average and the original reference (--ref) will be used in the next iteration.""" ) ''' OTHER ADVANCED parameters ''' parser.add_argument( "--radius", type=float, default=0, help= """Default=0 (which means it's not used by default). Hydrodynamic radius of the particle in Angstroms. This will be used to automatically calculate the angular steps to use in search of the best alignment. Make sure the apix is correct on the particles' headers, sine the radius will be converted from Angstroms to pixels. Then, the fine angular step is equal to 360/(2*pi*radius), and the coarse angular step 4 times that.""" ) parser.add_argument( "--precision", type=float, default=1.0, help= """Default=1.0. Precision in pixels to use when figuring out alignment parameters automatically using --radius. Precision would be the number of pixels that the the edge of the specimen is moved (rotationally) during the finest sampling, --falign. If precision is 1, then the precision of alignment will be that of the sampling (apix of your images) times the --shrinkfine factor specified.""" ) parser.add_argument( "--search", type=int, default=8, help= """"Default=8. During COARSE alignment translational search in X, Y and Z, in pixels. Default=8. This WILL overwrite any search: provided through --align, EXCEPT if you provide --search=8, which is the default. In general, just avoid providing search twice (through here and through the aligner, --align). If you do, just be careful to make them consistent to minimize misinterpretation and error.""" ) parser.add_argument( "--searchfine", type=int, default=2, help= """"Default=2. During FINE alignment translational search in X, Y and Z, in pixels. Default=2. This WILL overwrite any search: provided through --falign, EXCEPT if you provide --searchfine=2, which is the default. In general, just avoid providing search twice (through here and through the fine aligner --falign). If you do, just be careful to make them consistent to minimize misinterpretation and error.""" ) #parser.add_argument("--donotaverage",action="store_true", help="""If e2spt_refinemulti.py is calling e2spt_classaverage.py, the latter need not average any particles, but rather only yield the alignment results.""", default=False) parser.add_argument( "--iterstop", type=int, default=0, help= """Default=0. (Not used). The program is called to convergence by default (all particles merge into one final average). To stop at an intermediate iteration, provide this parameter. For example, --iterstop=1, will only allow the algorithm to complete 1 iteration; --iterstop=2 will allow it to go through 2, etc.""" ) parser.add_argument( "--savesteps", action="store_true", default=False, help= """Default=False. If set, will save the average after each iteration to class_#.hdf. Each class in a separate file. Appends to existing files.""", guitype='boolbox', row=4, col=0, rowspan=1, colspan=1, mode='alignment,breaksym') parser.add_argument( "--saveali", action="store_true", default=False, help= """Default=False. If set, will save the aligned particle volumes in class_ptcl.hdf. Overwrites existing file.""", guitype='boolbox', row=4, col=1, rowspan=1, colspan=1, mode='alignment,breaksym') parser.add_argument( "--saveallalign", action="store_true", default=False, help= """Default=False. If set, will save the alignment parameters after each iteration""", guitype='boolbox', row=4, col=2, rowspan=1, colspan=1, mode='alignment,breaksym') parser.add_argument( "--sym", dest="sym", default='', help= """Default=None (equivalent to c1). Symmetry to impose -choices are: c<n>, d<n>, h<n>, tet, oct, icos""", guitype='symbox', row=9, col=1, rowspan=1, colspan=2, mode='alignment,breaksym') parser.add_argument( "--postprocess", type=str, default='', help= """A processor to be applied to the FINAL volume after averaging the raw volumes in their FINAL orientations, after all iterations are done.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=16, col=0, rowspan=1, colspan=3, mode='alignment,breaksym') parser.add_argument( "--procfinelikecoarse", action='store_true', default=False, help= """If you supply this parameters, particles for fine alignment will be preprocessed identically to particles for coarse alignment by default. If you supply this, but want specific parameters for preprocessing particles for also supply: fine alignment, nd supply fine alignment parameters, such as --lowpassfine, --highpassfine, etc; to preprocess the particles for FINE alignment differently than for COARSE alignment.""" ) parser.add_argument( "--falign", type=str, default=None, help= """Default=None. This is the second stage aligner used to fine-tune the first alignment.""", returnNone=True, guitype='comboparambox', choicelist='re_filter_list(dump_aligners_list(),\'refine.*3d\')', row=14, col=0, rowspan=1, colspan=3, nosharedb=True, mode='alignment,breaksym[None]') parser.add_argument( "--faligncmp", type=str, default="ccc.tomo.thresh", help= """Default=ccc.tomo.thresh. The comparator used by the second stage aligner.""", guitype='comboparambox', choicelist='re_filter_list(dump_cmps_list(),\'tomo\')', row=15, col=0, rowspan=1, colspan=3, mode="alignment,breaksym") #parser.add_argument("--nopreprocprefft",action="store_true",default=False,help="""Turns off all preprocessing that happens only once before alignment (--normproc, --mask, --maskfile, --clip, --threshold; i.e., all preprocessing excepting filters --highpass, --lowpass, --preprocess, and --shrink.""") #parser.add_argument("--keep",type=float,default=1.0,help="""Default=1.0 (all particles kept). The fraction of particles to keep in each class.""", guitype='floatbox', row=6, col=0, rowspan=1, colspan=1, mode='alignment,breaksym') #parser.add_argument("--keepsig", action="store_true", default=False,help="""Default=False. Causes the keep argument to be interpreted in standard deviations.""", guitype='boolbox', row=6, col=1, rowspan=1, colspan=1, mode='alignment,breaksym') #parser.add_argument("--inixforms",type=str,default="",help="""Default=None. .json file containing a dict of transforms to apply to 'pre-align' the particles.""", guitype='dirbox', dirbasename='spt_|sptsym_', row=7, col=0,rowspan=1, colspan=2, nosharedb=True, mode='breaksym') parser.add_argument( "--breaksym", action="store_true", default=False, help= """Default=False. Break symmetry. Do not apply symmetrization after averaging, even if searching the asymmetric unit provided through --sym only for alignment. Default=False""", guitype='boolbox', row=7, col=2, rowspan=1, colspan=1, nosharedb=True, mode=',breaksym[True]') #parser.add_argument("--groups",type=int,default=0,help="""Default=0 (not used; data not split). This parameter will split the data into a user defined number of groups. For purposes of gold-standard FSC computation later, select --group=2.""") parser.add_argument( "--randomizewedge", action="store_true", default=False, help= """Default=False. This parameter is EXPERIMENTAL. It randomizes the position of the particles BEFORE alignment, to minimize missing wedge bias and artifacts during symmetric alignment where only a fraction of space is scanned""" ) #parser.add_argument("--savepreproc",action="store_true", default=False,help="""Default=False. Will save stacks of preprocessed particles (one for coarse alignment and one for fine alignment if preprocessing options are different).""") parser.add_argument( "--autocenter", type=str, default='', help= """Default=None. Autocenters each averaged pair during initial average generation with --btref and --hacref. Will also autocenter the average of all particles after each iteration of iterative refinement. Options are --autocenter=xform.centerofmass (self descriptive), or --autocenter=xform.centeracf, which applies auto-convolution on the average.""" ) parser.add_argument( "--autocentermask", type=str, default='', help= """Default=None. Masking processor to apply before autocentering. See 'e2help.py processors -v 10' at the command line.""" ) parser.add_argument( "--autocenterpreprocess", action='store_true', default=False, help= """Default=False. This will apply a highpass filter at a frequency of half the box size times the apix, shrink by 2, and apply a low pass filter at half nyquist frequency to any computed average for autocentering purposes if --autocenter is provided. Default=False.""" ) parser.add_argument( "--tweak", action='store_true', default=False, help= """WARNING: BUGGY. This will perform a final alignment with no downsampling [without using --shrink or --shrinkfine] if --shrinkfine > 1.""" ) ''' BT SPECIFIC PARAMETERS ''' parser.add_argument("--nseedlimit", type=int, default=0, help="""Maximum number of particles to use. For example, if you supply a stack with 150 subtomograms, the program will automatically select 128 as the limit to use because it's the largest power of 2 that is smaller than 150. But if you provide, say --nseedlimit=100, then the number of particles used will be 64, because it's the largest power of 2 that is still smaller than 100.""" ) (options, args) = parser.parse_args() (optionsUnparsed, args) = parser.parse_args() options.nopreprocprefft = False if options.shrink < options.shrinkfine: options.shrink = options.shrinkfine print( "\n(e2spt_binarytree)(main) it makes no sense for shrinkfine to be larger than shrink; therefore, shrink will be made to match shrinkfine" ) options = checkinput(options) from e2spt_classaverage import checksaneimagesize checksaneimagesize(options, options.input) ''' Make the directory where to create the database where the results will be stored ''' #from e2spt_classaverage import sptmakepath #options = sptmakepath(options,'spt_bt') from EMAN2_utils import makepath options = makepath(options, 'spt_bt') optionsUnparsed.path = options.path rootpath = os.getcwd() if rootpath not in options.path: options.path = rootpath + '/' + options.path if not options.input: parser.print_help() exit(0) elif options.subset: subsetStack = options.path + '/subset' + str(options.subset).zfill( len(str(options.subset))) + '.hdf' print("\nSubset to be written to", subsetStack) subsetcmd = 'e2proc3d.py ' + options.input + ' ' + subsetStack + ' --first=0 --last=' + str( options.subset - 1) print("Subset cmd is", subsetcmd) p = subprocess.Popen(subsetcmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) text = p.communicate() p.stdout.close() options.input = subsetStack from e2spt_classaverage import sptParseAligner options = sptParseAligner(options) ''' If --radius of the particle is provided, we calculate the optimal alignment steps for coarse and fine alignment rounds using --shrink and --shrinkfine options and apix info ''' if options.radius: from e2spt_classaverage import calcAliStep options = calcAliStep(options) '''c: c:Parse parameters such that "None" or "none" are adequately interpreted to turn off an option c:''' options = sptOptionsParser(options) print("\nAFTER PARSING options type={}".format(type(options))) print("\nAFTER PARSING optionsUnparsed type={}".format( type(optionsUnparsed))) writeParameters(options, 'e2spt_binarytree.py', 'spt_bt') hdr = EMData(options.input, 0, True) nx = hdr["nx"] ny = hdr["ny"] nz = hdr["nz"] if nx != ny or ny != nz: print("ERROR, input volumes are not cubes") sys.exit(1) logger = E2init(sys.argv, options.ppid) ''' Initialize parallelism if being used; it will be turned on automatically by detectThreads unless --parallel-=None ''' options = detectThreads(options) if options.parallel: print("\n\n(e2spt_classaverage.py) INITIALIZING PARALLELISM!") print("\n\n") from EMAN2PAR import EMTaskCustomer etc = EMTaskCustomer(options.parallel) pclist = [options.input] etc.precache(pclist) else: etc = '' options.raw = options.input """ if 'tree' in options.align: options.falign = None options.mask = None options.lowpass = None options.highpass = None options.normproc = None options.lowpassfine = None options.highpassfine = None options.preprocess = None options.preprocessfine = None else: from e2spt_classaverage import cmdpreproc cmdpreproc( options.input, options, False ) """ nptcl = EMUtil.get_image_count(options.input) if nptcl < 1: print("ERROR : at least 2 particles required in input stack") sys.exit(1) ptclnums = list(range(nptcl)) nptclForRef = len(ptclnums) nseed = 2**int( floor(log(len(ptclnums), 2)) ) # we stick with powers of 2 for this to make the tree easier to collapse if options.nseedlimit: nseed = 2**int(floor(log(options.nseedlimit, 2))) #binaryTreeRef(options,nptclForRef,nseed,-1,etc) binaryTreeRef(options, optionsUnparsed, nptclForRef, nseed, etc) print("Will end logger") E2end(logger) print("logger ended") sys.stdout.flush() return
def main(): progname = os.path.basename(sys.argv[0]) usage = """prog <output> [options] This program produces iterative class-averages, one of the secrets to EMAN's rapid convergence. Normal usage is to provide a stack of particle images and a classification matrix file defining class membership. Members of each class are then iteratively aligned to each other and averaged together with (optional) CTF correction. It is also possible to use this program on all of the images in a single stack. """ parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_argument("--input", type=str, help="The name of the input particle stack", default=None) parser.add_argument("--output", type=str, help="The name of the output class-average stack", default=None) parser.add_argument( "--oneclass", type=int, help="Create only a single class-average. Specify the number.", default=None) parser.add_argument( "--classmx", type=str, help= "The name of the classification matrix specifying how particles in 'input' should be grouped. If omitted, all particles will be averaged.", default=None) parser.add_argument( "--ref", type=str, help= "Reference image(s). Used as an initial alignment reference and for final orientation adjustment if present. Also used to assign euler angles to the generated classes. This is typically the projections that were used for classification.", default=None) parser.add_argument( "--storebad", action="store_true", help= "Even if a class-average fails, write to the output. Forces 1->1 numbering in output", default=False) parser.add_argument( "--decayedge", action="store_true", help= "Applies an edge decay to zero on the output class-averages. A very good idea if you plan on 3-D reconstruction.", default=False) parser.add_argument( "--resultmx", type=str, help= "Specify an output image to store the result matrix. This contains 5 images where row is particle number. Rows in the first image contain the class numbers and in the second image consist of 1s or 0s indicating whether or not the particle was included in the class. The corresponding rows in the third, fourth and fifth images are the refined x, y and angle (respectively) used in the final alignment, these are updated and accurate, even if the particle was excluded from the class.", default=None) parser.add_argument( "--iter", type=int, help="The number of iterations to perform. Default is 1.", default=1) parser.add_argument( "--prefilt", action="store_true", help= "Filter each reference (c) to match the power spectrum of each particle (r) before alignment and comparison", default=False) parser.add_argument( "--align", type=str, help= "This is the aligner used to align particles to the previous class average. Default is None.", default=None) parser.add_argument( "--aligncmp", type=str, help="The comparitor used for the --align aligner. Default is ccc.", default="ccc") parser.add_argument( "--ralign", type=str, help= "This is the second stage aligner used to refine the first alignment. This is usually the \'refine\' aligner.", default=None) parser.add_argument( "--raligncmp", type=str, help="The comparitor used by the second stage aligner.", default="ccc") parser.add_argument( "--averager", type=str, help="The type of averager used to produce the class average.", default="mean") parser.add_argument( "--setsfref", action="store_true", help= "This will impose the 1-D structure factor of the reference on the class-average (recommended when a reference is available)", default=False) parser.add_argument( "--cmp", type=str, help= "The comparitor used to generate quality scores for the purpose of particle exclusion in classes, strongly linked to the keep argument.", default="ccc") parser.add_argument( "--keep", type=float, help="The fraction of particles to keep in each class.", default=1.0) parser.add_argument( "--keepsig", action="store_true", help= "Causes the keep argument to be interpreted in standard deviations.", default=False) parser.add_argument( "--automask", action="store_true", help= "Applies a 2-D automask before centering. Can help with negative stain data, and other cases where centering is poor." ) parser.add_argument( "--center", type=str, default="xform.center", help= "If the default centering algorithm (xform.center) doesn't work well, you can specify one of the others here (e2help.py processor center)" ) parser.add_argument("--bootstrap", action="store_true", help="Ignored. Present for historical reasons only.") parser.add_argument( "--normproc", type=str, help= "Normalization processor applied to particles before alignment. Default is normalize.edgemean. If you want to turn this option off specify \'None\'", default="normalize.edgemean") parser.add_argument( "--usefilt", dest="usefilt", default=None, help= "Specify a particle data file that has been low pass or Wiener filtered. Has a one to one correspondence with your particle data. If specified will be used to align particles to the running class average, however the original particle will be used to generate the actual final class average" ) parser.add_argument("--idxcache", default=False, action="store_true", help="Ignored. Present for historical reasons.") parser.add_argument("--dbpath", help="Ignored. Present for historical reasons.", default=".") parser.add_argument( "--resample", action="store_true", help= "If set, will perform bootstrap resampling on the particle data for use in making variance maps.", default=False) parser.add_argument( "--odd", default=False, help= "Used by EMAN2 when running eotests. Includes only odd numbered particles in class averages.", action="store_true") parser.add_argument( "--even", default=False, help= "Used by EMAN2 when running eotests. Includes only even numbered particles in class averages.", action="store_true") parser.add_argument("--parallel", default=None, help="parallelism argument") parser.add_argument("--force", "-f", dest="force", default=False, action="store_true", help="Force overwrite the output file if it exists.") parser.add_argument( "--saveali", action="store_true", help= "Writes aligned particle images to aligned.hdf. Normally resultmx produces more useful informtation. This can be used for debugging.", default=False) parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help= "verbose level [0-9], higner number means higher level of verboseness") parser.add_argument( "--debug", "-d", action="store_true", help= "Print debugging infromation while the program is running. Default is off.", default=False) parser.add_argument( "--nofilecheck", action="store_true", help= "Turns file checking off in the check functionality - used by e2refine.py.", default=False) parser.add_argument("--check", "-c", action="store_true", help="Performs a command line argument check only.", default=False) parser.add_argument( "--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-1) (options, args) = parser.parse_args() if (options.check): options.verbose = 9 # turn verbose on if the user is only checking... error = check(options, True) if options.align: options.align = parsemodopt(options.align) if options.ralign: options.ralign = parsemodopt(options.ralign) if options.aligncmp: options.aligncmp = parsemodopt(options.aligncmp) if options.raligncmp: options.raligncmp = parsemodopt(options.raligncmp) if options.averager: options.averager = parsemodopt(options.averager) if options.cmp: options.cmp = parsemodopt(options.cmp) if options.normproc: options.normproc = parsemodopt(options.normproc) if options.resultmx != None: options.storebad = True if (options.verbose > 0): if (error): print "e2classaverage.py command line arguments test.... FAILED" else: print "e2classaverage.py command line arguments test.... PASSED" # returning a different error code is currently important to e2refine.py - returning 0 tells e2refine.py that it has enough # information to execute this script if error: exit(1) if options.check: exit(0) logger = E2init(sys.argv, options.ppid) print "Class averaging beginning" try: classmx = EMData.read_images( options.classmx ) # we keep the entire classification matrix in memory, since we need to update it in most cases ncls = int(classmx[0]["maximum"]) + 1 except: ncls = 1 if options.resultmx != None: print "resultmx can only be specified in conjunction with a valid classmx input." sys.exit(1) nptcl = EMUtil.get_image_count(options.input) try: apix = EMData(options.input, 0, True)["apix_x"] except: apix = 1.0 print "WARNING: could not get apix from first image. Setting to 1.0. May impact results !" # Initialize parallelism if options.parallel: from EMAN2PAR import EMTaskCustomer etc = EMTaskCustomer(options.parallel) pclist = [options.input] if options.ref: pclist.append(options.ref) if options.usefilt: pclist.append(options.usefilt) etc.precache(pclist) # prepare tasks tasks = [] if ncls > 1: if options.oneclass == None: clslst = range(ncls) else: clslst = [options.oneclass] for cl in clslst: ptcls = classmx_ptcls(classmx[0], cl) if options.resample: ptcls = [ random.choice(ptcls) for i in ptcls ] # this implements bootstrap resampling of the class-average if options.odd: ptcls = [i for i in ptcls if i % 2 == 1] if options.even: ptcls = [i for i in ptcls if i % 2 == 0] tasks.append( ClassAvTask(options.input, ptcls, options.usefilt, options.ref, options.iter, options.normproc, options.prefilt, options.align, options.aligncmp, options.ralign, options.raligncmp, options.averager, options.cmp, options.keep, options.keepsig, options.automask, options.saveali, options.setsfref, options.verbose, cl, options.center)) else: ptcls = range(nptcl) if options.resample: ptcls = [random.choice(ptcls) for i in ptcls] if options.odd: ptcls = [i for i in ptcls if i % 2 == 1] if options.even: ptcls = [i for i in ptcls if i % 2 == 0] tasks.append( ClassAvTask(options.input, range(nptcl), options.usefilt, options.ref, options.iter, options.normproc, options.prefilt, options.align, options.aligncmp, options.ralign, options.raligncmp, options.averager, options.cmp, options.keep, options.keepsig, options.automask, options.saveali, options.setsfref, options.verbose, 0, options.center)) # execute task list if options.parallel: # run in parallel taskids = etc.send_tasks(tasks) alltaskids = taskids[:] while len(taskids) > 0: curstat = etc.check_task(taskids) for i, j in enumerate(curstat): if j == 100: rslt = etc.get_results(taskids[i]) if rslt[1]["average"] != None: rslt[1]["average"]["class_ptcl_src"] = options.input if options.decayedge: nx = rslt[1]["average"]["nx"] rslt[1]["average"].process_inplace( "normalize.circlemean", {"radius": nx / 2 - nx / 15}) rslt[1]["average"].process_inplace( "mask.gaussian", { "inner_radius": nx / 2 - nx / 15, "outer_radius": nx / 20 }) #rslt[1]["average"].process_inplace("mask.decayedge2d",{"width":nx/15}) if options.ref != None: rslt[1]["average"][ "projection_image"] = options.ref if options.storebad: rslt[1]["average"].write_image( options.output, rslt[1]["n"]) else: rslt[1]["average"].write_image(options.output, -1) # Update the resultsmx if requested if options.resultmx != None: allinfo = rslt[1][ "info"] # the info result array list of (qual,xform,used) tuples pnums = rslt[0].data["images"][ 2] # list of image numbers corresponding to information for n, info in enumerate(allinfo): y = pnums[n] # actual particle number # find the matching class in the existing classification matrix for x in range(classmx[0]["nx"]): if classmx[0][x, y] == rslt[1][ "n"]: # if the class number in the classmx matches the current class-average number break else: print "Resultmx error: no match found ! (%d %d %d)" % ( x, y, rslt[1]["n"]) continue xform = info[1].get_params("2d") classmx[1][x, y] = info[2] # used classmx[2][x, y] = xform["tx"] # dx classmx[3][x, y] = xform["ty"] # dy classmx[4][x, y] = xform["alpha"] # da classmx[5][x, y] = xform["mirror"] # flip try: classmx[6][x, y] = xform["scale"] except: pass # failed average elif options.storebad: blk = EMData(options.ref, 0) apix = blk["apix_x"] blk = EMData(blk["nx"], blk["ny"], 1) blk["apix_x"] = apix blk.to_zero() blk.set_attr("ptcl_repr", 0) blk.set_attr("apix_x", apix) blk.write_image(options.output, rslt[1]["n"]) taskids = [j for i, j in enumerate(taskids) if curstat[i] != 100] if options.verbose and 100 in curstat: print "%d/%d tasks remain" % (len(taskids), len(alltaskids)) if 100 in curstat: E2progress(logger, 1.0 - (float(len(taskids)) / len(alltaskids))) time.sleep(3) if options.verbose: print "Completed all tasks" # single thread else: for t in tasks: rslt = t.execute() if rslt == None: sys.exit(1) if rslt["average"] != None: rslt["average"]["class_ptcl_src"] = options.input if options.decayedge: nx = rslt["average"]["nx"] rslt["average"].process_inplace( "normalize.circlemean", {"radius": nx / 2 - nx / 15}) rslt["average"].process_inplace( "mask.gaussian", { "inner_radius": nx / 2 - nx / 15, "outer_radius": nx / 20 }) #rslt["average"].process_inplace("mask.decayedge2d",{"width":nx/15}) if options.ref != None: rslt["average"]["projection_image"] = options.ref try: if options.storebad: rslt["average"].write_image(options.output, t.options["n"]) else: rslt["average"].write_image(options.output, -1) except: traceback.print_exc() print "Error writing class average {} to {}".format( t.options["n"], options.output) print "Image attr: ", rslt["average"].get_attr_dict() display(rslt["average"]) sys.exit(1) # Update the resultsmx if requested if options.resultmx != None: allinfo = rslt[ "info"] # the info result array list of (qual,xform,used) tuples pnums = t.data["images"][ 2] # list of image numbers corresponding to information for n, info in enumerate(allinfo): y = pnums[n] # actual particle number # find the matching class in the existing classification matrix for x in range(classmx[0]["nx"]): if classmx[0][x, y] == rslt[ "n"]: # if the class number in the classmx matches the current class-average number break else: print "Resultmx error: no match found ! (%d %d %d)" % ( x, y, rslt[1]["n"]) continue xform = info[1].get_params("2d") classmx[1][x, y] = info[2] # used classmx[2][x, y] = xform["tx"] # dx classmx[3][x, y] = xform["ty"] # dy classmx[4][x, y] = xform["alpha"] # da classmx[5][x, y] = xform["mirror"] # flip try: classmx[6][x, y] = xform["scale"] except: pass # Failed average elif options.storebad: blk = EMData(options.ref, 0) apix = blk["apix_x"] blk = EMData(blk["nx"], blk["ny"], 1) blk["apix_x"] = apix blk.to_zero() blk.set_attr("ptcl_repr", 0) blk.set_attr("apix_x", apix) blk.write_image(options.output, t.options["n"]) if options.resultmx != None: if options.verbose: print "Writing results matrix" for i, j in enumerate(classmx): j.write_image(options.resultmx, i) print "Class averaging complete" E2end(logger)
class EMParallelSimMX: def __init__(self,options,args,logger=None): ''' @param options the options produced by (options, args) = parser.parse_args() @param args the options produced by (options, args) = parser.parse_args() @param logger and EMAN2 logger, i.e. logger=E2init(sys.argv) assumes you have already called the check function. ''' self.options = options self.args = args self.logger = logger from EMAN2PAR import EMTaskCustomer self.etc=EMTaskCustomer(options.parallel) if options.colmasks!=None : self.etc.precache([args[0],args[1],options.colmasks]) else : self.etc.precache([args[0],args[1]]) self.num_cpus = self.etc.cpu_est() if self.num_cpus < 32: # lower limit self.num_cpus = 32 self.__task_options = None def __get_task_options(self,options): ''' Get the options required by each task as a dict @param options is always self.options - the initialization argument. Could be changed. ''' if self.__task_options == None: d = {} d["align"] = parsemodopt(options.align) d["aligncmp"] = parsemodopt(options.aligncmp) d["cmp"] = parsemodopt(options.cmp) if hasattr(options,"ralign") and options.ralign != None: d["ralign"] = parsemodopt(options.ralign) d["raligncmp"] = parsemodopt(options.raligncmp) # raligncmp must be specified if using ralign else: d["ralign"] = None d["raligncmp"] = None d["prefilt"]=options.prefilt if hasattr(options,"shrink") and options.shrink != None: d["shrink"] = options.shrink else: d["shrink"] = None self.__task_options = d return self.__task_options def __init_memory(self,options): ''' @param options is always self.options - the initialization argument. Could be changed. Establishes several important attributes they are: ---- self.clen - the number of images in the image defined by args[0], the number of columns in the similarity matrix self.rlen - the number of images in the image defined by args[1], the number of rows in the similarity matrix ---- Also, since we adopted region output writing as our preferred approach, this function makes sure the output image(s) exists on disk and has the correct dimensions - seeing as this is the way region writing works (the image has to exist on disk and have its full dimensions) ''' self.clen=EMUtil.get_image_count(self.args[0]) self.rlen=EMUtil.get_image_count(self.args[1]) output = self.args[2] if file_exists(output) and not options.fillzero: if options.force: remove_file(output) else: raise RuntimeError("The output file exists. Please remove it or specify the force option") e = EMData(self.clen,self.rlen) e.to_zero() e.set_attr(PROJ_FILE_ATTR,self.args[0]) e.set_attr(PART_FILE_ATTR,self.args[1]) n = 1 if self.options.saveali: n = 6 # the total number of images written to disk if not options.fillzero : e.write_image(output,0) for i in range(1,n): e.write_image(output,i) def __get_blocks(self): ''' Gets the blocks that will be processed in parallel, these are essentially ranges ''' steve_factor = 3 # increase number of jobs a bit for better distribution total_jobs = steve_factor*self.num_cpus [col_div,row_div] = opt_rectangular_subdivision(self.clen,self.rlen,total_jobs) block_c = self.clen/col_div block_r = self.rlen/row_div residual_c = self.clen-block_c*col_div # residual left over by integer division blocks = [] current_c = 0 for c in xrange(0,col_div): last_c = current_c + block_c if residual_c > 0: last_c += 1 residual_c -= 1 current_r = 0 residual_r = self.rlen-block_r*row_div # residual left over by integer division for r in xrange(0,row_div) : last_r = current_r + block_r if residual_r > 0: last_r += 1 residual_r -= 1 blocks.append([current_c,last_c,current_r,last_r]) current_r = last_r current_c = last_c # print col_div,row_div,col_div*row_div # print self.clen,self.rlen,residual_c,residual_r return blocks def execute(self): ''' The main function to be called ''' if len(self.options.parallel) > 1 : self.__init_memory(self.options) blocks = self.__get_blocks() # print blocks # self.check_blocks(blocks) # testing function can be removed at some point tasks=[] for bn,block in enumerate(blocks): data = {} data["references"] = ("cache",self.args[0],block[0],block[1]) data["particles"] = ("cache",self.args[1],block[2],block[3]) if self.options.colmasks!=None : data["colmasks"] = ("cache",self.options.colmasks,block[0],block[1]) if self.options.mask!=None : data["mask"] = ("cache",self.options.mask,0,1) if self.options.fillzero : # for each particle check to see which portion of the matrix we need to fill if (bn%10==0) : print "%d/%d \r"%(bn,len(blocks)), sys.stdout.flush() rng=[] for i in range(block[2],block[3]): c=EMData() c.read_image(self.args[2],0,False,Region(block[0],i,block[1]-block[0]+1,1)) inr=0 st=0 for j in range(c["nx"]): if c[j]==0 and not inr: st=j inr=1 if c[j]!=0 and inr: rng.append((i,st+block[0],j-1+block[0])) inr=0 if inr : rng.append((i,st+block[0],j+block[0])) data["partial"]=rng # print "%d) %s\t"%(bn,str(block)),rng if self.options.fillzero and len(data["partial"])==0 : continue # nothing to compute in this block, skip it completely else : task = EMSimTaskDC(data=data,options=self.__get_task_options(self.options)) #print "Est %d CPUs"%etc.cpu_est() tasks.append(task) # This just verifies that all particles have at least one class #a=set() #for i in tasks: #for k in i.data["partial"] : a.add(k[0]) #b=set(range(self.rlen)) #b-=a #print b print "%d/%d "%(bn,len(blocks)) self.tids=self.etc.send_tasks(tasks) print len(self.tids)," tasks submitted" # while 1: if len(self.tids) == 0: break print len(self.tids),"simmx tasks left in main loop \r", sys.stdout.flush() st_vals = self.etc.check_task(self.tids) for i in xrange(len(self.tids)-1,-1,-1): st = st_vals[i] if st==100: tid = self.tids[i] try: rslts = self.etc.get_results(tid) # display(rslts[1]["rslt_data"][0]) self.__store_output_data(rslts[1]) except: traceback.print_exc() print "ERROR storing results for task %d. Rerunning."%tid self.etc.rerun_task(tid) continue if self.logger != None: E2progress(self.logger,1.0-len(self.tids)/float(len(blocks))) if self.options.verbose>0: print "%d/%d\r"%(len(self.tids),len(blocks)) sys.stdout.flush() self.tids.pop(i) print len(self.tids),"simmx tasks left in main loop \r", sys.stdout.flush() time.sleep(10) print "\nAll simmx tasks complete " # if using fillzero, we must fix the -1.0e38 values placed into empty cells if self.options.fillzero : l=EMData(self.args[2],0,True) rlen=l["ny"] clen=l["nx"] # launch_childprocess("e2proc2d.py %s %s"%(self.args[2],self.args[2]+"_x")) print "Filling noncomputed regions in similarity matrix (%dx%d)"%(clen,rlen) l=EMData() for r in range(rlen): l.read_image(self.args[2],0,False,Region(0,r,clen,1)) fill=l["maximum"]+.0001 l.process_inplace("threshold.belowtominval",{"minval":-1.0e37,"newval":fill}) l.write_image(self.args[2],0,EMUtil.ImageType.IMAGE_UNKNOWN,False,Region(0,r,clen,1)) print "Filling complete" else: raise NotImplementedError("The parallelism option you specified (%s) is not supported" %self.options.parallel ) def __store_output_data(self,rslts): ''' Store output data to internal images (matrices) @param a dictionary return by the EMSimTaskDC ''' result_data = rslts["rslt_data"] output = self.args[2] insertion_c = rslts["min_ref_idx"] insertion_r = rslts["min_ptcl_idx"] result_mx = result_data[0] r = Region(insertion_c,insertion_r,result_mx.get_xsize(),result_mx.get_ysize()) # Note this is region io - the init_memory function made sure the images exist and are the right dimensions (on disk) for i,mxout in enumerate(result_data): mxout.write_image(output,i,EMUtil.ImageType.IMAGE_UNKNOWN,False,r)
def main(): usage = """e2tomopreproc.py <imgs> <options> . This program takes a tiltseries ('.st' or '.ali' file from IMOD) and applies preprocessing operations to them, such as lowpass, highpass, masking, etc. The options should be supplied in "--option=value" format, replacing "option" for a valid option name, and "value" for an acceptable value for that option. """ parser = EMArgumentParser(usage=usage,version=EMANVERSION) parser.add_argument("--path",type=str,default='',help="""Directory to store results in. The default is a numbered series of directories containing the prefix 'tomopreproc'; for example, tomopreproc_02 will be the directory by default if 'tomopreproc_01' already exists.""") parser.add_pos_argument(name="stack_files",default="",help="Stacks or images to process.") parser.add_argument("--input",type=str,default='',help=""""tiltseries to process. redundant with --tiltseries, or with providing images as arguments (separated by a space: e2tomopreproc.py stack1.hdf stack2.hdf), but --input takes precedence.""") parser.add_argument("--tiltseries",type=str,default='',help=""""tiltseries to process. redundant with --input""") parser.add_argument("--tltfile",type=str,default='',help="""".tlt file containing the tilt angles for --tiltseries""") parser.add_argument("--outmode", type=str, default='', help="""All EMAN2 programs write images with 4-byte floating point values when possible by default. This allows specifying an alternate format when supported: float, int8, int16, int32, uint8, uint16, uint32. Values are rescaled to fill MIN-MAX range.""") parser.add_argument("--dontcleanup", action='store_true', default=False, help="""If specified, intermediate files will be kept.""") parser.add_argument("--clip",type=str,default='',help="""Default=None. This resizes the 2-D images in the tilt series. If one number is provided, then x and y dimensions will be made the same. To specify both dimensions, supply two numbers, --clip=x,y. Clipping will be about the center of the image.""") #parser.add_argument("--apix",type=float,default=0.0,help="""True apix of images to be written on final stack.""") parser.add_argument("--shrink", type=float,default=0.0,help="""Default=0.0 (no shrinking). Can use decimal numbers, larger than 1.0. Optionally shrink the images by this factor. Uses processor math.fft.resample.""") parser.add_argument("--threshold",type=str,default='',help="""Default=None. A threshold processor applied to each image.""") parser.add_argument("--mask",type=str,default='', help="""Default=None. Masking processor applied to each image.""") parser.add_argument("--maskbyangle",action='store_true',default=False,help="""Default=False. Requires --tltfile. This will mask out from tilted images the info that isn't present at the 0 tilt angle. It uses the tomo.tiltedgemask processor (type 'e2help.py processors' at the commandline to read a description of the processor and its parameters). Provide --maskbyanglefalloff and --maskbyanglesigma to modify the default parameters.""") parser.add_argument("--maskbyanglefalloff", type=int, default=4,help="""Default=4. Number of pixels over which --maskbyangle will fall off to zero.""") parser.add_argument("--maskbyanglesigma", type=float, default=2.0,help="""Default=2.0. Number of sigmas for the width of the gaussian fall off in --maskbyangle and --maskbyanglefalloff""") parser.add_argument("--normproc",type=str, default='',help="""Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to each image.""") parser.add_argument("--normalizeimod",action='store_true',default=False,help="""Default=False. This will apply 'newstack -float 2' to the input stack. requires IMOD.""") parser.add_argument("--preprocess",type=str,default='',help="""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each image.""") parser.add_argument("--lowpassfrac",type=float,default=0.0,help="""Default=0.0 (not used). Fraction of Nyquist to lowpass at. The processor used is filter.lowpass.tanh""") parser.add_argument("--highpasspix",type=int,default=0,help="""Default=0 (not used). Number of Fourier pixels to apply highpass filter at. The processor used is filter.highpass.gauss.""") parser.add_argument("--parallel",type=str, default="thread:1", help="""default=thread:1. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel""") parser.add_argument("--prenadminite",type=int, default=0, help="""Default=0. Requires IMOD to be installed. Used to apply prenad filtering to a tiltseries. This is the --minite parameter in IMOD's preNAD program (minimum number of iterations).""") parser.add_argument("--prenadmaxite",type=int, default=0, help="""Default=0. Requires IMOD to be installed. Used to apply prenad filtering to a tiltseries. This is the --maxite parameter in IMOD's preNAD program (maximum number of iterations).""") parser.add_argument("--prenadsigma",type=int, default=0, help="""Default=0. Requires IMOD to be installed. Used to apply prenad filtering to a tiltseries. This is the --sigma parameter in IMOD's preNAD program (initial sigma for 'smoothing structure tensor').""") parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n",type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness.") parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1) (options, args) = parser.parse_args() logger = E2init(sys.argv, options.ppid) print "\n(e2tomopreproc)(main) started log" from e2spt_classaverage import sptmakepath options = sptmakepath(options,'tomopreproc') #print "args are",args infiles = [] if not options.input: #try: # infiles.append( sys.argv[1] ) #except: if options.tiltseries: infiles.append( options.tiltseries ) else: if args: print "copying args to infiles" infiles = list(args) print "infiles are", infiles else: print "\n(e2tomopreproc)(main) ERROR: must provide input files as arguments or via the --input or --tiltseries parameters." if infiles: print "\n(e2tomopreproc)(main) identified --input", options.input #print " .ali in options.input[:-4]", '.ali' in options.input[-4:] #print "options.input[-4] is", options.input[-4:] for infile in infiles: if '.ali' in infile[-4:] or '.st' in infile[-3:] or '.mrc' in infile[-4:] or '.mrcs' in infile[-5:] or '.hdf' in infile[-4:]: pass else: print "\n(e2tomopreproc)(main) ERROR: invalid image extension %s for image %s. Extension must be .st, .ali, .hdf, .mrc or .mrcs" %(options.input.split('.')[-1], infile) sys.exit(1) else: print "\n(e2tomopreproc)(main) ERROR: no images found/provided" sys.exit(1) originalextension = infiles[0].split('.')[-1] angles = {} if options.maskbyangle or (options.prenadminite and options.prenadmaxite and options.prenadsigma): if not options.tltfile: print "\n(e2tomopreproc)(main) ERROR: --maskbyangle and --prenad parameters require --tltfile" sys.exit(1) else: f = open( options.tltfile, 'r' ) lines = f.readlines() print "\nnumber of lines read from --tltfile", len(lines) f.close() #print "lines in tlt file are", lines k=0 for line in lines: line = line.replace('\t','').replace('\n','') if line: angle = float(line) angles.update( { k:angle } ) if options.verbose: print "appending angle", angle k+=1 if len(angles) < 2: print "\nERROR: something went terribly wrong with parsing the --tltlfile. This program does not work on single images" sys.exit() if len(angles) < 2: print "\nERROR: (second angle check) something went terribly wrong with parsing the --tltlfile. This program does not work on single images" sys.exit() print "\n(e2spt_preproc)(main) - INITIALIZING PARALLELISM!\n" from EMAN2PAR import EMTaskCustomer etc=EMTaskCustomer(options.parallel) pclist=[options.input] etc.precache(pclist) print "\n(e2spt_preproc)(main) - precaching --input" tasks=[] results=[] mrcstacks = [] print "there are these many infiles to loop over", len(infiles) if options.lowpassfrac: hdr = EMData( infiles[0], 0, True ) apix = hdr['apix_x'] print "\n(e2spt_preproc)(main) apix is",apix nyquist = 2.0 * apix print "\n(e2spt_preproc)(main) therefore nyquist resolution is", nyquist print lowpassres = nyquist/options.lowpassfrac options.lowpassfrac = 1.0/(lowpassres) if float(options.shrink) > 1.0: options.lowpassfrac /= float(options.shrink) print "there's shrinking", options.shrink lowpassres = nyquist/options.lowpassfrac print "\n(e2spt_preproc)(main) and final lowpass frequency is", options.lowpassfrac print "corresponding to lowpassres of",lowpassres for infile in infiles: mrcstack = options.path + '/' + infile print "infile is", infile print "infile[-5:] is ", infile[-5:] if '.hdf' in infile[-5:]: print "replacing .hdf extension" mrcstack = options.path + '/' + infile.replace('.hdf','.mrc') if '.mrcs' in infile[-5:]: print "replacing .mrcs extension" mrcstack = options.path + '/' + infile.replace('.mrcs','.mrc') if '.st' in infile[-5:]: print "replacing .st extension" mrcstack = options.path + '/' + infile.replace('.st','.mrc') if '.ali' in infile[-5:]: print "replacing .ali extension" mrcstack = options.path + '/' + infile.replace('.ali','.mrc') if '.tif' in infile[-5:]: print "replacing .ali extension" mrcstack = options.path + '/' + infile.replace('.tif','.mrc') #go = 0 #if go: print "mrcstack is",mrcstack #outname = outname.replace('.mrc','.mrcs') mrcstacks.append( mrcstack ) go = 0 if options.maskbyangle: outname = mrcstack.replace('.mrc','_UNSTACKED.mrc') print "therefore, outname is", outname cmd = 'e2proc2d.py ' + infile + ' ' + outname + ' --unstacking --threed2twod' #from shutil import copyfile #copyfile(options.input, outname) #print "copied input to", outname if options.outmode: cmd += ' --outmode=' + options.outmode if options.verbose: cmd += ' --verbose=' + str(options.verbose) print "\ncommand to unstack original input tiltseries is", cmd print "\n(e2tomopreproc)(main) unstacking command is", cmd p = subprocess.Popen( cmd , shell=True,stdout=subprocess.PIPE, stderr=subprocess.PIPE) #p = subprocess.Popen( cmd , shell=True, stdout=subprocess.PIPE) text = p.communicate() #p.stdout.close() p.wait() if p.returncode == 0: go = 1 else: go = 1 if go: imgs = [] if options.maskbyangle: c = os.getcwd() + '/' + options.path findir = os.listdir( os.getcwd() + '/' + options.path ) print "\n(e2tomopreproc)(main) directory to look for images is", c for f in findir: #if '.mrcs' in f: if "_UNSTACKED" in f: imgs.append( options.path + '/' +f ) kk=0 imgs.sort() print "\n(e2spt_preproc)(main) found these many images", len( imgs ) for img in imgs: #task=None #if options.maskbyangle: outimage = img.replace('.mrc','_preproc.mrc') task = TomoPreproc2DTask( img, options, angles[kk], outimage ) tasks.append(task) kk+=1 else: outimage = options.path + '/' + infile.replace('.mrc','_preproc.mrcs') task = TomoPreproc2DTask( infile, options, 0, outimage ) tasks.append(task) #else: # newmrcs = mrcstack.replace('.mrc','.mrcs') # print "copying file %s to %s" %(infile,newmrcs) # copyfile( infile, newmrcs ) # imgs.append( newmrcs ) #print "and the final lowpass frequency will be", options.lowpassfrac tids = etc.send_tasks(tasks) if options.verbose: print "\n(e2spt_preproc)(main) preprocessing %d tasks queued" % (len(tids)) results = get_results( etc, tids, options ) print "\n(e2tomopreproc)(main) these many images have been processsed",len(results) imgspreproc = [] findir = os.listdir( os.getcwd() + '/' + options.path ) #for mrcstack in mrcstacks: for f in findir: if "_preproc.mrc" in f: print "found preprocessed image", f imgspreproc.append( options.path + '/' + f ) else: print "this file is NOT a preprocessed image", f imgspreproc.sort() print "\n(e2tomopreproc)(main) these many preprocessed images loaded", len(imgspreproc) finalfiles=[] if options.maskbyangle: outfile = mrcstack.replace('.mrc','.mrcs') print "for RESTACKING" print "\n\n\noutfile is", outfile for f in imgspreproc: print "appending image %s to outfile %s" %(f,outfile) cmd = 'e2proc2d.py ' + f + ' ' + outfile if options.outmode: cmd += ' --outmode=' + options.outmode if options.verbose: cmd += ' --verbose ' + str(options.verbose) print "\ncmd is with .mrcs outputformat is", cmd print "becauase outfile is",outfile p = subprocess.Popen( cmd , shell=True,stdout=subprocess.PIPE, stderr=subprocess.PIPE) text = p.communicate() p.stdout.close() finaloutput = outfile.replace('.mrcs', '.' + originalextension) os.rename( outfile, finaloutput ) finalfiles.append( finaloutput ) else: finalfiles = list( imgspreproc ) for finalf in finalfiles: if not options.tltfile: break if options.normalizeimod: try: cmd = 'newstack ' + finalf + ' ' + finalf + ' --float 2' print "normalizeimod cmd is", cmd p = subprocess.Popen( cmd , shell=True,stdout=subprocess.PIPE, stderr=subprocess.PIPE) text = p.communicate() p.wait() except: print "\nERROR: --normalizeimod skipped. Doesn't seem like IMOD is installed on this machine" if not options.dontcleanup and options.maskbyangle: purge( options.path, '_preproc.mrc') purge( options.path, '_UNSTACKED') purge( options.path, '~') if options.tltfile: if options.prenadminite or options.prenadmaxite or options.prenadsigma: if options.prenadminite and options.prenadmaxite and options.prenadsigma: cmd = 'preNAD -input ' + finalf + ' -output ' + finalf.replace('.'+originalextension, '_prenad.' + originalextension) + ' -minite ' + str(options.prenadminite) + ' -maxite ' + str(options.prenadmaxite) + ' -sigma ' + str(options.prenadsigma) + ' -angles ' + options.tltfile if options.verbose: print "\n(e2tomopreproc)(main) prenad cmd to run is", cmd try: p = subprocess.Popen( cmd , shell=True,stdout=subprocess.PIPE, stderr=subprocess.PIPE) text = p.communicate() p.wait() except: print "\nERROR: check that a version of IMOD containing the preNAD program is correctly installed on this machine" else: if options.prenadminite: if not options.prenadmaxite: print "\nERROR: --prenadmaxite required with --prenadminite" if not options.prenadsigma: print "\nERROR: --prenadsigma required with --prenadminite" if options.prenadmaxite: if not options.prenadminite: print "\nERROR: --prenadminite required with --prenadmaxite" if not options.prenadsigma: print "\nERROR: --prenadsigma required with --prenadmaxite" if options.prenadsigma: if not options.prenadminite: print "\nERROR: --prenadminite required with --prenadsigma" if not options.prenadmaxite: print "\nERROR: --prenadmaxite required with --prenadsigma" E2end(logger) return()
class EMParallelProject3D: def __init__(self, options, fsp, sym, start, modeln=0, logger=None): """ @param options the options produced by (options, args) = parser.parse_args() @param args the options produced by (options, args) = parser.parse_args() @param logger and EMAN2 logger, i.e. logger=E2init(sys.argv) assumes you have already called the check function. """ self.options = options self.args = fsp self.sym = sym self.logger = logger self.start = start self.modeln = modeln from EMAN2PAR import EMTaskCustomer self.etc = EMTaskCustomer(options.parallel) print "Precache ", fsp self.etc.precache([fsp]) self.num_cpus = self.etc.cpu_est() print self.num_cpus, " total CPUs available" if self.num_cpus > 64: # upper limit self.num_cpus = 64 self.__task_options = None def __init_memory(self, options): """ """ sym_object = parsesym(self.sym) [og_name, og_args] = parsemodopt(options.orientgen) self.eulers = sym_object.gen_orientations(og_name, og_args) def __get_task_options(self, options): if self.__task_options == None: d = {} d["projector"] = parsemodopt(options.projector) d["prethreshold"] = options.prethreshold self.__task_options = d return self.__task_options def execute(self): # from EMAN2PAR import EMTaskCustomer if len(self.options.parallel) > 1: self.__init_memory(self.options) num_tasks = self.num_cpus # In the worst case we can only spawn as many tasks as there are eulers if self.num_cpus > len(self.eulers): num_tasks = len(self.eulers) eulers_per_task = len(self.eulers) / num_tasks resid_eulers = len(self.eulers) - eulers_per_task * num_tasks # we can distribute the residual evenly first = 0 task_customers = [] tids = [] # self.etc=EMTaskCustomer(self.options.parallel) for i in xrange(0, num_tasks): last = first + eulers_per_task if resid_eulers > 0: last += 1 resid_eulers -= 1 tmp_eulers = self.eulers[first:last] indices = range(first, last) data = {} data["input"] = ("cache", self.args, 0) data["eulers"] = tmp_eulers data["indices"] = indices task = EMProject3DTaskDC(data=data, options=self.__get_task_options(self.options)) # print "Est %d CPUs"%etc.cpu_est() tid = self.etc.send_task(task) # print "Task submitted tid=",tid tids.append(tid) first = last print "Task ids are", tids while 1: print len(tids), "projection tasks left in main loop" st_vals = self.etc.check_task(tids) for i in xrange(len(tids) - 1, -1, -1): st = st_vals[i] if st == 100: tid = tids[i] rslts = self.etc.get_results(tid) if not self.__write_output_data(rslts[1]): print "There was a problem with the task of id", tid if self.logger != None: E2progress(self.logger, 1.0 - len(tids) / float(num_tasks)) if self.options.verbose > 0: print "%d/%d\r" % (num_tasks - len(tids), num_tasks) sys.stdout.flush() print "Task", tids.pop(i), "completed" print "These tasks are remaining:", tids if len(tids) == 0: break time.sleep(5) return len(self.eulers) else: raise NotImplementedError( "The parallelism option you specified (%s) is not suppored" % self.options.parallel ) def __write_output_data(self, rslts): for idx, image in rslts.items(): if not isinstance(image, EMData): continue # this is here because we get the dimensions of the database as a key (e.g. '40x40x1'). image["model_id"] = self.modeln if self.options.append: image.write_image(self.options.outfile, -1) else: image.write_image(self.options.outfile, idx + self.start) return True
def main(): progname = os.path.basename(sys.argv[0]) usage = """prog <output> [options] This program produces iterative class-averages, one of the secrets to EMAN's rapid convergence. Normal usage is to provide a stack of particle images and a classification matrix file defining class membership. Members of each class are then iteratively aligned to each other and averaged together with (optional) CTF correction. It is also possible to use this program on all of the images in a single stack. """ parser = EMArgumentParser(usage=usage,version=EMANVERSION) parser.add_argument("--input", type=str, help="The name of the input particle stack", default=None) parser.add_argument("--output", type=str, help="The name of the output class-average stack", default=None) parser.add_argument("--oneclass", type=int, help="Create only a single class-average. Specify the number.",default=None) parser.add_argument("--classmx", type=str, help="The name of the classification matrix specifying how particles in 'input' should be grouped. If omitted, all particles will be averaged.", default=None) parser.add_argument("--ref", type=str, help="Reference image(s). Used as an initial alignment reference and for final orientation adjustment if present. Also used to assign euler angles to the generated classes. This is typically the projections that were used for classification.", default=None) parser.add_argument("--storebad", action="store_true", help="Even if a class-average fails, write to the output. Forces 1->1 numbering in output",default=False) parser.add_argument("--decayedge", action="store_true", help="Applies an edge decay to zero on the output class-averages. A very good idea if you plan on 3-D reconstruction.",default=False) parser.add_argument("--resultmx",type=str,help="Specify an output image to store the result matrix. This contains 5 images where row is particle number. Rows in the first image contain the class numbers and in the second image consist of 1s or 0s indicating whether or not the particle was included in the class. The corresponding rows in the third, fourth and fifth images are the refined x, y and angle (respectively) used in the final alignment, these are updated and accurate, even if the particle was excluded from the class.", default=None) parser.add_argument("--iter", type=int, help="The number of iterations to perform. Default is 1.", default=1) parser.add_argument("--prefilt",action="store_true",help="Filter each reference (c) to match the power spectrum of each particle (r) before alignment and comparison",default=False) parser.add_argument("--align",type=str,help="This is the aligner used to align particles to the previous class average. Default is None.", default=None) parser.add_argument("--aligncmp",type=str,help="The comparitor used for the --align aligner. Default is ccc.",default="ccc") parser.add_argument("--ralign",type=str,help="This is the second stage aligner used to refine the first alignment. This is usually the \'refine\' aligner.", default=None) parser.add_argument("--raligncmp",type=str,help="The comparitor used by the second stage aligner.",default="ccc") parser.add_argument("--averager",type=str,help="The type of averager used to produce the class average.",default="mean") parser.add_argument("--setsfref",action="store_true",help="This will impose the 1-D structure factor of the reference on the class-average (recommended when a reference is available)",default=False) parser.add_argument("--cmp",type=str,help="The comparitor used to generate quality scores for the purpose of particle exclusion in classes, strongly linked to the keep argument.", default="ccc") parser.add_argument("--keep",type=float,help="The fraction of particles to keep in each class.",default=1.0) parser.add_argument("--keepsig", action="store_true", help="Causes the keep argument to be interpreted in standard deviations.",default=False) parser.add_argument("--automask",action="store_true",help="Applies a 2-D automask before centering. Can help with negative stain data, and other cases where centering is poor.") parser.add_argument("--center",type=str,default="xform.center",help="If the default centering algorithm (xform.center) doesn't work well, you can specify one of the others here (e2help.py processor center)") parser.add_argument("--bootstrap",action="store_true",help="Ignored. Present for historical reasons only.") parser.add_argument("--normproc",type=str,help="Normalization processor applied to particles before alignment. Default is normalize.edgemean. If you want to turn this option off specify \'None\'", default="normalize.edgemean") parser.add_argument("--usefilt", dest="usefilt", default=None, help="Specify a particle data file that has been low pass or Wiener filtered. Has a one to one correspondence with your particle data. If specified will be used to align particles to the running class average, however the original particle will be used to generate the actual final class average") parser.add_argument("--idxcache", default=False, action="store_true", help="Ignored. Present for historical reasons.") parser.add_argument("--dbpath", help="Ignored. Present for historical reasons.", default=".") parser.add_argument("--resample",action="store_true",help="If set, will perform bootstrap resampling on the particle data for use in making variance maps.",default=False) parser.add_argument("--odd", default=False, help="Used by EMAN2 when running eotests. Includes only odd numbered particles in class averages.", action="store_true") parser.add_argument("--even", default=False, help="Used by EMAN2 when running eotests. Includes only even numbered particles in class averages.", action="store_true") parser.add_argument("--parallel", default=None, help="parallelism argument") parser.add_argument("--force", "-f",dest="force",default=False, action="store_true",help="Force overwrite the output file if it exists.") parser.add_argument("--saveali",action="store_true",help="Writes aligned particle images to aligned.hdf. Normally resultmx produces more useful informtation. This can be used for debugging.",default=False) parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n",type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness") parser.add_argument("--debug","-d",action="store_true",help="Print debugging infromation while the program is running. Default is off.",default=False) parser.add_argument("--nofilecheck",action="store_true",help="Turns file checking off in the check functionality - used by e2refine.py.",default=False) parser.add_argument("--check","-c",action="store_true",help="Performs a command line argument check only.",default=False) parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1) (options, args) = parser.parse_args() if (options.check): options.verbose = 9 # turn verbose on if the user is only checking... error = check(options,True) if options.align : options.align=parsemodopt(options.align) if options.ralign : options.ralign=parsemodopt(options.ralign) if options.aligncmp : options.aligncmp=parsemodopt(options.aligncmp) if options.raligncmp : options.raligncmp=parsemodopt(options.raligncmp) if options.averager : options.averager=parsemodopt(options.averager) if options.cmp : options.cmp=parsemodopt(options.cmp) if options.normproc : options.normproc=parsemodopt(options.normproc) if options.resultmx!=None : options.storebad=True if (options.verbose>0): if (error): print "e2classaverage.py command line arguments test.... FAILED" else: print "e2classaverage.py command line arguments test.... PASSED" # returning a different error code is currently important to e2refine.py - returning 0 tells e2refine.py that it has enough # information to execute this script if error : exit(1) if options.check: exit(0) logger=E2init(sys.argv,options.ppid) print "Class averaging beginning" try: classmx=EMData.read_images(options.classmx) # we keep the entire classification matrix in memory, since we need to update it in most cases ncls=int(classmx[0]["maximum"])+1 except: ncls=1 if options.resultmx!=None : print "resultmx can only be specified in conjunction with a valid classmx input." sys.exit(1) nptcl=EMUtil.get_image_count(options.input) try: apix=EMData(options.input,0,True)["apix_x"] except: apix=1.0 print "WARNING: could not get apix from first image. Setting to 1.0. May impact results !" # Initialize parallelism if options.parallel : from EMAN2PAR import EMTaskCustomer etc=EMTaskCustomer(options.parallel) pclist=[options.input] if options.ref: pclist.append(options.ref) if options.usefilt: pclist.append(options.usefilt) etc.precache(pclist) # prepare tasks tasks=[] if ncls>1: if options.oneclass==None : clslst=range(ncls) else : clslst=[options.oneclass] for cl in clslst: ptcls=classmx_ptcls(classmx[0],cl) if options.resample : ptcls=[random.choice(ptcls) for i in ptcls] # this implements bootstrap resampling of the class-average if options.odd : ptcls=[i for i in ptcls if i%2==1] if options.even: ptcls=[i for i in ptcls if i%2==0] tasks.append(ClassAvTask(options.input,ptcls,options.usefilt,options.ref,options.iter,options.normproc,options.prefilt, options.align,options.aligncmp,options.ralign,options.raligncmp,options.averager,options.cmp,options.keep,options.keepsig, options.automask,options.saveali,options.setsfref,options.verbose,cl,options.center)) else: ptcls=range(nptcl) if options.resample : ptcls=[random.choice(ptcls) for i in ptcls] if options.odd : ptcls=[i for i in ptcls if i%2==1] if options.even: ptcls=[i for i in ptcls if i%2==0] tasks.append(ClassAvTask(options.input,range(nptcl),options.usefilt,options.ref,options.iter,options.normproc,options.prefilt, options.align,options.aligncmp,options.ralign,options.raligncmp,options.averager,options.cmp,options.keep,options.keepsig, options.automask,options.saveali,options.setsfref,options.verbose,0,options.center)) # execute task list if options.parallel: # run in parallel taskids=etc.send_tasks(tasks) alltaskids=taskids[:] while len(taskids)>0 : curstat=etc.check_task(taskids) for i,j in enumerate(curstat): if j==100 : rslt=etc.get_results(taskids[i]) if rslt[1]["average"]!=None: rslt[1]["average"]["class_ptcl_src"]=options.input if options.decayedge: nx=rslt[1]["average"]["nx"] rslt[1]["average"].process_inplace("normalize.circlemean",{"radius":nx/2-nx/15}) rslt[1]["average"].process_inplace("mask.gaussian",{"inner_radius":nx/2-nx/15,"outer_radius":nx/20}) #rslt[1]["average"].process_inplace("mask.decayedge2d",{"width":nx/15}) if options.ref!=None : rslt[1]["average"]["projection_image"]=options.ref if options.storebad : rslt[1]["average"].write_image(options.output,rslt[1]["n"]) else: rslt[1]["average"].write_image(options.output,-1) # Update the resultsmx if requested if options.resultmx!=None: allinfo=rslt[1]["info"] # the info result array list of (qual,xform,used) tuples pnums=rslt[0].data["images"][2] # list of image numbers corresponding to information for n,info in enumerate(allinfo): y=pnums[n] # actual particle number # find the matching class in the existing classification matrix for x in range(classmx[0]["nx"]): if classmx[0][x,y]==rslt[1]["n"] : # if the class number in the classmx matches the current class-average number break else : print "Resultmx error: no match found ! (%d %d %d)"%(x,y,rslt[1]["n"]) continue xform=info[1].get_params("2d") classmx[1][x,y]=info[2] # used classmx[2][x,y]=xform["tx"] # dx classmx[3][x,y]=xform["ty"] # dy classmx[4][x,y]=xform["alpha"] # da classmx[5][x,y]=xform["mirror"] # flip try: classmx[6][x,y]=xform["scale"] except: pass # failed average elif options.storebad : blk=EMData(options.ref,0) apix=blk["apix_x"] blk=EMData(blk["nx"],blk["ny"],1) blk["apix_x"]=apix blk.to_zero() blk.set_attr("ptcl_repr", 0) blk.set_attr("apix_x",apix) blk.write_image(options.output,rslt[1]["n"]) taskids=[j for i,j in enumerate(taskids) if curstat[i]!=100] if options.verbose and 100 in curstat : print "%d/%d tasks remain"%(len(taskids),len(alltaskids)) if 100 in curstat : E2progress(logger,1.0-(float(len(taskids))/len(alltaskids))) time.sleep(3) if options.verbose : print "Completed all tasks" # single thread else: for t in tasks: rslt=t.execute() if rslt==None : sys.exit(1) if rslt["average"]!=None : rslt["average"]["class_ptcl_src"]=options.input if options.decayedge: nx=rslt["average"]["nx"] rslt["average"].process_inplace("normalize.circlemean",{"radius":nx/2-nx/15}) rslt["average"].process_inplace("mask.gaussian",{"inner_radius":nx/2-nx/15,"outer_radius":nx/20}) #rslt["average"].process_inplace("mask.decayedge2d",{"width":nx/15}) if options.ref!=None : rslt["average"]["projection_image"]=options.ref if options.storebad : rslt["average"].write_image(options.output,t.options["n"]) else: rslt["average"].write_image(options.output,-1) # Update the resultsmx if requested if options.resultmx!=None: allinfo=rslt["info"] # the info result array list of (qual,xform,used) tuples pnums=t.data["images"][2] # list of image numbers corresponding to information for n,info in enumerate(allinfo): y=pnums[n] # actual particle number # find the matching class in the existing classification matrix for x in range(classmx[0]["nx"]): if classmx[0][x,y]==rslt["n"] : # if the class number in the classmx matches the current class-average number break else : print "Resultmx error: no match found ! (%d %d %d)"%(x,y,rslt[1]["n"]) continue xform=info[1].get_params("2d") classmx[1][x,y]=info[2] # used classmx[2][x,y]=xform["tx"] # dx classmx[3][x,y]=xform["ty"] # dy classmx[4][x,y]=xform["alpha"] # da classmx[5][x,y]=xform["mirror"] # flip try: classmx[6][x,y]=xform["scale"] except: pass # Failed average elif options.storebad : blk=EMData(options.ref,0) apix=blk["apix_x"] blk=EMData(blk["nx"],blk["ny"],1) blk["apix_x"]=apix blk.to_zero() blk.set_attr("ptcl_repr", 0) blk.set_attr("apix_x",apix) blk.write_image(options.output,t.options["n"]) if options.resultmx!=None: if options.verbose : print "Writing results matrix" for i,j in enumerate(classmx) : j.write_image(options.resultmx,i) print "Class averaging complete" E2end(logger)
def main(): progname = os.path.basename(sys.argv[0]) usage = """prog [options] This program aligns a paricle to its symmetry axis. There are two algorithmic modes. A coarse search followed by simplex minimization (not yet implimented) OR monte carlo course search followed by simplex minimization. The Goal is to align the paricle to its symmetry axis so symmetry can be applied for avergaing and for alignment speed up (it is only necessary to search over one asymmetric unit! """ parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_header( name="symsearch3dheader", help="""Options below this label are specific to e2symsearch3d""", title="### e2symsearch3d options ###", row=3, col=0, rowspan=1, colspan=2, ) parser.add_argument( "--input", dest="input", default="", type=str, help="""The name of input volume or hdf stack of volumes""", guitype="filebox", browser="EMBrowserWidget(withmodal=True,multiselect=False)", row=0, col=0, rowspan=1, colspan=2, ) # parser.add_argument("--output", dest="output", default="""e2symsearch3d_OUTPUT.hdf""", type=str, help="The name of the output volume", guitype='strbox', filecheck=False, row=1, col=0, rowspan=1, colspan=2) parser.add_argument( "--ref", type=str, default="", help="""Default=None. If provided and --average is also provided and --keep < 1.0 or --keepsig is specified, 'good particles' will be determined by correlation to --ref.""", ) parser.add_argument( "--mirror", type=str, default="", help="""Axis across of which to generate a mirrored copy of --ref. All particles will be compared to it in addition to the unmirrored image in --ref if --keepsig is provided or if --keep < 1.0.""", ) parser.add_argument( "--path", type=str, default="", help="""Name of path for output file""", guitype="strbox", row=2, col=0, rowspan=1, colspan=2, ) parser.add_argument( "--plots", action="store_true", default=False, help="""Default=False. Turn this option on to generate a plot of the ccc scores if --average is supplied. Running on a cluster or via ssh remotely might not support plotting.""", ) parser.add_argument( "--sym", dest="sym", default="c1", help="""Specify symmetry -choices are: c<n>, d<n>, h<n>, tet, oct, icos. For asymmetric reconstruction ommit this option or specify c1.""", guitype="symbox", row=4, col=0, rowspan=1, colspan=2, ) parser.add_argument( "--shrink", dest="shrink", type=int, default=0, help="""Optionally shrink the input particles by an integer amount prior to computing similarity scores. For speed purposes. Default=0, no shrinking""", guitype="shrinkbox", row=5, col=0, rowspan=1, colspan=1, ) parser.add_argument( "--mask", type=str, help="""Mask processor applied to particles before alignment. Default is mask.sharp:outer_radius=-2. IF using --clipali, make sure to express outer mask radii as negative pixels from the edge.""", returnNone=True, default="mask.sharp:outer_radius=-2", guitype="comboparambox", choicelist="re_filter_list(dump_processors_list(),'mask')", row=11, col=0, rowspan=1, colspan=3, ) parser.add_argument( "--maskfile", type=str, default="", help="""Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None.""", ) parser.add_argument( "--normproc", type=str, default="", help="""Normalization processor applied to particles before alignment. Default is to use normalize. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'""", ) parser.add_argument( "--nopreprocprefft", action="store_true", default=False, help="""Turns off all preprocessing that happens only once before alignment (--normproc, --mask, --maskfile, --clipali, --threshold; i.e., all preprocessing excepting filters --highpass, --lowpass, --preprocess, and --shrink.""", ) parser.add_argument( "--threshold", default="", type=str, help="""A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""", guitype="comboparambox", choicelist="re_filter_list(dump_processors_list(),'filter')", row=10, col=0, rowspan=1, colspan=3, ) parser.add_argument( "--preprocess", default="", type=str, help="""Any processor (as in e2proc3d.py) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype="comboparambox", choicelist="re_filter_list(dump_processors_list(),'filter')", row=10, col=0, rowspan=1, colspan=3, ) parser.add_argument( "--lowpass", type=str, default="", help="""A lowpass filtering processor (from e2proc3d.py; see e2help.py processors) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype="comboparambox", choicelist="re_filter_list(dump_processors_list(),'filter')", row=17, col=0, rowspan=1, colspan=3, ) parser.add_argument( "--highpass", type=str, default="", help="""A highpass filtering processor (from e2proc3d.py, see e2help.py processors) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype="comboparambox", choicelist="re_filter_list(dump_processors_list(),'filter')", row=18, col=0, rowspan=1, colspan=3, ) parser.add_argument( "--clipali", type=int, default=0, help="""Boxsize to clip particles as part of preprocessing to speed up alignment. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary; still, there are some benefits from 'oversampling' the data during averaging; so you might still want an average of size 2x, but perhaps particles in a box of 1.5x are sufficiently good for alignment. In this case, you would supply --clipali=75""", ) parser.add_argument( "--savepreproc", action="store_true", default=False, help="""Default=False. Will save stacks of preprocessed particles (one for coarse alignment and one for fine alignment if preprocessing options are different).""", ) parser.add_argument( "--average", action="store_true", default=False, help="""Default=False. If supplied and a stack is provided through --input, the average of the aligned and/or symmetrized stack will also be saved.""", ) parser.add_argument( "--averager", type=str, default="mean.tomo", help="""Default=mean.tomo. The type of averager used to produce the class average. Default=mean.tomo.""", ) parser.add_argument( "--keep", type=float, default=1.0, help="""Fraction of particles to include if --average is on, after correlating the particles with the average.""", ) parser.add_argument( "--keepsig", action="store_true", default=False, help="""Default=False. Causes theoptions.keep argument to be interpreted in standard deviations.""", guitype="boolbox", row=6, col=1, rowspan=1, colspan=1, mode="alignment,breaksym", ) parser.add_argument( "--avgiter", type=int, default=1, help="""Default=1. If --keep is different from 1.0 and --average is on, the initial average will include all the particles, but then the percent specified byoptions.keep will be kept (the rest thrown away) and a new average will be computed. If --avgiter > 1, this new average will be compared again against all the particles. The procedure will be repeated for however many iterations --avgiter is given, or the process will stop automatically if in two consecutive rounds exactly the same particles are kept""", ) parser.add_argument( "--subset", type=int, default=0, help="""Number of particles in a subset of particles from the --input stack of particles to run the alignments on.""", ) parser.add_argument( "--steps", dest="steps", type=int, default=10, help="""Number of steps (for the MC). Default=10.""", guitype="intbox", row=5, col=1, rowspan=1, colspan=1, ) parser.add_argument( "--symmetrize", default=False, action="store_true", help="""Symmetrize volume after alignment.""", guitype="boolbox", row=6, col=0, rowspan=1, colspan=1, ) parser.add_argument( "--cmp", type=str, help="""The name of a 'cmp' to be used in comparing the symmtrized object to unsymmetrized""", default="ccc", guitype="comboparambox", choicelist="re_filter_list(dump_cmps_list(),'tomo', True)", row=7, col=0, rowspan=1, colspan=2, ) parser.add_argument( "--parallel", "-P", type=str, help="""Run in parallel, specify type:<option>=<value>:<option>:<value>""", default=None, guitype="strbox", row=8, col=0, rowspan=1, colspan=2, ) parser.add_argument( "--ppid", type=int, help="""Set the PID of the parent process, used for cross platform PPID.""", default=-1 ) parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="""verbose level [0-9], higner number means higher level ofoptions.verboseness.""", ) parser.add_argument( "--nopath", action="store_true", default=False, help="""If supplied, this option will save results in the directory where the command is run. A directory to store the results will not be made.""", ) parser.add_argument( "--nolog", action="store_true", default=False, help="""If supplied, this option will prevent logging the command run in .eman2log.txt.""", ) parser.add_argument( "--saveali", action="store_true", default=False, help="""Save the stack of aligned/symmetrized particles.""" ) parser.add_argument( "--savesteps", action="store_true", default=False, help="""If --avgiter > 1, save all intermediate averages and intermediate aligned kept stacks.""", ) parser.add_argument( "--notmatchimgs", action="store_true", default=False, help="""Default=True. This option prevents applying filter.match.to to one image so that it matches the other's spectral profile during preprocessing for alignment purposes.""", ) parser.add_argument( "--preavgproc1", type=str, default="", help="""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)""", ) parser.add_argument( "--preavgproc2", type=str, default="", help="""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)""", ) parser.add_argument( "--weighbytiltaxis", type=str, default="", help="""Default=None. A,B, where A is an integer number and B a decimal. A represents the location of the tilt axis in the tomogram in pixels (eg.g, for a 4096x4096xZ tomogram, this value should be 2048), and B is the weight of the particles furthest from the tomogram. For example, --weighbytiltaxis=2048,0.5 means that praticles at the tilt axis (with an x coordinate of 2048) will have a weight of 1.0 during averaging, while the distance in the x coordinates of particles not-on the tilt axis will be used to weigh their contribution to the average, with particles at the edge(0+radius or 4096-radius) weighing 0.5, as specified by the value provided for B.""", ) parser.add_argument( "--weighbyscore", action="store_true", default=False, help="""Default=False. This option will weigh the contribution of each subtomogram to the average by score/bestscore.""", ) parser.add_argument( "--align", type=str, default="symalignquat", help="""Default=symalignquat. WARNING: The aligner cannot be changed for this program currently. Option ignored.""", ) parser.add_argument( "--tweak", action="store_true", default=False, help="""WARNING: Not used for anything yet. This will perform a final alignment with no downsampling [without using --shrink or --shrinkfine] if --shrinkfine > 1.""", ) (options, args) = parser.parse_args() if not options.input: parser.print_help() sys.exit(0) # If no failures up until now, initialize logger log = 0 if not options.nolog: logid = E2init(sys.argv, options.ppid) log = 1 # inimodeldir = os.path.join(".",options.path) # if not os.access(inimodeldir, os.R_OK): # os.mkdir(options.path) # Make directory to save results from e2spt_classaverage import ( sptmakepath, preprocessingprefft, Preprocprefft3DTask, get_results_preproc, preprocfilter, sptOptionsParser, ) options = sptmakepath(options, "symsearch") if options.nopath: options.path = "." rootpath = os.getcwd() if rootpath not in options.path: options.path = rootpath + "/" + options.path if options.parallel: from EMAN2PAR import EMTaskCustomer options = sptOptionsParser(options) avgr = Averagers.get(options.averager[0], options.averager[1]) resultsdict = {} scores = [] outputstack = options.path + "/all_ptcls_ali.hdf" # Determine number of particles in the stack n = EMUtil.get_image_count(options.input) if options.subset and options.subset < n: n = options.subset options.raw = options.input if not options.nopreprocprefft: if options.mask or options.normproc or options.threshold or options.clipali: preprocprefftstack = options.path + "/" + os.path.basename(options.input).replace(".hdf", "_preproc.hdf") # save "dummy" images for preproc images for i in range(n): dimg = EMData(8, 8, 8) dimg.to_one() dimg.write_image(preprocprefftstack, i) originalsavepreproc = options.savepreproc options.savepreproc = True print "\n(e2spt_hac.py) (allvsall) Initializing parallelism for preprocessing" if options.parallel: # Initialize parallelism if being used # from EMAN2PAR import EMTaskCustomer etc = EMTaskCustomer(options.parallel) pclist = [options.input] etc.precache(pclist) tasks = [] results = [] # preprocprefftstack = options.path + '/' + options.input.replace('.hdf','_preproc.hdf') for i in range(n): img = EMData(options.input, i) if options.parallel: task = Preprocprefft3DTask(["cache", options.input, i], options, i, preprocprefftstack) tasks.append(task) else: pimg = preprocessingprefft(img, options) pimg.write_image(preprocprefftstack, i) print "\nthere are these many tasks to send", len(tasks) if options.parallel and tasks: tids = etc.send_tasks(tasks) print "therefore these many tids", len(tids) if options.verbose: print "%d preprocessing tasks queued" % (len(tids)) results = get_results_preproc(etc, tids, options.verbose) print "results are", results options.input = preprocprefftstack options.savepreproc = originalsavepreproc for i in range(n): print "\nI'll look for symmetry in particle number", i # Load particle and make a copy to modify if preprocessing options are specified volume = EMData(options.input, i) preprocvol = volume.copy() # Preprocess volume if any preprocessing options are specified preprocprefftstack = options.path + "/" + os.path.basename(options.input).replace(".hdf", "_preproc.hdf") if ( (options.shrink and options.shrink > 1) or options.lowpass or options.highpass or options.normproc or options.preprocess or options.threshold or options.clipali ): print "\nHowever, I will first preprocess particle number", i print "\nWill call preprocessing on ptcl", i preprocvol = preprocfilter(preprocvol, options, i) if options.savepreproc: preprocvol.write_image(preprocprefftstack, i) # preprocessing(s2image,options, ptclindx, savetagp ,'no',round) print "\nDone preprocessing on ptcl", i if options.parallel: etc = EMTaskCustomer(options.parallel) else: etc = EMTaskCustomer("thread:1") symalgorithm = SymALignStrategy(preprocvol, options.sym, options.steps, options.cmp, etc) ret = symalgorithm.execute() symxform = ret[0] score = ret[1] scores.append(score) resultsdict.update({score: [symxform, i]}) print "\nWriting output for best alignment found for particle number", i if options.shrink and options.shrink > 1: trans = symxform.get_trans() symxform.set_trans(trans[0] * options.shrink, trans[1] * options.shrink, trans[2] * options.shrink) print "\nWrittng to output ptcl", i # Rotate volume to the best orientation found, set the orientation in the header, apply symmetry if specified and write out the aligned (and symmetrized) particle to the output stack output = volume.process("xform", {"transform": symxform}) output.set_attr("symxform", symxform) print "\nApplying this transform to particle", symxform if options.symmetrize: output = output.process("xform.applysym", {"sym": options.sym}) output["spt_score"] = score output.write_image(outputstack, -1) # Averaging here only makes sense if all particles are going to be kept. Otherwise, different code is needed (below) if options.average: avgr.add_image(output) # Finalize average of all particles if non were set to be excluded. Otherwise, determine the discrimination threshold and then average the particles that pass it. if options.average: final_avg = avgr.finish() final_avg["origin_x"] = 0 final_avg["origin_y"] = 0 # The origin needs to be reset to ZERO to avoid display issues in Chimera final_avg["origin_z"] = 0 final_avg["xform.align3d"] = Transform() if options.keep == 1.0 and not options.keepsig: final_avg.write_image(options.path + "/final_avg.hdf", 0) if options.avgiter > 1: print """WARNING: --avgiter > 1 must be accompanied by --keepsig, or by --keep < 1.0""" elif options.keep < 1.0 or options.keepsig: if options.ref: ref = EMData(options.ref, 0) refComp(options, outputstack, ref, resultsdict, "") if options.mirror: ref.process_inplace("xform.mirror", {"axis": options.mirror}) refComp(options, outputstack, ref, results, "_vs_mirror") else: ref2compare = final_avg refComp(options, outputstack, final_avg, resultsdict, "") del final_avg if log: E2end(logid) return