def main():
progname = os.path.basename(sys.argv[0])
usage = """prog <output> [options]
This program is used to preprocess subtomograms before aligning them. The same can be accomplished with
e2proc3d, except that this program is parallelized and thus should be substantially faster for large subtomograms.
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--input", type=str, default='',help="""Default=None. The name of the input volume stack. MUST be HDF since volume stack support is required.""")
parser.add_argument("--output", type=str, default='',help="""Default=None. Specific name of HDF file to write processed particles to.""")
parser.add_argument("--parallel",type=str, default='', help="""default=None. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel""")
parser.add_argument("--ppid", type=int, help="""Default=-1. Set the PID of the parent process, used for cross platform PPID""",default=-1)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="""Default=0. Verbose level [0-9], higner number means higher level of verboseness""")
parser.add_argument("--subset",type=int,default=0,help="""Default=0 (not used). Refine only this substet of particles from the stack provided through --input""")
parser.add_argument("--apix",type=float,default=0.0,help="""Default=0.0 (not used). Use this apix value where relevant instead of whatever is in the header of the reference and the particles. Will overwrite particle header as well.""")
parser.add_argument("--shrink", type=int,default=0,help="""Default=0 (no shrinking). Optionally shrink the input volumes by an integer amount for coarse alignment.""")
parser.add_argument("--threshold",type=str,default='',help="""Default=None. A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""")
parser.add_argument("--mask",type=str,default='', help="""Default=None. Masking processor applied to particles before alignment. IF using --clip, make sure to express outer mask radii as negative pixels from the edge.""")
parser.add_argument("--maskfile",type=str,default='',help="""Default=None. Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None.""")
parser.add_argument("--normproc",type=str, default='',help="""Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to particles before alignment. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'""")
parser.add_argument("--preprocess",type=str,default='',help="""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""")
parser.add_argument("--lowpass",type=str,default='',help="""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""")
parser.add_argument("--highpass",type=str,default='',help="""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""")
parser.add_argument("--clip",type=int,default=0,help="""Default=0 (which means it's not used). Boxsize to clip particles. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary.""")
parser.add_argument("--nopath",action='store_true',default=False,help="""If supplied, this option will save results in the directory where the command is run. A directory to store the results will not be made.""")
parser.add_argument("--path",type=str,default='sptpreproc',help="""Default=spt. Directory to store results in. The default is a numbered series of directories containing the prefix 'sptpreproc'; for example, sptpreproc_02 will be the directory by default if 'sptpreproc_01' already exists.""")
(options, args) = parser.parse_args()
logger = E2init(sys.argv, options.ppid)
print "\n(e2spt_preproc)(main) started log"
from e2spt_classaverage import sptmakepath
if options.path and not options.nopath:
options = sptmakepath(options,'sptpreproc')
if options.parallel=='None' or options.parallel=='none':
options.parallel=None
if not options.input:
try:
options.input = sys.argv[1]
except:
print "\n(e2spt_preproc)(main) ERROR: invalid input file"
if options.mask or options.maskfile or options.threshold or options.clip or options.threshold or options.normproc or options.preprocess or options.lowpass or options.highpass or int(options.shrink) > 1:
preprocstack = str(os.path.basename(options.input).replace('.hdf','_preproc.hdf'))
if options.path and not options.nopath:
preprocstack = options.path + '/' + preprocstack
if options.output:
if '.hdf' in options.output[-4:]:
preprocstack = options.output
else:
print "\n(e2spt_preproc)(main) ERROR: '.hdf' must be the last four characters of the output filename."
print "\n(e2spt_preproc)(main) output stack will be %s" %( preprocstack)
n = 0
try:
n = EMUtil.get_image_count( options.input )
except:
print "\n(e2spt_preproc)(main) ERROR: --input stack seems to be invalid"
sys.exit()
print "\n(e2spt_preproc)(main) number of particles is %d" %( n)
c = os.getcwd()
findir = os.listdir( c )
if preprocstack not in findir:
dimg = EMData(8,8,8)
dimg.to_one()
for i in range(n):
dimg.write_image( preprocstack, i )
else:
print "\n(e2spt_preproc)(main) WARNING: a file with the name of the output stack %s is already in the current directory and will be DELETED" %( preprocstack )
os.remove( preprocstack )
dimg = EMData(8,8,8)
dimg.to_one()
for i in range(n):
dimg.write_image( preprocstack, i )
finalbox = EMData(options.input,0,True)['nx']
if options.clip:
finalbox=options.clip
#dimglarge=EMData(finalbox,finalbox,finalbox)
#dimglarge.to_one()
#dimglarge.write_image(preprocstack,0)
#n=EMUtil.get_image_count(options.input)
#if options.subset:
# n=options.subset
#dimglarge.write_image(preprocstack,n-1)
if options.verbose:
print "\n(e2spt_preproc)(main) wrote dummy ptcls to %s" %( preprocstack)
print "\n(e2spt_preproc)(main) - INITIALIZING PARALLELISM!\n"
if options.parallel:
from EMAN2PAR import EMTaskCustomer
etc=EMTaskCustomer(options.parallel)
pclist=[options.input]
etc.precache(pclist)
print "\n(e2spt_preproc)(main) - precaching --input"
tasks=[]
results=[]
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser( options )
for j in range(n):
#print "processing particle", j
img = EMData( options.input, j )
if options.parallel:
#task = Preproc3DTask( ["cache",options.input,j], options, j, preprocstack )
task = Preproc3DTask( img, options, j, preprocstack )
tasks.append(task)
else:
img = EMData( options.input, j )
pimg = preprocfunc( img, options, j, preprocstack)
if options.parallel and tasks:
tids = etc.send_tasks(tasks)
if options.verbose:
print "\n(e2spt_preproc)(main) preprocessing %d tasks queued" % (len(tids))
results = get_results( etc, tids, options )
#print "\n(e2spt_preproc)(main) preprocessing results are", results
#print "\n(e2spt_preproc)(main) input changing to preprocstack"
#options.input = preprocstack
#cache needs to be reloaded with the new options.input
else:
print "\n(e2spt_preproc)(main) Nothing to do. No preprocessing parameters specified."
E2end(logger)
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog <output> [options]
Program to build an initial subtomogram average by averaging pairs from the largest subset
in --input that is a power of 2. For example, if you supply an input stack with 100 subtomograms,
this program will build an initial reference using 64, since 64 is the largest power of 2 contained in 100.
In the first iteration, particle 1 will be averaged with 2, 3 with 4, 5 with 6... etc.
32 new averages (each an average of 2 subtomograms) will be used for the second iteration.
Again, 1 will be averaged with 2, 3 with 4, etc... yielding 16 new averages.
The algorithm continues until the entire subset (64) has been merged into 1 average.
This program imports 'preprocfunc' from e2spt_preproc.py and 'alignment' from e2spt_classaverage.py
--mask=mask.sharp:outer_radius=<safe radius>
--preprocess=filter.lowpass.gauss:cutoff_freq=<1/resolution in A>
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_header(name="sptbtheader", help="""Options below this label are specific to
sptbinarytree""", title="### sptbinarytree options ###", row=6, col=0, rowspan=1, colspan=3,mode="align")
parser.add_header(name="caheader", help="""Options below this label are specific to sptclassaverage""", title="### sptclassaverage options ###", row=3, col=0, rowspan=1, colspan=3, mode='alignment,breaksym')
parser.add_argument("--path",type=str,default='spt',help="""Default=spt. Directory to store results in. The default is a numbered series of directories containing the prefix 'spt'; for example, spt_02 will be the directory by default if 'spt_01' already exists.""")
parser.add_argument("--input", type=str, default='',help="""Default=None. The name of the input volume stack. MUST be HDF since volume stack support is required.""", guitype='filebox', browser='EMSubTomosTable(withmodal=True,multiselect=False)', row=0, col=0, rowspan=1, colspan=3, mode='alignment,breaksym')
parser.add_argument("--npeakstorefine", type=int, help="""Default=1. The number of best coarse alignments to refine in search of the best final alignment. Default=1.""", default=4, guitype='intbox', row=9, col=0, rowspan=1, colspan=1, nosharedb=True, mode='alignment,breaksym[1]')
parser.add_argument("--parallel",default="thread:1",help="""default=thread:1. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel""", guitype='strbox', row=19, col=0, rowspan=1, colspan=3, mode='alignment,breaksym')
parser.add_argument("--ppid", type=int, help="""Default=-1. Set the PID of the parent process, used for cross platform PPID""",default=-1)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="""Default=0. Verbose level [0-9], higner number means higher level of verboseness""")
#parser.add_argument("--resume",type=str,default='',help="""(Not working currently). tomo_fxorms.json file that contains alignment information for the particles in the set. If the information is incomplete (i.e., there are less elements in the file than particles in the stack), on the first iteration the program will complete the file by working ONLY on particle indexes that are missing. For subsequent iterations, all the particles will be used.""")
parser.add_argument("--plots", action='store_true', default=False,help="""Default=False. Turn this option on to generatea plot of the ccc scores during each iteration. Running on a cluster or via ssh remotely might not support plotting.""")
parser.add_argument("--subset",type=int,default=0,help="""Default=0 (not used). Refine only this substet of particles from the stack provided through --input""")
parser.add_argument("--preavgproc1",type=str,default='',help="""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)""")
parser.add_argument("--preavgproc2",type=str,default='',help="""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)""")
parser.add_argument("--weighbytiltaxis",type=str,default='',help="""Default=None. A,B, where A is an integer number and B a decimal. A represents the location of the tilt axis in the tomogram in pixels (eg.g, for a 4096x4096xZ tomogram, this value should be 2048), and B is the weight of the particles furthest from the tomogram. For example, --weighbytiltaxis=2048,0.5 means that praticles at the tilt axis (with an x coordinate of 2048) will have a weight of 1.0 during averaging, while the distance in the x coordinates of particles not-on the tilt axis will be used to weigh their contribution to the average, with particles at the edge(0+radius or 4096-radius) weighing 0.5, as specified by the value provided for B.""")
parser.add_argument("--weighbyscore",action='store_true',default=False,help="""Default=False. This option will weigh the contribution of each subtomogram to the average by score/bestscore.""")
parser.add_argument("--align",type=str,default="rotate_translate_3d:search=8:delta=12:dphi=12",help="""This is the aligner used to align particles to the previous class average. Default is rotate_translate_3d:search=8:delta=12:dphi=12, specify 'None' (with capital N) to disable.""", returnNone=True,guitype='comboparambox', choicelist='re_filter_list(dump_aligners_list(),\'3d\')', row=12, col=0, rowspan=1, colspan=3, nosharedb=True, mode="alignment,breaksym['rotate_symmetry_3d']")
parser.add_argument("--aligncmp",type=str,default="ccc.tomo.thresh",help="""Default=ccc.tomo.thresh. The comparator used for the --align aligner. Do not specify unless you need to use anotherspecific aligner.""",guitype='comboparambox',choicelist='re_filter_list(dump_cmps_list(),\'tomo\')', row=13, col=0, rowspan=1, colspan=3,mode="alignment,breaksym")
#parser.add_argument("--output", type=str, default='avg.hdf', help="""Default=avg.hdf. The name of the output class-average stack. MUST be HDF since volume stack support is required.""", guitype='strbox', row=2, col=0, rowspan=1, colspan=3, mode='alignment,breaksym')
#parser.add_argument("--classmx", type=str, default='', help="""Default=None. The name of the classification matrix specifying how particles in 'input' should be grouped. If omitted, all particles will be averaged.""")
#parser.add_argument("--ref", type=str, default='', help="""Default=None. Reference image(s). Used as an initial alignment reference and for final orientation adjustment if present. This is typically the projections that were used for classification.""", guitype='filebox', browser='EMBrowserWidget(withmodal=True,multiselect=True)', filecheck=False, row=1, col=0, rowspan=1, colspan=3, mode='alignment')
#parser.add_argument("--refpreprocess",action="store_true",default=False,help="""Default=False. This will preprocess the reference identically to the particles. It is off by default, but it is internally turned on when no reference is supplied.""")
#parser.add_argument("--resultmx",type=str,default=None,help="""Default=Npone. Specify an output image to store the result matrix. This is in the same format as the classification matrix. http://blake.bcm.edu/emanwiki/EMAN2/ClassmxFiles""")
#parser.add_argument("--refinemultireftag", type=str, default='', help="""Default=''. DO NOT USE THIS PARAMETER. It is passed on from e2spt_refinemulti.py if needed.""")
'''
ADVANCED parameters
'''
parser.add_argument("--averager",type=str,default="mean.tomo",help="""Default=mean.tomo. The type of averager used to produce the class average. Default=mean.tomo.""")
'''
PRE-FFT processing parameters
'''
#parser.add_argument("--nopreprocprefft",action="store_true",default=False,help="""Turns off all preprocessing that happens only once before alignment (--normproc, --mask, --maskfile, --clipali, --threshold; i.e., all preprocessing excepting filters --highpass, --lowpass, --preprocess, and --shrink.""")
parser.add_argument("--shrink", type=int,default=1,help="""Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount for coarse alignment.""", guitype='shrinkbox', row=5, col=1, rowspan=1, colspan=1, mode='alignment,breaksym')
parser.add_argument("--shrinkfine", type=int,default=1,help="""Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount for refine alignment.""", guitype='intbox', row=5, col=2, rowspan=1, colspan=1, mode='alignment')
parser.add_argument("--threshold",type=str,default='',help="""Default=None. A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=10, col=0, rowspan=1, colspan=3, mode='alignment,breaksym')
parser.add_argument("--mask",type=str,default='', help="""Default=None. Masking processor applied to particles before alignment. IF using --clipali, make sure to express outer mask radii as negative pixels from the edge.""", returnNone=True, guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'mask\')', row=11, col=0, rowspan=1, colspan=3, mode='alignment,breaksym')
parser.add_argument("--maskfile",type=str,default='',help="""Default=None. Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None.""")
parser.add_argument("--normproc",type=str, default='',help="""Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to particles before alignment. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'""")
parser.add_argument("--clipali",type=int,default=0,help="""Default=0 (which means it's not used). Boxsize to clip particles as part of preprocessing to speed up alignment. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary; still, there are some benefits from 'oversampling' the data during averaging; so you might still want an average of size 2x, but perhaps particles in a box of 1.5x are sufficiently good for alignment. In this case, you would supply --clipali=75""")
'''
POST-FFT filtering parameters
'''
parser.add_argument("--preprocess",type=str,default='',help="""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=10, col=0, rowspan=1, colspan=3, mode='alignment,breaksym')
parser.add_argument("--preprocessfine",type=str,default='',help="""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging.""")
parser.add_argument("--lowpass",type=str,default='',help="""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=17, col=0, rowspan=1, colspan=3, mode='alignment,breaksym')
parser.add_argument("--lowpassfine",type=str,default='',help="""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging.""")
parser.add_argument("--highpass",type=str,default='',help="""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""", guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=18, col=0, rowspan=1, colspan=3, mode='alignment,breaksym')
parser.add_argument("--highpassfine",type=str,default='',help="""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging.""")
parser.add_argument("--matchimgs",action='store_true',default=False,help="""Default=False. Applies filter.matchto to one image so that it matches the other's spectral profile during preprocessing for pair-wise alignment purposes.""")
parser.add_argument("--filterbyfsc",action='store_true',default=False,help="""Default=False. If on, this parameter will use dynamic FSC filtering. --lowpass will be used to build initial references if no --ref supplied, then, the FSC between the even and odd initial references will be used to filter the data during preprocessing. If --ref is supplied, --lowpass will be used during the first iteration to align the particles against the reference. Thereafter, the FSC between the most current particle average and the original reference (--ref) will be used in the next iteration.""")
'''
OTHER ADVANCED parameters
'''
parser.add_argument("--radius", type=float, default=0, help="""Default=0 (which means it's not used by default). Hydrodynamic radius of the particle in Angstroms. This will be used to automatically calculate the angular steps to use in search of the best alignment. Make sure the apix is correct on the particles' headers, sine the radius will be converted from Angstroms to pixels. Then, the fine angular step is equal to 360/(2*pi*radius), and the coarse angular step 4 times that.""")
parser.add_argument("--precision",type=float,default=1.0,help="""Default=1.0. Precision in pixels to use when figuring out alignment parameters automatically using --radius. Precision would be the number of pixels that the the edge of the specimen is moved (rotationally) during the finest sampling, --falign. If precision is 1, then the precision of alignment will be that of the sampling (apix of your images) times the --shrinkfine factor specified.""")
parser.add_argument("--search", type=int,default=8,help=""""Default=8. During COARSE alignment translational search in X, Y and Z, in pixels. Default=8. This WILL overwrite any search: provided through --align, EXCEPT if you provide --search=8, which is the default. In general, just avoid providing search twice (through here and through the aligner, --align). If you do, just be careful to make them consistent to minimize misinterpretation and error.""")
parser.add_argument("--searchfine", type=int,default=2,help=""""Default=2. During FINE alignment translational search in X, Y and Z, in pixels. Default=2. This WILL overwrite any search: provided through --falign, EXCEPT if you provide --searchfine=2, which is the default. In general, just avoid providing search twice (through here and through the fine aligner --falign). If you do, just be careful to make them consistent to minimize misinterpretation and error.""")
#parser.add_argument("--donotaverage",action="store_true", help="""If e2spt_refinemulti.py is calling e2spt_classaverage.py, the latter need not average any particles, but rather only yield the alignment results.""", default=False)
parser.add_argument("--iterstop", type=int, default=0, help="""Default=0. (Not used). The program is called to convergence by default (all particles merge into one final average). To stop at an intermediate iteration, provide this parameter. For example, --iterstop=1, will only allow the algorithm to complete 1 iteration; --iterstop=2 will allow it to go through 2, etc.""")
parser.add_argument("--savesteps",action="store_true", default=False, help="""Default=False. If set, will save the average after each iteration to class_#.hdf. Each class in a separate file. Appends to existing files.""", guitype='boolbox', row=4, col=0, rowspan=1, colspan=1, mode='alignment,breaksym')
parser.add_argument("--saveali",action="store_true", default=False, help="""Default=False. If set, will save the aligned particle volumes in class_ptcl.hdf. Overwrites existing file.""", guitype='boolbox', row=4, col=1, rowspan=1, colspan=1, mode='alignment,breaksym')
parser.add_argument("--saveallalign",action="store_true", default=False, help="""Default=False. If set, will save the alignment parameters after each iteration""", guitype='boolbox', row=4, col=2, rowspan=1, colspan=1, mode='alignment,breaksym')
parser.add_argument("--sym", dest = "sym", default='', help = """Default=None (equivalent to c1). Symmetry to impose -choices are: c<n>, d<n>, h<n>, tet, oct, icos""", guitype='symbox', row=9, col=1, rowspan=1, colspan=2, mode='alignment,breaksym')
parser.add_argument("--postprocess",type=str,default='',help="""A processor to be applied to the FINAL volume after averaging the raw volumes in their FINAL orientations, after all iterations are done.""",guitype='comboparambox', choicelist='re_filter_list(dump_processors_list(),\'filter\')', row=16, col=0, rowspan=1, colspan=3, mode='alignment,breaksym')
parser.add_argument("--procfinelikecoarse",action='store_true',default=False,help="""If you supply this parameters, particles for fine alignment will be preprocessed identically to particles for coarse alignment by default. If you supply this, but want specific parameters for preprocessing particles for also supply: fine alignment, nd supply fine alignment parameters, such as --lowpassfine, --highpassfine, etc; to preprocess the particles for FINE alignment differently than for COARSE alignment.""")
parser.add_argument("--falign",type=str,default='',help="""Default=None. This is the second stage aligner used to fine-tune the first alignment.""", returnNone=True, guitype='comboparambox', choicelist='re_filter_list(dump_aligners_list(),\'refine.*3d\')', row=14, col=0, rowspan=1, colspan=3, nosharedb=True, mode='alignment,breaksym[None]')
parser.add_argument("--faligncmp",type=str,default="ccc.tomo.thresh",help="""Default=ccc.tomo.thresh. The comparator used by the second stage aligner.""", guitype='comboparambox', choicelist='re_filter_list(dump_cmps_list(),\'tomo\')', row=15, col=0, rowspan=1, colspan=3,mode="alignment,breaksym")
#parser.add_argument("--nopreprocprefft",action="store_true",default=False,help="""Turns off all preprocessing that happens only once before alignment (--normproc, --mask, --maskfile, --clipali, --threshold; i.e., all preprocessing excepting filters --highpass, --lowpass, --preprocess, and --shrink.""")
#parser.add_argument("--keep",type=float,default=1.0,help="""Default=1.0 (all particles kept). The fraction of particles to keep in each class.""", guitype='floatbox', row=6, col=0, rowspan=1, colspan=1, mode='alignment,breaksym')
#parser.add_argument("--keepsig", action="store_true", default=False,help="""Default=False. Causes the keep argument to be interpreted in standard deviations.""", guitype='boolbox', row=6, col=1, rowspan=1, colspan=1, mode='alignment,breaksym')
#parser.add_argument("--inixforms",type=str,default="",help="""Default=None. .json file containing a dict of transforms to apply to 'pre-align' the particles.""", guitype='dirbox', dirbasename='spt_|sptsym_', row=7, col=0,rowspan=1, colspan=2, nosharedb=True, mode='breaksym')
parser.add_argument("--breaksym",action="store_true", default=False,help="""Default=False. Break symmetry. Do not apply symmetrization after averaging, even if searching the asymmetric unit provided through --sym only for alignment. Default=False""", guitype='boolbox', row=7, col=2, rowspan=1, colspan=1, nosharedb=True, mode=',breaksym[True]')
#parser.add_argument("--groups",type=int,default=0,help="""Default=0 (not used; data not split). This parameter will split the data into a user defined number of groups. For purposes of gold-standard FSC computation later, select --group=2.""")
parser.add_argument("--randomizewedge",action="store_true", default=False,help="""Default=False. This parameter is EXPERIMENTAL. It randomizes the position of the particles BEFORE alignment, to minimize missing wedge bias and artifacts during symmetric alignment where only a fraction of space is scanned""")
#parser.add_argument("--savepreproc",action="store_true", default=False,help="""Default=False. Will save stacks of preprocessed particles (one for coarse alignment and one for fine alignment if preprocessing options are different).""")
parser.add_argument("--autocenter",type=str, default='',help="""Default=None. Autocenters each averaged pair during initial average generation with --btref and --hacref. Will also autocenter the average of all particles after each iteration of iterative refinement. Options are --autocenter=xform.centerofmass (self descriptive), or --autocenter=xform.centeracf, which applies auto-convolution on the average.""")
parser.add_argument("--autocentermask",type=str, default='',help="""Default=None. Masking processor to apply before autocentering. See 'e2help.py processors -v 10' at the command line.""")
parser.add_argument("--autocenterpreprocess",action='store_true', default=False,help="""Default=False. This will apply a highpass filter at a frequency of half the box size times the apix, shrink by 2, and apply a low pass filter at half nyquist frequency to any computed average for autocentering purposes if --autocenter is provided. Default=False.""")
parser.add_argument("--tweak",action='store_true',default=False,help="""WARNING: BUGGY. This will perform a final alignment with no downsampling [without using --shrink or --shrinkfine] if --shrinkfine > 1.""")
'''
BT SPECIFIC PARAMETERS
'''
parser.add_argument("--nseedlimit",type=int,default=0,help="""Maximum number of particles
to use. For example, if you supply a stack with 150 subtomograms, the program will
automatically select 128 as the limit to use because it's the largest power of 2 that is
smaller than 150. But if you provide, say --nseedlimit=100, then the number of particles
used will be 64, because it's the largest power of 2 that is still smaller than 100.""")
(options, args) = parser.parse_args()
options.nopreprocprefft = False
if options.shrink < options.shrinkfine:
options.shrink = options.shrinkfine
print "\n(e2spt_binarytree)(main) it makes no sense for shrinkfine to be larger than shrink; therefore, shrink will be made to match shrinkfine"
from e2spt_classaverage import checksaneimagesize
checksaneimagesize( options, options.input )
'''
Make the directory where to create the database where the results will be stored
'''
from e2spt_classaverage import sptmakepath
options = sptmakepath(options,'spt_bt')
rootpath = os.getcwd()
if rootpath not in options.path:
options.path = rootpath + '/' + options.path
if not options.input:
parser.print_help()
exit(0)
elif options.subset:
subsetStack = options.path + '/subset' + str( options.subset ).zfill( len( str( options.subset))) + '.hdf'
print "\nSubset to be written to", subsetStack
subsetcmd = 'e2proc3d.py ' + options.input + ' ' + subsetStack + ' --first=0 --last=' + str(options.subset-1)
print "Subset cmd is", subsetcmd
p=subprocess.Popen( subsetcmd, shell=True,stdout=subprocess.PIPE, stderr=subprocess.PIPE )
text=p.communicate()
p.stdout.close()
options.input = subsetStack
from e2spt_classaverage import sptParseAligner
options = sptParseAligner( options )
'''
If --radius of the particle is provided, we calculate the optimal alignment steps for
coarse and fine alignment rounds using --shrink and --shrinkfine options and apix info
'''
if options.radius:
from e2spt_classaverage import calcAliStep
options = calcAliStep(options)
'''
Parse parameters such that "None" or "none" are adequately interpreted to turn of an option
'''
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser( options )
from e2spt_classaverage import writeParameters
writeParameters(options,'e2spt_binarytree.py', 'bt')
hdr = EMData(options.input,0,True)
nx = hdr["nx"]
ny = hdr["ny"]
nz = hdr["nz"]
if nx!=ny or ny!=nz :
print "ERROR, input volumes are not cubes"
sys.exit(1)
logger = E2init(sys.argv, options.ppid)
'''
Initialize parallelism if being used
'''
if options.parallel :
if options.parallel == 'none' or options.parallel == 'None' or options.parallel == 'NONE':
options.parallel = ''
etc = ''
else:
print "\n\n(e2spt_classaverage.py) INITIALIZING PARALLELISM!"
print "\n\n"
from EMAN2PAR import EMTaskCustomer
etc=EMTaskCustomer(options.parallel)
pclist=[options.input]
etc.precache(pclist)
else:
etc=''
options.raw = options.input
"""
if 'tree' in options.align:
options.falign = None
options.mask = None
options.lowpass = None
options.highpass = None
options.normproc = None
options.lowpassfine = None
options.highpassfine = None
options.preprocess = None
options.preprocessfine = None
else:
from e2spt_classaverage import cmdpreproc
cmpreproc( options.input, options, False )
"""
nptcl=EMUtil.get_image_count(options.input)
if nptcl < 1:
print "ERROR : at least 2 particles required in input stack"
sys.exit(1)
ptclnums=range(nptcl)
nptclForRef = len(ptclnums)
nseed=2**int(floor(log(len(ptclnums),2))) # we stick with powers of 2 for this to make the tree easier to collapse
if options.nseedlimit:
nseed=2**int(floor(log( options.nseedlimit , 2)))
#binaryTreeRef(options,nptclForRef,nseed,-1,etc)
binaryTreeRef(options,nptclForRef,nseed,etc)
print "Will end logger"
E2end(logger)
print "logger ended"
sys.stdout.flush()
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """
This program takes a subtomgoram tiltseries (subtiltseries) as extracted with
e2spt_subtilt.py, and computes the resolution of two volumes reconstructed with
the even and the odd images in the tilt series. Must be in HDF format.
Note that the apix in the header must be accurate to get sensible results.
(You can fix the header of an image with e2fixheaderparam.py).
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--inputstem",
type=str,
default='',
help=
"""Default=None. Aligned tilt series. String common to all files to be processed, in the current folder. For example, if you have many subtiltseries named subt00.hdf, subt01.hdf, ...subt99.hdf, you would supply --stem=subt to have all these processed."""
)
parser.add_argument(
'--path',
type=str,
default='sptintrafsc',
help="""Default=sptintrafsc. Directory to save the results.""")
parser.add_argument(
'--nonewpath',
action='store_true',
default=False,
help=
"""Default=False. If True, a new --path directory will not be made. Therefore, whatever is sepcified in --path will be used as the output directory. Note that this poses the risk of overwriting data."""
)
parser.add_argument(
'--input',
type=str,
default='',
help=
"""Default=None. Subtiltseries file to process. If processing a single file, --inputstem will work too, but you can also just provide the entire filename here --input=subt00.hdf"""
)
parser.add_argument(
'--savehalftiltseries',
action='store_true',
default=False,
help=
"""Default=False. If this parameter is on, the odd and even subtiltseries will be saved."""
)
parser.add_argument(
'--savehalfvolumes',
action='store_true',
default=False,
help=
"""Default=False. If this parameter is on, the odd and even volumes will be saved."""
)
parser.add_argument(
"--reconstructor",
type=str,
default="fourier:mode=gauss_2",
help=
"""Default=fourier:mode=gauss_2. The reconstructor to use to reconstruct the tilt series into a tomogram. Type 'e2help.py reconstructors' at the command line to see all options and parameters available. To specify the interpolation scheme for the fourier reconstructor, specify 'mode'. Options are 'nearest_neighbor', 'gauss_2', 'gauss_3', 'gauss_5'. For example --reconstructor=fourier:mode=gauss_5 """
)
parser.add_argument(
"--pad2d",
type=float,
default=0.0,
help=
"""Default=0.0. Padding factor (e.g., 2.0, to make the box twice as big) to zero-pad the 2d images in the tilt series for reconstruction purposes (the final reconstructed subvolumes will be cropped back to the original size though)."""
)
parser.add_argument(
"--pad3d",
type=float,
default=0.0,
help=
"""Default=0.0. Padding factor (e.g., 2.0, to make the box twice as big) to zero-pad the volumes for reconstruction purposes (the final reconstructed subvolumes will be cropped back to the original size though)."""
)
parser.add_argument(
"--averager",
type=str,
default="mean.tomo",
help=
"""Default=mean.tomo. The type of averager used to produce the class average."""
)
parser.add_argument(
"--averagehalves",
action="store_true",
default=False,
help="""Default=False. This will averager the even and odd volumes.""")
parser.add_argument(
"--ppid",
type=int,
default=-1,
help=
"""default=-1. Set the PID of the parent process, used for cross platform PPID."""
)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness")
parser.add_argument(
"--nolog",
action="store_true",
default=False,
help=
"Default=False. Turn off recording of the command ran for this program onto the .eman2log.txt file"
)
(options, args) = parser.parse_args()
#if options.reconstructor == 'None' or options.reconstructor == 'none':
# options.reconstructor = None
#if options.reconstructor and options.reconstructor != 'None' and options.reconstructor != 'none':
# options.reconstructor=parsemodopt(options.reconstructor)
#if options.averager:
# options.averager=parsemodopt(options.averager)
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser(options)
logger = E2init(sys.argv, options.ppid)
'''
Make the directory where to create the database where the results will be stored
'''
if not options.nonewpath:
from e2spt_classaverage import sptmakepath
options = sptmakepath(options, 'sptintrafsc')
else:
try:
findir = os.lisdir(options.path)
except:
print "ERROR: The path specified %s does not exist" % (
options.path)
sys.exit()
inputfiles = []
if options.inputstem:
c = os.getcwd()
findir = os.listdir(c)
for f in findir:
if '.hdf' in f and options.inputstem in f:
if options.verbose > 8:
print "\nFound tiltseries!", f
inputfiles.append(
f
) #C:The input files are put into a dictionary in the format {originalseriesfile:[originalseriesfile,volumefile]}
elif options.input:
inputfiles.append(options.input)
for fi in inputfiles:
#genOddAndEvenVols( options, fi )
ret = genOddAndEvenVols(options, fi, [])
volOdd = ret[1]
volEven = ret[0]
if options.savehalfvolumes and volOdd and volEven:
volOdd.write_image(
options.path + '/' + fi.replace('.hdf', '_ODDVOL.hdf'), 0)
volEven.write_image(
options.path + '/' + fi.replace('.hdf', '_EVENVOL.hdf'), 0)
retfsc = fscOddVsEven(options, fi, volOdd, volEven)
fscfilename = retfsc[0]
fscarea = retfsc[1]
if options.averagehalves:
avgr = Averagers.get(options.averager[0], options.averager[1])
avgr.add_image(recOdd)
avgr.add_image(recEven)
avg = avgr.finish()
avg['origin_x'] = 0
avg['origin_y'] = 0
avg['origin_z'] = 0
avg['apix_x'] = apix
avg['apix_y'] = apix
avg['apix_z'] = apix
avgfile = options.path + '/AVG.hdf'
avg.write_image(avgfile, 0)
E2end(logger)
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """
This program takes a subtomgoram tiltseries (subtiltseries) as extracted with
e2spt_subtilt.py, and computes the resolution of two volumes reconstructed with
the even and the odd images in the tilt series. Must be in HDF format.
Note that the apix in the header must be accurate to get sensible results.
(You can fix the header of an image with e2fixheaderparam.py).
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--inputstem", type=str, default='', help="""Default=None. Aligned tilt series. String common to all files to be processed, in the current folder. For example, if you have many subtiltseries named subt00.hdf, subt01.hdf, ...subt99.hdf, you would supply --stem=subt to have all these processed.""")
parser.add_argument('--path',type=str,default='sptintrafsc',help="""Default=sptintrafsc. Directory to save the results.""")
parser.add_argument('--nonewpath',action='store_true',default=False,help="""Default=False. If True, a new --path directory will not be made. Therefore, whatever is sepcified in --path will be used as the output directory. Note that this poses the risk of overwriting data.""")
parser.add_argument('--input',type=str,default='',help="""Default=None. Subtiltseries file to process. If processing a single file, --inputstem will work too, but you can also just provide the entire filename here --input=subt00.hdf""")
parser.add_argument('--savehalftiltseries',action='store_true',default=False,help="""Default=False. If this parameter is on, the odd and even subtiltseries will be saved.""")
parser.add_argument('--savehalfvolumes',action='store_true',default=False,help="""Default=False. If this parameter is on, the odd and even volumes will be saved.""")
parser.add_argument("--reconstructor", type=str,default="fourier:mode=gauss_2",help="""Default=fourier:mode=gauss_2. The reconstructor to use to reconstruct the tilt series into a tomogram. Type 'e2help.py reconstructors' at the command line to see all options and parameters available. To specify the interpolation scheme for the fourier reconstructor, specify 'mode'. Options are 'nearest_neighbor', 'gauss_2', 'gauss_3', 'gauss_5'. For example --reconstructor=fourier:mode=gauss_5 """)
parser.add_argument("--pad2d", type=float,default=0.0,help="""Default=0.0. Padding factor (e.g., 2.0, to make the box twice as big) to zero-pad the 2d images in the tilt series for reconstruction purposes (the final reconstructed subvolumes will be cropped back to the original size though).""")
parser.add_argument("--pad3d", type=float,default=0.0,help="""Default=0.0. Padding factor (e.g., 2.0, to make the box twice as big) to zero-pad the volumes for reconstruction purposes (the final reconstructed subvolumes will be cropped back to the original size though).""")
parser.add_argument("--averager",type=str,default="mean.tomo",help="""Default=mean.tomo. The type of averager used to produce the class average.""")
parser.add_argument("--averagehalves",action="store_true", default=False,help="""Default=False. This will averager the even and odd volumes.""")
parser.add_argument("--ppid", type=int, default=-1, help="""default=-1. Set the PID of the parent process, used for cross platform PPID.""")
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n",type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness")
parser.add_argument("--nolog",action="store_true",default=False,help="Default=False. Turn off recording of the command ran for this program onto the .eman2log.txt file")
(options, args) = parser.parse_args()
#if options.reconstructor == 'None' or options.reconstructor == 'none':
# options.reconstructor = None
#if options.reconstructor and options.reconstructor != 'None' and options.reconstructor != 'none':
# options.reconstructor=parsemodopt(options.reconstructor)
#if options.averager:
# options.averager=parsemodopt(options.averager)
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser( options )
logger = E2init( sys.argv, options.ppid )
'''
Make the directory where to create the database where the results will be stored
'''
if not options.nonewpath:
from e2spt_classaverage import sptmakepath
options = sptmakepath (options, 'sptintrafsc')
else:
try:
findir = os.lisdir( options.path )
except:
print "ERROR: The path specified %s does not exist" %( options.path )
sys.exit()
inputfiles = []
if options.inputstem:
c = os.getcwd()
findir = os.listdir( c )
for f in findir:
if '.hdf' in f and options.inputstem in f:
if options.verbose > 8:
print "\nFound tiltseries!", f
inputfiles.append( f ) #C:The input files are put into a dictionary in the format {originalseriesfile:[originalseriesfile,volumefile]}
elif options.input:
inputfiles.append( options.input )
for fi in inputfiles:
#genOddAndEvenVols( options, fi )
ret = genOddAndEvenVols( options, fi,[] )
volOdd = ret[1]
volEven = ret[0]
if options.savehalfvolumes and volOdd and volEven:
volOdd.write_image( options.path + '/' + fi.replace('.hdf','_ODDVOL.hdf'), 0 )
volEven.write_image( options.path + '/' + fi.replace('.hdf','_EVENVOL.hdf'), 0 )
retfsc = fscOddVsEven( options, fi, volOdd, volEven )
fscfilename = retfsc[0]
fscarea = retfsc[1]
if options.averagehalves:
avgr = Averagers.get( options.averager[0], options.averager[1] )
avgr.add_image( recOdd )
avgr.add_image( recEven )
avg = avgr.finish()
avg['origin_x'] = 0
avg['origin_y'] = 0
avg['origin_z'] = 0
avg['apix_x'] = apix
avg['apix_y'] = apix
avg['apix_z'] = apix
avgfile = options.path + '/AVG.hdf'
avg.write_image( avgfile, 0 )
E2end(logger)
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """Program for individual subtomogram refinement (ISR) based on subtiltseries
for each subtomogram extracted with e2spt_subtilt.py."""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
'--input',
type=str,
default='',
help=
"""Comma separated files in .ali, .st .hdf format of the aligned subtiltseries."""
)
parser.add_argument(
'--inputstem',
type=str,
default='',
help=
"""Alternative to supplying --input. This is a string common to multiple files to be processed in the CURERENT directory. The common string doesn't need to be at a particular location in the filenames. For example, a series of files "tiltA.hdf, tiltB.hdf, tiltC.hdf" could have either 'hdf', '.hdf', 't,','ti', 'til', 'tilt', etc., as a common string. The key is to choose a string shared ONLY by the files of interest. The files should be multiple subtiltseries in .hdf format; each file should correspond to an individual subtiltseries for a different particle: That is, each file should be a subtiltseries corresponding to an individual subtomogram, as extracted by e2spt_subtilt.py, or as simulated by e2spt_simulation.py"""
)
parser.add_argument(
'--inputdir',
type=str,
default='',
help=
"""Alternative to --input and --inputstem. Path to a directory containing individual subtiltseries stacks."""
)
parser.add_argument(
"--path",
type=str,
default='',
help=
"Directory to store results in. The default is a numbered series of directories containing the prefix 'sptisr'; for example, sptisr02 will be the directory by default if 'sptisr_01' already exists."
)
parser.add_argument(
"--ppid",
type=int,
help=
"Default=1. Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument(
"--verbose",
"-v",
type=int,
default=0,
help=
"Default 0. Verbose level [0-9], higner number means higher level of verboseness",
dest="verbose",
action="store",
metavar="n")
parser.add_argument(
"--reconstructor",
type=str,
default="fourier:mode=gauss_2",
help=
"""Default=fourier:mode=gauss_2. The reconstructor to use to reconstruct the tilt series into a tomogram. Type 'e2help.py reconstructors' at the command line to see all options and parameters available. To specify the interpolation scheme for the fourier reconstructor, specify 'mode'. Options are 'nearest_neighbor', 'gauss_2', 'gauss_3', 'gauss_5'. For example --reconstructor=fourier:mode=gauss_5 """
)
parser.add_argument("--iter",
type=int,
default=1,
help="""Number of iterations to run algorithm for.""")
parser.add_argument(
"--tltfile",
type=str,
default='',
help=
"""IMOD-like .tlt file with tilt angles for the aligned tiltseries (or set of subtiltseries)."""
)
parser.add_argument(
"--tiltaxis",
type=str,
default='y',
help=
"""Axis to produce projections about. Default is 'y'; the only other valid option is 'x'."""
)
parser.add_argument(
"--pad2d",
type=float,
default=0.0,
help=
"""Default=0.0. Padding factor (e.g., 2.0, to make the box twice as big) to zero-pad the 2d images in the tilt series for reconstruction purposes (the final reconstructed subvolumes will be cropped back to the original size though)."""
)
parser.add_argument(
"--pad3d",
type=float,
default=0.0,
help=
"""Default=0.0. Padding factor (e.g., 2.0, to make the box twice as big) to zero-pad the volumes for reconstruction purposes (the final reconstructed subvolumes will be cropped back to the original size though)."""
)
parser.add_argument(
"--savevols",
action='store_true',
default=False,
help=
"""This option will save the reconstructed volumes at each iteration."""
)
parser.add_argument(
"--outxsize",
type=int,
default=0,
help=
'''Clip the output volume in x to this size. The default size is the nx size of the input images.'''
)
parser.add_argument(
"--outysize",
type=int,
default=0,
help=
'''Clip the output volume in y to this size. The default size is the ny size of the input images.'''
)
parser.add_argument(
"--outzsize",
type=int,
default=0,
help=
'''Clip the output volume in z to this size. The default size is the nx size of the input images.'''
)
parser.add_argument(
"--mask",
type=str,
help=
"""Default=None. Masking processor (see e2help.py --verbose=10) applied to the images to aid alignment. Default=None."""
)
parser.add_argument(
"--preprocess",
type=str,
help=
"""Default=None. Any processor (see e2help.py --verbose=10) applied to the images to aid alignment."""
)
parser.add_argument(
"--lowpass",
type=str,
default='',
help=
"""Default=None. A lowpass filtering processor (see e2help.py --verbose=10) applied to each volume prior to reprojection generation.."""
)
parser.add_argument(
"--highpass",
type=str,
default='',
help=
"""Default=None. A highpass filtering processor (see e2help.py --verbose=10) applied to each volume prior to reprojection generation."""
)
parser.add_argument(
"--threshold",
type=str,
default='',
help=
"""Default=None. A threshold processor (see e2help.py --verbose=10) applied to each volume prior to reprojection generation."""
)
parser.add_argument(
"--saveali",
action="store_true",
default=False,
help=
"""Default=False. If set, will save the recentered subtiltseries after each iteration."""
)
(options, args) = parser.parse_args()
logger = E2init(sys.argv, options.ppid)
if not options.input and not options.inputdir and not options.inputstem:
print "ERROR: Either of the following required: --input, --inputstemp, --inputdir"
sys.exit()
if options.input and options.inputstem:
print "ERROR: Cannot provide --input and --inputstem simultaneously"
sys.exit()
if options.inputstem and options.inputdir:
print "ERROR: Cannot provide --inputstem and --inputdir simultaneously"
sys.exit()
if options.input and options.inputdir:
print "ERROR: Cannot provide --input and --inputdir simultaneously"
sys.exit()
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser(options)
#print "Options have been parsed, for example, mask is", options.mask, type(options.mask), len(options.mask)
from e2spt_classaverage import sptmakepath
options = sptmakepath(options, 'sptisr')
originalpath = options.path
if options.verbose > 9:
print "\n(e2spt_isr.py) I've read the options"
inputfiles = {
} #C:Figure out whether there's a single HDF stack to process,
#C:or a directory with many HDF stacks
c = os.getcwd()
if options.inputdir:
c = os.getcwd() + '/' + options.inputdir
findir = os.listdir(c)
if options.inputstem:
for f in findir:
if '.hdf' in f and options.inputstem in f:
if options.verbose > 8:
print "\nFound tiltseries!", f
inputfiles.update(
{f: [f, None]}
) #C:The input files are put into a dictionary in the format {originalseriesfile:[originalseriesfile,volumefile]}
elif options.inputdir:
for f in findir:
if '.hdf' in f:
if options.verbose > 8:
print "\nFound tiltseries!", f
inputfiles.update(
{f: [f, None]}
) #C:The input files are put into a dictionary in the format {originalseriesfile:[originalseriesfile,volumefile]}
elif options.input:
inputfiles.update({options.input: [options.input, None]})
tiltstep = 0
newvol = vol = None
newfiles = {}
firstiterdir = originalpath
print "\n\nThere are these many iterations", options.iter
fstats = {}
convergedfs = []
fmeanscores2d = {}
itermeanscores2d = {}
fmeanerrors = {}
itermeanerrors = {}
fmeanfscs = {
-1: {}
} #Compute initial FSC before any iterations of correction or alignment have occurred
itermeanfscs = {}
fmeanscores3d = {-1: {}}
itermeanscores3d = {}
errorswitch = 0
for i in range(options.iter): #Iterate over options.iter
itermeanscores2d.update({i: 0})
itermeanerrors.update({i: 0})
itermeanfscs.update({i: 0})
itermeanscores3d.update({i: 0})
print "***************\nStarting iteration number", i
print "****************"
previouspath = ''
if options.iter > 1:
iterdir = 'iter_' + str(i + 1).zfill(len(str(options.iter)))
os.system(
'mkdir ' + originalpath + '/' + iterdir
) #C:Make a new subdirectory within options.path only if options.iter > 0
options.path = originalpath + '/' + iterdir #C:Update path to include the newly created subdirectory
previouspath = originalpath + '/iter_' + str(i).zfill(
len(str(options.iter)))
if i == 0:
firstiterdir = options.path
kk = 0
statsfile = options.path + '/stats.txt'
statslines = []
if len(convergedfs) == len(inputfiles):
print "\nAll files have converged. Terminating. Line 193"
break
fmeanscores2d.update({i: {}})
fmeanerrors.update({i: {}})
fmeanfscs.update({i: {}})
fmeanscores3d.update({i: {}})
for f in inputfiles:
fmeanscores2d[i].update({f: [0]})
fmeanerrors[i].update({f: [0]})
fmeanfscs[i].update({f: [0]})
fmeanscores3d[i].update({f: [0]}) #C:Iterate over files
#originalseries = f
if i == 0:
fstats.update({f: list([])})
print "fmeanfscs is", fmeanfscs
fmeanfscs[-1].update(
{f: [0]}
) #Compute initial FSC before any iterations of correction or alignment have occurred
print "fmeanscores3d is", fmeanscores3d
fmeanscores3d[-1].update({f: [0]})
print "set initial -1 fmeanscores3d!!"
if 'converged' not in fstats[f]:
if i == 0:
if options.tltfile:
originalangles = getangles(
options) #C:Read angles from tlt file if available
else:
originalangles = calcangles(
f
) #C:Get the angles of the actual data tilts from the header or calculate from input parameters
stackfile = options.path + '/' + os.path.basename(f).replace(
'.hdf', '_ISR.hdf')
print "\nWill refine center for file", f
if i == 0:
hdr = EMData(
f, 0, True
) #See if angles are in the header of the data, for each file; if not, write them
#print "\nRead header and its type", hdr, type(hdr)
#print "\nAnd the dictionary", hdr.get_attr_dict()
size = hdr['nx']
aux = 0
if 'spt_tiltangle' not in hdr.get_attr_dict():
print "\nspt_tiltangle not in header, therefore will write it by calling writeparamtoheader"
aux = writeparamtoheader(f, originalangles,
'spt_tiltangle')
print "\naux returned is", aux
else:
aux = 1
if 'sptisrtx' not in hdr.get_attr_dict(
) or 'sptisrty' not in hdr.get_attr_dict(
) or 'sptisrdr' not in hdr.get_attr_dict():
tvals = [0.0 for tt in range(len(originalangles))]
auxx = auxy = auxr = 0
if 'sptisrtx' not in hdr.get_attr_dict():
print "\nsptisrtx not in header, therefore will write it by calling writeparamtoheader"
auxx = writeparamtoheader(f, tvals, 'sptisrtx')
else:
auxx = 1
if 'sptisrty' not in hdr.get_attr_dict():
print "\nsptisrty not in header, therefore will write it by calling writeparamtoheader"
auxy = writeparamtoheader(f, tvals, 'sptisrty')
else:
auxy = 1
if 'sptisrdr' not in hdr.get_attr_dict():
print "\nsptisrdr not in header, therefore will write it by calling writeparamtoheader"
auxr = writeparamtoheader(f, tvals, 'sptisrdr')
else:
auxr = 1
if auxx and auxy and auxr:
aux = 1
else:
aux = 1
if aux:
series = {}
nimgs = EMUtil.get_image_count(f)
for ii in range(nimgs):
img = EMData(f, ii)
try:
angle = img['spt_tiltangle']
series.update({angle: img})
except:
print "ERROR: spt_tiltangle not found in image", ii
print "\nHeader is", img.get_attr_dict()
sys.exit()
retm = makevol(
options,
f,
series,
i,
originalangles,
size,
writevols=1,
initialfsc=1
) #C:In the first iteration, first reconstruct the tilt series into a 3D volume
vol = retm[0]
#newvolfile = retm[1]
fscarea = retm[2]
score3d = retm[3]
#fsc = retm[3]
#initialfscfilename = options.path + '/' + os.path.basename( f ).replace('.hdf', '_initial_evenOddFSC.txt')
fmeanfscs[-1][f].append(fscarea)
fmeanscores3d[-1][f].append(score3d)
print "\nVol and its type are", vol, type(vol)
else:
print "ERROR: Something went wrong. spt_tiltangle found in image headers, but somehow is unusuable"
sys.exit()
elif i > 0:
#vol = EMData( inputfiles[f][1] ) #C:For iterations > 0 (after the first iteration), the updated volumes
#C:for each tilt series should exist from the previous iteration
#previousstackfile = previouspath + '/' + os.path.basename(f).replace('.hdf','_IPET.hdf')
#print "\n\n\previousstackfile is", previousstackfile
#vol = EMData( previousstackfile, 0 )
vol = newvol
#C:Make reprojections from all known angles
reprojections = reprojectvolume(options, vol, f,
originalangles)
originalsize = vol['nx']
retrct = recentertilts(options, reprojections, f,
originalangles, i)
newseries = retrct[0]
statsline = retrct[1]
fscores2d = retrct[2]
ferrors = retrct[3]
if ferrors:
errorswitch = 1
#print "fscores2d received", fscores2d
#print "ferrors received", ferrors
if ferrors == fscores2d:
print "ERROR: errors and scores2d are the same"
sys.exit()
fmeanscore2d = sum(fscores2d) / len(fscores2d)
fmeanscores2d[i][f].append(fmeanscore2d)
if ferrors and errorswitch:
fmeanerror = sum(ferrors) / len(ferrors)
fmeanerrors[i][f].append(fmeanerror)
#fmeanscores2d.update( i:{ f:fmeanscore2d } )
statslines.append(statsline)
line2append = statsline
#print "fstats[f] and type are", fstats[f], type(fstats[f])
#sys.exit()
if statsline in fstats[f]:
line2append = 'converged'
convergedfs.append(f)
fstats[f].append(line2append)
fstats.update({f: fstats[f]})
retmkvol = makevol(options,
f,
newseries,
i,
originalangles,
originalsize,
writevols=1)
newvol = retmkvol[0]
newvolfile = retmkvol[1]
fscarea = retmkvol[2]
fmeanfscs[i][f].append(fscarea)
fmeanscore3d = retmkvol[3]
fmeanscores3d[i][f].append(fmeanscore3d)
if i == 0:
pass
#print "\nEEEEEEEEEEEEEEE\n\n\n\n\nNewvolfile returned is", newvolfile
newfiles.update({f: [newseries, newvolfile]})
#newvol.write_image( stackfile, 0 )
kk += 1
#firstrawvolfile = firstiterdir + '/' + os.path.basename(f).replace('.hdf','_3D.hdf')
#fscfile = options.path + '/' + os.path.basename(f).replace('.hdf','FSC.txt')
#cmdfsc = 'e2proc3d.py ' + firstrawvolfile + ' ' + fscfile + ' --calcfsc=' + newvolfile
#os.popen( cmdfsc )
else:
statsline = fstats[f][-2]
statslines.append(statsline)
#Write mean tx, ty and dr of all files to a text file, for each iteration
f = open(statsfile, 'w')
f.writelines(statslines)
f.close()
inputfiles = newfiles
iterscores2d = []
#print "in iteration", i
#print "fmeanscores2d are",fmeanscores2d
for ff in fmeanscores2d[i]:
#print "therefore ff in fmeanscores2d[i] is", ff
iterscores2d.append(fmeanscores2d[i][ff][-1])
itermeanscore2d = sum(iterscores2d) / len(iterscores2d)
itermeanscores2d[i] = itermeanscore2d
ys = []
lines = []
for s in itermeanscores2d:
y = itermeanscores2d[s]
ys.append(y)
#print "appending this y", itermeanscores2d[s]
line = str(s) + '\t' + str(y) + '\n'
lines.append(line)
#if plots:
fs = open(options.path + '/scores2d_' + str(i + 1) + '.txt', 'w')
fs.writelines(lines)
fs.close()
try:
if errorswitch:
itererrors = []
#print "in iteration", i
#print "fmeanerrors are",fmeanerrors
for fscores2df in fmeanerrors[i]:
#print "therefore ff in fmeanerrors[i] is", ff
itererrors.append(fmeanerrors[i][ff][-1])
itermeanerror = sum(itererrors) / len(itererrors)
itermeanerrors[i] = itermeanerror
yse = []
linese = []
for s in itermeanerrors:
ye = itermeanerrors[s]
yse.append(ye)
#print "appending this error", itermeanerrors[s]
linee = str(s) + '\t' + str(ye) + '\n'
linese.append(linee)
#if plots:
fse = open(options.path + '/error_' + str(i + 1) + '.txt', 'w')
fse.writelines(linese)
fse.close()
except:
pass
if i == 0:
iterfscs = []
#print "in iteration", i
#print "APPENDING INITIAL fmeanfscs are", fmeanfscs
for ff in fmeanfscs[-1]:
#print "therefore INITIAL ff in fmeanfscs[-1] are", ff
iterfscs.append(fmeanfscs[-1][ff][-1])
itermeanfsc = sum(iterfscs) / len(iterfscs)
itermeanfscs[-1] = itermeanfsc
ysf = []
linesf = []
for s in [-1, 0]:
yf = itermeanfscs[s]
ysf.append(yf)
#print "for s", s
#print "appending this fscarea", itermeanfscs[s]
linef = str(s + 1) + '\t' + str(yf) + '\n'
linesf.append(linef)
#if plots:
fsf = open(options.path + '/fscsareas_' + str(i + 1) + '.txt', 'w')
fsf.writelines(linesf)
fsf.close()
iterscores3d = []
#print "in iteration", i
#print "fmeanscores2d are",fmeanscores2d
for ff in fmeanscores3d[-1]:
#print "therefore ff in fmeanscores2d[i] is", ff
iterscores3d.append(fmeanscores3d[-1][ff][-1])
itermeanscore3d = sum(iterscores3d) / len(iterscores3d)
itermeanscores3d[-1] = itermeanscore3d
ys3 = []
lines3 = []
for s in [-1, 0]:
y3 = itermeanscores3d[s]
ys3.append(y)
#print "appending this y", itermeanscores2d[s]
line3 = str(s + 1) + '\t' + str(y3) + '\n'
lines3.append(line3)
#if plots:
fs = open(options.path + '/scores3d_' + str(i + 1) + '.txt', 'w')
fs.writelines(lines3)
fs.close()
iterfscs = []
#print "in iteration", i
#print "fmeanfscs are",fmeanfscs
for ff in fmeanfscs[i]:
#print "therefore ff in fmeanfscs[i] are", ff
iterfscs.append(fmeanfscs[i][ff][-1])
itermeanfsc = sum(iterfscs) / len(iterfscs)
itermeanfscs[i] = itermeanfsc
ysf = []
linesf = []
fscsindxs = [s for s in itermeanfscs]
fscsindxs.sort()
for s in fscsindxs:
yf = itermeanfscs[s]
ysf.append(yf)
#print "for s", s
#print "appending this fscarea", itermeanfscs[s]
linef = str(s + 1) + '\t' + str(yf) + '\n'
linesf.append(linef)
#if plots:
fsf = open(options.path + '/fscsareas_' + str(i + 1) + '.txt', 'w')
fsf.writelines(linesf)
fsf.close()
iterscores3d = []
#print "in iteration", i
#print "fmeanscores2d are",fmeanscores2d
for ff in fmeanscores3d[i]:
#print "therefore ff in fmeanscores2d[i] is", ff
iterscores3d.append(fmeanscores3d[i][ff][-1])
itermeanscore3d = sum(iterscores3d) / len(iterscores3d)
itermeanscores3d[i] = itermeanscore3d
ys3 = []
lines3 = []
score3dindxs = [s for s in itermeanscores3d]
score3dindxs.sort()
for s in score3dindxs:
y3 = itermeanscores3d[s]
ys3.append(y)
#print "appending this y", itermeanscores2d[s]
line3 = str(s + 1) + '\t' + str(y3) + '\n'
lines3.append(line3)
#if plots:
fs = open(options.path + '/scores3d_' + str(i + 1) + '.txt', 'w')
fs.writelines(lines3)
fs.close()
if i > 0:
if itermeanscores2d[i] == itermeanscores2d[i - 1]:
print "The meanscore2d for two consecutive iterations is the same, suggesting the algorithm has converged."
sys.exit()
try:
if itermeanerrors[i] == itermeanerrors[i - 1]:
print "The meanerror for two consecutive iterations is the same, suggesting the algorithm has converged."
sys.exit()
except:
pass
if itermeanfscs[i] == itermeanfscs[i - 1]:
print "The fscsarea for two consecutive iterations is the same, suggesting the algorithm has converged."
sys.exit()
if itermeanscores3d[i] == itermeanscores3d[i - 1]:
print "The meanscore3d for two consecutive iterations is the same, suggesting the algorithm has converged."
sys.exit()
difscore = math.fabs(
float(itermeanscores2d[i]) - float(itermeanscores2d[i - 1]))
if float(difscore) < 0.000001:
print "In iter %d/%d global score difference with previous iteration is smaller than 0.000001, see: %f \nExiting program." % (
i + 1, options.iter, difscore)
sys.exit()
try:
diferror = math.fabs(
float(itermeanerrors[i]) - float(itermeanerrors[i - 1]))
if float(difscore) < 0.000001:
print "In iter %d/%d global error difference with previous iteration is smaller than 0.000001, see: %f \nExiting program." % (
i + 1, options.iter, diferror)
sys.exit()
except:
pass
diffscarea = math.fabs(
float(itermeanfscs[i]) - float(itermeanfscs[i - 1]))
if float(difscore) < 0.000001:
print "In iter %d/%d global fscarea difference with previous iteration is smaller than 0.000001, see: %f \nExiting program." % (
i + 1, options.iter, diffscarea)
sys.exit()
difscore3d = math.fabs(
float(itermeanscores3d[i]) - float(itermeanscores3d[i - 1]))
if float(difscore3d) < 0.000001:
print "In iter %d/%d global score3dipetResult difference with previous iteration is smaller than 0.000001, see: %f \nExiting program." % (
i + 1, options.iter, difscore)
sys.exit()
E2end(logger)
return ()
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog <output> [options]
This program is used to preprocess subtomograms before aligning them. The same can be accomplished with
e2proc3d, except that this program is parallelized and thus should be substantially faster for large subtomograms.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--input",
type=str,
default='',
help=
"""Default=None. The name of the input volume stack. MUST be HDF since volume stack support is required."""
)
parser.add_argument(
"--output",
type=str,
default='',
help=
"""Default=None. Specific name of HDF file to write processed particles to."""
)
parser.add_argument(
"--parallel",
type=str,
default='',
help=
"""default=None. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel"""
)
parser.add_argument(
"--ppid",
type=int,
help=
"""Default=-1. Set the PID of the parent process, used for cross platform PPID""",
default=-1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"""Default=0. Verbose level [0-9], higner number means higher level of verboseness"""
)
parser.add_argument(
"--subset",
type=int,
default=0,
help=
"""Default=0 (not used). Refine only this substet of particles from the stack provided through --input"""
)
parser.add_argument(
"--apix",
type=float,
default=0.0,
help=
"""Default=0.0 (not used). Use this apix value where relevant instead of whatever is in the header of the reference and the particles. Will overwrite particle header as well."""
)
parser.add_argument(
"--shrink",
type=int,
default=0,
help=
"""Default=0 (no shrinking). Optionally shrink the input volumes by an integer amount for coarse alignment."""
)
parser.add_argument(
"--threshold",
type=str,
default='',
help=
"""Default=None. A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score."""
)
parser.add_argument(
"--mask",
type=str,
default='',
help=
"""Default=None. Masking processor applied to particles before alignment. IF using --clip, make sure to express outer mask radii as negative pixels from the edge."""
)
parser.add_argument(
"--maskfile",
type=str,
default='',
help=
"""Default=None. Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None."""
)
parser.add_argument(
"--normproc",
type=str,
default='',
help=
"""Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to particles before alignment. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'"""
)
parser.add_argument(
"--preprocess",
type=str,
default='',
help=
"""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--lowpass",
type=str,
default='',
help=
"""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--highpass",
type=str,
default='',
help=
"""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--clip",
type=int,
default=0,
help=
"""Default=0 (which means it's not used). Boxsize to clip particles. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary."""
)
parser.add_argument(
"--nopath",
action='store_true',
default=False,
help=
"""If supplied, this option will save results in the directory where the command is run. A directory to store the results will not be made."""
)
parser.add_argument(
"--path",
type=str,
default='sptpreproc',
help=
"""Default=spt. Directory to store results in. The default is a numbered series of directories containing the prefix 'sptpreproc'; for example, sptpreproc_02 will be the directory by default if 'sptpreproc_01' already exists."""
)
(options, args) = parser.parse_args()
logger = E2init(sys.argv, options.ppid)
print "\n(e2spt_preproc)(main) started log"
from e2spt_classaverage import sptmakepath
if options.path and not options.nopath:
options = sptmakepath(options, 'sptpreproc')
if options.parallel == 'None' or options.parallel == 'none':
options.parallel = None
if not options.input:
try:
options.input = sys.argv[1]
except:
print "\n(e2spt_preproc)(main) ERROR: invalid input file"
if options.mask or options.maskfile or options.threshold or options.clip or options.threshold or options.normproc or options.preprocess or options.lowpass or options.highpass or int(
options.shrink) > 1:
preprocstack = str(
os.path.basename(options.input).replace('.hdf', '_preproc.hdf'))
if options.path and not options.nopath:
preprocstack = options.path + '/' + preprocstack
if options.output:
if '.hdf' in options.output[-4:]:
preprocstack = options.output
else:
print "\n(e2spt_preproc)(main) ERROR: '.hdf' must be the last four characters of the output filename."
print "\n(e2spt_preproc)(main) output stack will be %s" % (
preprocstack)
n = 0
try:
n = EMUtil.get_image_count(options.input)
except:
print "\n(e2spt_preproc)(main) ERROR: --input stack seems to be invalid"
sys.exit()
print "\n(e2spt_preproc)(main) number of particles is %d" % (n)
c = os.getcwd()
findir = os.listdir(c)
if preprocstack not in findir:
dimg = EMData(8, 8, 8)
dimg.to_one()
for i in range(n):
dimg.write_image(preprocstack, i)
else:
print "\n(e2spt_preproc)(main) WARNING: a file with the name of the output stack %s is already in the current directory and will be DELETED" % (
preprocstack)
os.remove(preprocstack)
dimg = EMData(8, 8, 8)
dimg.to_one()
for i in range(n):
dimg.write_image(preprocstack, i)
finalbox = EMData(options.input, 0, True)['nx']
if options.clip:
finalbox = options.clip
#dimglarge=EMData(finalbox,finalbox,finalbox)
#dimglarge.to_one()
#dimglarge.write_image(preprocstack,0)
#n=EMUtil.get_image_count(options.input)
#if options.subset:
# n=options.subset
#dimglarge.write_image(preprocstack,n-1)
if options.verbose:
print "\n(e2spt_preproc)(main) wrote dummy ptcls to %s" % (
preprocstack)
print "\n(e2spt_preproc)(main) - INITIALIZING PARALLELISM!\n"
if options.parallel:
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel)
pclist = [options.input]
etc.precache(pclist)
print "\n(e2spt_preproc)(main) - precaching --input"
tasks = []
results = []
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser(options)
for j in range(n):
#print "processing particle", j
img = EMData(options.input, j)
if options.parallel:
#task = Preproc3DTask( ["cache",options.input,j], options, j, preprocstack )
task = Preproc3DTask(img, options, j, preprocstack)
tasks.append(task)
else:
img = EMData(options.input, j)
pimg = preprocfunc(img, options, j, preprocstack)
if options.parallel and tasks:
tids = etc.send_tasks(tasks)
if options.verbose:
print "\n(e2spt_preproc)(main) preprocessing %d tasks queued" % (
len(tids))
results = get_results(etc, tids, options)
#print "\n(e2spt_preproc)(main) preprocessing results are", results
#print "\n(e2spt_preproc)(main) input changing to preprocstack"
#options.input = preprocstack
#cache needs to be reloaded with the new options.input
else:
print "\n(e2spt_preproc)(main) Nothing to do. No preprocessing parameters specified."
E2end(logger)
return
def main():
#import pylab
#import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
progname = os.path.basename(sys.argv[0])
usage = """Produces mean intensity histograms of stack of sub-volumes"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--input",
type=str,
default='',
help=
"""Default=None. Comma-separated stacks of images whose mean intensity distribution you want to plot."""
)
parser.add_argument(
"--subset",
type=int,
default=0,
help=
"""Default=0 (not used). N > 2 number of particles to from each stack provided through --input to consider."""
)
parser.add_argument(
"--path",
type=str,
default='',
help=
"Directory to store results in. The default is a numbered series of directories containing the prefix 'sptsim'; for example, sptsim_02 will be the directory by default if 'sptsim_01' already exists."
)
#parser.add_argument("--output",type=str,default='',help="""Name of output plot if comparing two populations or more.""")
parser.add_argument(
"--shrink",
type=int,
default=1,
help=
"Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount n > 1."
)
parser.add_argument(
"--bins",
type=int,
default=0,
help=
"""Default=0 (not used). Number of bins for histogram. If not provided, the optimal bin number will be automatically calculated based on bin-width, computed using Scott's normal reference rule, width = (3.5*std)/cuberoot(n), where 'std' is the standard deviation of the mean intensity distribution of population and n is the number of mean intensity values considered (this is affected by --removesigma). Then, bins will be nbins = (max(intensities) - min(intensities)) / width."""
)
#parser.add_argument("--sym", type=str, default='c1', help = "Symmetry to enforce before computing mean intensity in the box. Note that this should only be used if the particles are properly aligned to the symmetry axis.")
parser.add_argument(
"--mask",
type=str,
default="mask.sharp:outer_radius=-2",
help=
"Default=mask.sharp:outer_radius=-2. Mask processor applied to the particles before alignment. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py)."
)
parser.add_argument(
"--maskfile",
type=str,
default='',
help=
"""Default=None. An image file containing an additional mask to apply besides --mask."""
)
parser.add_argument(
"--clip",
type=int,
default=0,
help=
"""Default=0 (not used). Boxsize to clip particles to before computing mean and standard deviation values for each image. (This can act as a mask, as you'd want to clip the boxes to a smaller size than their current, original size, excluding neighboring particles and background pixels/voxels)."""
)
parser.add_argument(
"--preprocess",
type=str,
default='',
help=
"""Any processor to be applied to each image before computing mean and standard deviation values. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py)."""
)
parser.add_argument(
"--lowpass",
type=str,
default='',
help=
"""Default=None. A lowpass filtering processor to be applied before computing mean and standard deviation values for each image. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py)."""
)
parser.add_argument(
"--highpass",
type=str,
default='',
help=
"""Default=None. A highpass filtering processor to be applied before computing mean and standard deviation values for each image. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py)."""
)
parser.add_argument(
"--threshold",
type=str,
default='',
help=
"""A thresholding processor to be applied before computing mean and standard deviation values for each image. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py)."""
)
parser.add_argument(
"--normproc",
type=str,
default="normalize.edgemean",
help=
"""Default=normalize.edgemean. Normalization processor applied to particles before computing mean and standard deviation values for each iamge. If normalize.mask is used, --mask will be passed in automatically. If you want to turn normalization off specify \'None\'. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py)."""
)
parser.add_argument(
"--savepreprocessed",
action="store_true",
default=False,
help=
"""Default=False. If provided, this option will save the image stacks in --input after all preprocessing options (lowpass, highpass, preprocess, masking, etc.) have been applied."""
)
parser.add_argument(
"--normalizeplot",
action="store_true",
default=False,
help=
"""Default=False. This will normalize the intensity values of the distribution to be between 0 and 1"""
)
parser.add_argument(
"--removesigma",
type=int,
default=0,
help=
"""Default=0. Provide a value for the number of standard deviations away from the mean to consider values to exclude. For example, if --removesigma=3, values further than 3 standard deviations away from the mean will be excluded."""
)
parser.add_argument(
"--ppid",
type=int,
help=
"Default=1. Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument(
"--verbose",
"-v",
type=int,
default=0,
help=
"Default 0. Verbose level [0-9], higner number means higher level of verboseness",
dest="verbose",
action="store",
metavar="n")
(options, args) = parser.parse_args()
logger = E2init(sys.argv, options.ppid)
'''
if options.mask:
options.mask=parsemodopt(options.mask)
if options.preprocess:
options.preprocess=parsemodopt(options.preprocess)
if options.lowpass:
options.lowpass=parsemodopt(options.lowpass)
if options.highpass:
options.highpass=parsemodopt(options.highpass)
if options.threshold:
options.threshold=parsemodopt(options.threshold)
if options.normproc:
options.normproc=parsemodopt(options.normproc)
'''
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser(options)
datafiles = options.input.split(',')
from e2spt_classaverage import sptmakepath
options = sptmakepath(options, 'meanintensityplots')
intensitiesSeveral = []
iwzSeveral = []
iminsSeveral = []
imaxsSeveral = []
istdsSeveral = []
means = []
stds = []
from e2spt_classaverage import writeParameters
cmdwp = writeParameters(options, 'e2spt_meanintensityplot.py',
'sptmeanintensity')
for datafile in datafiles:
n = EMUtil.get_image_count(datafile)
if options.subset:
if options.subset < 3:
print "ERROR:Subset must be > 2."
sys.exit(1)
n = options.subset
if n < 3:
print "ERROR: All stacks must have at least 3 particles in them. This one doesn't:", datafile
sys.exit(1)
for datafile in datafiles:
ret = calcintensities(options, datafile)
intensitiesSingle = ret[0]
iwz = ret[1]
imins = ret[2]
imaxs = ret[3]
istds = ret[4]
intensitiesSeveral.append([datafile, list(intensitiesSingle)])
iwzSeveral.append([datafile, list(iwz)])
iminsSeveral.append([datafile, list(imins)])
imaxsSeveral.append([datafile, list(imaxs)])
istdsSeveral.append([datafile, list(istds)])
intensitiesSingleNorm = intensitiesSingle
if options.normalizeplot:
intensitiesSingleNorm = normintensities(intensitiesSingle, 0, 0)
#print "\]n\\n\nIntensities before plotting are", intensitiesSingleNorm
#print "\n\n\n\n\n"
ret = plotintensities(intensitiesSingleNorm, options, datafile)
mean = ret[0]
std = ret[1]
means.append(mean)
stds.append(std)
ret = plotintensities(iwz, options, datafile, 'wz')
ret = plotintensities(imins, options, datafile, 'mins')
ret = plotintensities(imaxs, options, datafile, 'maxs')
ret = plotintensities(istds, options, datafile, 'stds')
#print "\nIntensities several len is", len( intensitiesSeveral )
if len(intensitiesSeveral) > 1:
datafile1 = intensitiesSeveral[0][0]
datafile2 = intensitiesSeveral[1][0]
intensities1 = intensitiesSeveral[0][1]
intensities2 = intensitiesSeveral[1][1]
n1 = len(intensities1)
n2 = len(intensities2)
zscore = (means[0] - means[1]) / np.sqrt((stds[0] * stds[0]) / n1 +
(stds[1] * stds[1]) / n2)
g = open(options.path + '/MIboth_INFO.txt', 'w')
zscoreline = 'zscore=' + str(
zscore) + ' for ' + datafile1 + ' vs ' + datafile2 + ' \n'
lines = [zscoreline]
g.writelines(lines)
g.close()
print "\nzzzzzzz\n%s" % (zscoreline)
absmax = absmin = 0
if options.normalizeplot:
minses = []
maxes = []
for intenS in intensitiesSeveral:
minS = float(min(intenS[1]))
maxS = float(max(intenS[1]))
minses.append(minS)
maxes.append(maxS)
absmin = min(minses)
absmax = max(maxes) - absmin
for intensities in intensitiesSeveral:
print "Type and len of intensities is", type(intensities[1]), len(
intensities[1])
intensitiesNorm = intensities[1]
if options.normalizeplot:
print "Normalizeplot on"
intensitiesNorm = normintensities(intensities[1], absmin,
absmax)
plotintensities(intensitiesNorm, options, datafile, 'no')
plt.savefig(options.path + '/MIbothPlot.png')
plt.clf()
E2end(logger)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options]
This program takes aligned stacks produced by e2spt_classaverage.py and raw tomograms
to recompute averages with different options (such as extracting recentered particles,
and possibly with a larger or smaller boxisze).
All the necessary files (aligned stacks in HDF format and raw tomograms in MRC format
ending in .rec, as produced by IMOD) should be in the running directory.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--stacks",
default='',
type=str,
help="""Comma separated list of HDF image stacks to process.""")
parser.add_argument(
"--tomograms",
default='',
type=str,
help=
"""Comma separated list of tomograms with REC extension from which all particles in --stacks came from."""
)
#parser.add_argument("--output", default="avg.hdf",type=str, help="The name of the output average volume.")
#parser.add_argument("--rotationtype", default="eman",type=str, help="Valid options are: eman,imagic,mrc,spider,quaternion,sgirot,spi,xyz")
#parser.add_argument("--averager",type=str,help="The type of averager used to produce the class average. Default=mean",default="mean")
parser.add_argument(
"--sym",
type=str,
default='',
help=
"Symmetry to impose - choices are: c<n>, d<n>, h<n>, tet, oct, icos")
parser.add_argument(
"--path",
default='',
type=str,
help="Name of directory where to save the output file.")
#parser.add_argument("--alifile",default='',type=str,help=".json file with alingment parameters, if raw stack supplied via --input.")
parser.add_argument(
"--boxsize",
"-B",
type=int,
default=0,
help=
"""Default=0 (option not used). Provide a value for the boxsize of the output average in pixels. If not provided, the boxsize of --stacks will be used."""
)
parser.add_argument(
"--normproc",
type=str,
default='normalize.edgemean',
help=
"""Normalization processor applied to particles before extraction. Default=normalize.edgemean. If using the latter, you must provide --masknorm, otherwise, a default --masknorm=mask.sharp:outer_radius=-2 will be used."""
)
parser.add_argument(
"--threshold",
type=str,
help=
"""Threshold processor to apply to particles before writing them out to get rid of too high and/or too low pixel values.""",
default='')
parser.add_argument(
"--usetomograms",
action='store_true',
default=False,
help=""""Re-extract particles from the original tomogram.""")
parser.add_argument(
"--useinverseali",
action='store_true',
default=False,
help=
""""Use the inverse of the value stored in xform.align3d in the header of each particle."""
)
parser.add_argument(
'--shrink',
type=int,
default=1,
help=
"""Shrink factor to shrink particles before averaging. Default=1, which means no shrinking."""
)
parser.add_argument(
"--lowpass",
type=str,
help=
"""Lowpass filtering processor to apply to particles before averaging. Default=None.""",
default='')
parser.add_argument(
"--preprocess",
type=str,
help=
"""A processor (as in e2proc3d.py) to be applied to the tomogram before opening it. \nFor example, a specific filter with specific parameters you might like. \nType 'e2proc3d.py --processors' at the commandline to see a list of the available processors and their usage""",
default=None)
parser.add_argument(
'--invert',
action="store_true",
default=False,
help=
'''Default=False. This parameer indicates you want the contrast to me inverted while boxing, AND for the extracted sub-volumes. Remember that EMAN2 **MUST** work with "white" protein. You can very easily figure out what the original color\nof the protein is in your data by looking at the gold fiducials or the edge of the carbon hole in your tomogram. If they look black you MUST specify this option'''
)
parser.add_argument(
"--averager",
type=str,
help=
"""The type of averager used to produce the class average. Default=mean.tomo""",
default='mean.tomo')
parser.add_argument(
"--keep",
type=float,
help="""The fraction of particles to keep in each class. Default=1.0""",
default=1.0)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness."
)
#parser.add_argument("--saveali",action="store_true", default=False,help="""If set, will save the
# aligned particle volumes in class_ptcl.hdf. Overwrites existing file.""")
(options, args) = parser.parse_args()
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser(options)
'''
if options.normproc:
options.normproc=parsemodopt(options.normproc)
if options.threshold:
options.threshold=parsemodopt(options.threshold)
if options.lowpass:
options.lowpass=parsemodopt(options.lowpass)
if options.preprocess:
options.preprocess=parsemodopt(options.preprocess)
if options.averager:
options.averager=parsemodopt(options.averager)
'''
#if not options.boxsize:
# print "\n(e2spt_recompute.py) (main) ERROR: Boxsize must be greater than zero. It is:", options.boxsize
# sys.exit()
logid = E2init(sys.argv, options.ppid)
from e2spt_classaverage import sptmakepath
options = sptmakepath(options, 'sptrecompute')
c = os.getcwd() #Get current directory
findir = os.listdir(c) #Get list of files in current directory
stacks = set([]) #Make list set to store particle stacks
if options.stacks:
stacks = set(options.stacks.split(','))
if not options.boxsize:
boxesdict = {}
boxes = []
for sta in stacks:
box = EMData(sta, 0)['nx']
boxesdict.update({sta: box})
boxes.append(box)
boxes = set(boxes)
if len(boxes) > 1:
print "ERROR: Your stacks are not all the same boxsize. There are %d many different boxsizes" % (
len(boxes))
print "which are", boxes
print "all input stacks in --stacks must have the same boxsize; otherwise, specify a unique boxsize through --boxsize."
sys.exit()
else:
for f in findir:
if '.hdf' in f:
stacks.add(f)
tomograms = set([])
tomogramsbases = set([])
if options.usetomograms: #Make list set to store tomograms
if options.tomograms:
tomograms = set(options.tomograms.split(','))
for tom in tomograms:
tombase = os.path.basename(tom)
tomogramsbases.append(tombase)
else:
for f in findir:
if '.rec' in f:
tomograms.add(f)
for stack in stacks: #Loop over stacks
n = EMUtil.get_image_count(
stack) #Determine number of particles in stack
avgr = Averagers.get(options.averager[0],
options.averager[1]) #initialize averager
print "\n(e2spt_recompute.py) (main) Processing stack", stack
for i in range(n): #Loop over particles in stack
hdr = EMData(stack, i,
True) #Load particle header by providing 'True' flag
a = None
box = hdr['nx']
if options.boxsize:
box = options.boxsize
if options.usetomograms and tomograms:
tomogram = hdr[
'ptcl_source_image'] #Determine what tomogram a particle comes from
if tomogram not in tomogramsbases and tomogram not in tomograms:
print "\n(e2spt_recompute.py) (main) ERROR: Tomogram %s not found" % (
tomogram)
sys.exit()
print "\n(e2spt_recompute.py) (main) Processing particle %d in stack %s which should come from tomogram %s" % (
i, stack, tomogram)
coords = hdr[
'ptcl_source_coord'] #And from what location exactly
x = coords[0] #Parse coordinates
y = coords[1]
z = coords[2]
r = Region((2 * x - box) / 2, (2 * y - box) / 2,
(2 * z - box) / 2, box, box,
box) #Define extraction region based on boxsize
a = EMData()
a.read_image(tomogram, 0, False,
r) #Actually 'read'/extract particle data
#Preprocess as needed
else:
a = EMData(stack, i)
if a:
if options.normproc:
a.process_inplace(options.normproc[0], options.normproc[1])
if options.invert:
a.mult(-1)
t = None
try:
t = hdr[
'xform.align3d'] #Rotate particle into aligned orientation
except:
print "WARNING: 'xform.align3d' not found in header of particle %d" % (
i)
try:
t = hdr['sptsim_randT']
except:
print "ERROR: 'sptsim_randT also not found in header or particle %d" % (
i)
if t:
tf = t
if options.useinverseali:
tf = t.inverse()
print "t is", t
print "and its inverse is", tf
#print "Applied this transform",tf
a.transform(tf)
if options.threshold:
a.process_inplace(options.threshold[0],
options.threshold[1])
if options.lowpass:
a.process_inplace(options.lowpass[0],
options.lowpass[1])
if options.preprocess:
a.process_inplace(options.preprocess[0],
options.preprocess[1])
if options.shrink and int(options.shrink) > 1:
shrinkfactor = options.shrink
a.process_inplace('math.meanshrink',
{'n': shrinkfactor})
#a['origin_x'] = 0
#a['origin_y'] = 0
#a['origin_z'] = 0
#a['ptcl_source_image'] = hdr['ptcl_source_image']
#a['ptcl_source_coord'] = hdr['ptcl_source_coord']
avgr.add_image(a)
else:
print "skipping particle", i
avg = avgr.finish()
avg.process_inplace('normalize')
if options.sym and not breaksym:
avg.process_inplace('xform.applysym', {'sym': options.sym})
outname = options.path + '/' + stack.replace('.hdf', '_AVG.hdf')
avg.write_image(outname, 0)
E2end(logid)
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options]
This program aligns a paricle to its symmetry axis. There are two algorithmic modes.
A coarse search followed by simplex minimization (not yet implimented) OR monte carlo course
search followed by simplex minimization. The Goal is to align the paricle to its
symmetry axis so symmetry can be applied for avergaing and for alignment speed up
(it is only necessary to search over one asymmetric unit!
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_header(
name="symsearch3dheader",
help="""Options below this label are specific to e2symsearch3d""",
title="### e2symsearch3d options ###",
row=3,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--input",
dest="input",
default='',
type=str,
help="""The name of input volume or hdf stack of volumes""",
guitype='filebox',
browser="EMBrowserWidget(withmodal=True,multiselect=False)",
row=0,
col=0,
rowspan=1,
colspan=2)
#parser.add_argument("--output", dest="output", default="""e2symsearch3d_OUTPUT.hdf""", type=str, help="The name of the output volume", guitype='strbox', filecheck=False, row=1, col=0, rowspan=1, colspan=2)
parser.add_argument(
"--ref",
type=str,
default='',
help=
"""Default=None. If provided and --average is also provided and --keep < 1.0 or --keepsig is specified, 'good particles' will be determined by correlation to --ref."""
)
parser.add_argument(
"--mirror",
type=str,
default='',
help=
"""Axis across of which to generate a mirrored copy of --ref. All particles will be compared to it in addition to the unmirrored image in --ref if --keepsig is provided or if --keep < 1.0."""
)
parser.add_argument("--path",
type=str,
default='',
help="""Name of path for output file""",
guitype='strbox',
row=2,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--plots",
action='store_true',
default=False,
help=
"""Default=False. Turn this option on to generate a plot of the ccc scores if --average is supplied. Running on a cluster or via ssh remotely might not support plotting."""
)
parser.add_argument(
"--sym",
dest="sym",
default="c1",
help=
"""Specify symmetry -choices are: c<n>, d<n>, h<n>, tet, oct, icos. For asymmetric reconstruction ommit this option or specify c1.""",
guitype='symbox',
row=4,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--shrink",
dest="shrink",
type=int,
default=0,
help=
"""Optionally shrink the input particles by an integer amount prior to computing similarity scores. For speed purposes. Default=0, no shrinking""",
guitype='shrinkbox',
row=5,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--mask",
type=str,
help=
"""Mask processor applied to particles before alignment. Default is mask.sharp:outer_radius=-2. IF using --clipali, make sure to express outer mask radii as negative pixels from the edge.""",
returnNone=True,
default="mask.sharp:outer_radius=-2",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'mask\')',
row=11,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--maskfile",
type=str,
default='',
help=
"""Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None."""
)
parser.add_argument(
"--normproc",
type=str,
default='',
help=
"""Normalization processor applied to particles before alignment. Default is to use normalize. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'"""
)
parser.add_argument(
"--threshold",
default='',
type=str,
help=
"""A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=10,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--preprocess",
default='',
type=str,
help=
"""Any processor (as in e2proc3d.py) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=10,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--lowpass",
type=str,
default='',
help=
"""A lowpass filtering processor (from e2proc3d.py; see e2help.py processors) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=17,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--highpass",
type=str,
default='',
help=
"""A highpass filtering processor (from e2proc3d.py, see e2help.py processors) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=18,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--clipali",
type=int,
default=0,
help=
"""Boxsize to clip particles as part of preprocessing to speed up alignment. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary; still, there are some benefits from 'oversampling' the data during averaging; so you might still want an average of size 2x, but perhaps particles in a box of 1.5x are sufficiently good for alignment. In this case, you would supply --clipali=75"""
)
parser.add_argument(
"--savepreprocessed",
action="store_true",
default=False,
help=
"""Default=False. Will save stacks of preprocessed particles (one for coarse alignment and one for fine alignment if preprocessing options are different)."""
)
parser.add_argument(
"--average",
action='store_true',
default=False,
help=
"""Default=False. If supplied and a stack is provided through --input, the average of the aligned and/or symmetrized stack will also be saved."""
)
parser.add_argument(
"--averager",
type=str,
default="mean.tomo",
help=
"""Default=mean.tomo. The type of averager used to produce the class average. Default=mean.tomo."""
)
parser.add_argument(
"--keep",
type=float,
default=1.0,
help=
"""Fraction of particles to include if --average is on, after correlating the particles with the average."""
)
parser.add_argument(
"--keepsig",
action="store_true",
default=False,
help=
"""Default=False. Causes theoptions.keep argument to be interpreted in standard deviations.""",
guitype='boolbox',
row=6,
col=1,
rowspan=1,
colspan=1,
mode='alignment,breaksym')
parser.add_argument(
"--avgiter",
type=int,
default=1,
help=
"""Default=1. If --keep is different from 1.0 and --average is on, the initial average will include all the particles, but then the percent specified byoptions.keep will be kept (the rest thrown away) and a new average will be computed. If --avgiter > 1, this new average will be compared again against all the particles. The procedure will be repeated for however many iterations --avgiter is given, or the process will stop automatically if in two consecutive rounds exactly the same particles are kept"""
)
parser.add_argument(
'--subset',
type=int,
default=0,
help=
"""Number of particles in a subset of particles from the --input stack of particles to run the alignments on."""
)
parser.add_argument("--steps",
dest="steps",
type=int,
default=10,
help="""Number of steps (for the MC). Default=10.""",
guitype='intbox',
row=5,
col=1,
rowspan=1,
colspan=1)
parser.add_argument("--symmetrize",
default=False,
action="store_true",
help="""Symmetrize volume after alignment.""",
guitype='boolbox',
row=6,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--cmp",
type=str,
help=
"""The name of a 'cmp' to be used in comparing the symmtrized object to unsymmetrized""",
default="ccc",
guitype='comboparambox',
choicelist='re_filter_list(dump_cmps_list(),\'tomo\', True)',
row=7,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--parallel",
"-P",
type=str,
help=
"""Run in parallel, specify type:<option>=<value>:<option>:<value>""",
default=None,
guitype='strbox',
row=8,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--ppid",
type=int,
help=
"""Set the PID of the parent process, used for cross platform PPID.""",
default=-1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"""verbose level [0-9], higner number means higher level ofoptions.verboseness."""
)
parser.add_argument(
"--nopath",
action='store_true',
default=False,
help=
"""If supplied, this option will save results in the directory where the command is run. A directory to store the results will not be made."""
)
parser.add_argument(
"--nolog",
action='store_true',
default=False,
help=
"""If supplied, this option will prevent logging the command run in .eman2log.txt."""
)
parser.add_argument(
"--saveali",
action='store_true',
default=False,
help="""Save the stack of aligned/symmetrized particles.""")
parser.add_argument(
"--savesteps",
action='store_true',
default=False,
help=
"""If --avgiter > 1, save all intermediate averages and intermediate aligned kept stacks."""
)
parser.add_argument(
"--notmatchimgs",
action='store_true',
default=False,
help=
"""Default=True. This option prevents applying filter.match.to to one image so that it matches the other's spectral profile during preprocessing for alignment purposes."""
)
parser.add_argument(
"--preavgproc1",
type=str,
default='',
help=
"""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)"""
)
parser.add_argument(
"--preavgproc2",
type=str,
default='',
help=
"""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)"""
)
parser.add_argument(
"--weighbytiltaxis",
type=str,
default='',
help=
"""Default=None. A,B, where A is an integer number and B a decimal. A represents the location of the tilt axis in the tomogram in pixels (eg.g, for a 4096x4096xZ tomogram, this value should be 2048), and B is the weight of the particles furthest from the tomogram. For example, --weighbytiltaxis=2048,0.5 means that praticles at the tilt axis (with an x coordinate of 2048) will have a weight of 1.0 during averaging, while the distance in the x coordinates of particles not-on the tilt axis will be used to weigh their contribution to the average, with particles at the edge(0+radius or 4096-radius) weighing 0.5, as specified by the value provided for B."""
)
parser.add_argument(
"--weighbyscore",
action='store_true',
default=False,
help=
"""Default=False. This option will weigh the contribution of each subtomogram to the average by score/bestscore."""
)
parser.add_argument(
"--align",
type=str,
default='symalignquat',
help=
"""Default=symalignquat. WARNING: The aligner cannot be changed for this program currently. Option ignored."""
)
parser.add_argument(
"--tweak",
action='store_true',
default=False,
help=
"""WARNING: Not used for anything yet. This will perform a final alignment with no downsampling [without using --shrink or --shrinkfine] if --shrinkfine > 1."""
)
(options, args) = parser.parse_args()
if not options.input:
parser.print_help()
sys.exit(0)
#If no failures up until now, initialize logger
log = 0
if not options.nolog:
logid = E2init(sys.argv, options.ppid)
log = 1
#inimodeldir = os.path.join(".",options.path)
#if not os.access(inimodeldir, os.R_OK):
# os.mkdir(options.path)
#Make directory to save results
from e2spt_classaverage import sptmakepath
options = sptmakepath(options, 'symsearch')
if options.nopath:
options.path = '.'
rootpath = os.getcwd()
if rootpath not in options.path:
options.path = rootpath + '/' + options.path
from e2spt_classaverage import preprocessing
from EMAN2PAR import EMTaskCustomer
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser(options)
avgr = Averagers.get(options.averager[0], options.averager[1])
results = {}
scores = []
outputstack = options.path + '/all_ptcls_ali.hdf'
#Determine number of particles in the stack
n = EMUtil.get_image_count(options.input)
if options.subset and options.subset < n:
n = options.subset
for i in range(n):
print "\nI'll look for symmetry in particle number", i
#Load particle and make a copy to modify if preprocessing options are specified
volume = EMData(options.input, i)
preprocvol = volume.copy()
#Preprocess volume if any preprocessing options are specified
if (
options.shrink and options.shrink > 1
) or options.mask or options.maskfile or options.lowpass or options.highpass or options.normproc or options.preprocess or options.threshold or options.clipali:
print "\nHowever, I will first preprocess particle number", i
print "\nWill call preprocessing on ptcl", i
preprocvol = preprocessing(preprocvol, options, i)
#preprocessing(s2image,options, ptclindx, savetagp ,'no',round)
print "\nDone preprocessing on ptcl", i
if options.parallel:
etc = EMTaskCustomer(options.parallel)
else:
etc = EMTaskCustomer("thread:1")
symalgorithm = SymALignStrategy(preprocvol, options.sym, options.steps,
options.cmp, etc)
ret = symalgorithm.execute()
symxform = ret[0]
score = ret[1]
scores.append(score)
results.update({score: [symxform, i]})
print "\nWriting output for best alignment found for particle number", i
if options.shrink and options.shrink > 1:
trans = symxform.get_trans()
symxform.set_trans(trans[0] * options.shrink,
trans[1] * options.shrink,
trans[2] * options.shrink)
print "\nWrittng to output ptcl", i
#Rotate volume to the best orientation found, set the orientation in the header, apply symmetry if specified and write out the aligned (and symmetrized) particle to the output stack
output = volume.process('xform', {'transform': symxform})
output.set_attr('symxform', symxform)
print "\nApplying this transform to particle", symxform
if options.symmetrize:
output = output.process('xform.applysym', {'sym': options.sym})
output['spt_score'] = score
output.write_image(outputstack, -1)
#Averaging here only makes sense if all particles are going to be kept. Otherwise, different code is needed (below)
if options.average and options.keep == 1.0 and not options.keepsig:
avgr.add_image(output)
#Finalize average of all particles if non were set to be excluded. Otherwise, determine the discrimination threshold and then average the particles that pass it.
if options.average:
final_avg = avgr.finish()
final_avg['origin_x'] = 0
final_avg[
'origin_y'] = 0 #The origin needs to be reset to ZERO to avoid display issues in Chimera
final_avg['origin_z'] = 0
final_avg['xform.align3d'] = Transform()
if options.keep == 1.0 and not options.keepsig:
final_avg.write_image(options.path + '/final_avg.hdf', 0)
if options.avgiter > 1:
print """WARNING: --avgiter > 1 must be accompanied by --keepsig, or by --keep < 1.0"""
elif options.keep < 1.0 or options.keepsig:
if options.ref:
ref = EMData(options.ref, 0)
refComp(options, outputstack, ref, results, '')
if options.mirror:
ref.process_inplace('xform.mirror',
{'axis': options.mirror})
refComp(options, outputstack, ref, results, '_vs_mirror')
else:
ref2compare = final_avg
refComp(options, outputstack, final_avg, results, '')
if log:
E2end(logid)
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """Program for individual subtomogram refinement (ISR) based on subtiltseries
for each subtomogram extracted with e2spt_subtilt.py."""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--input",
type=str,
default="",
help="""Comma separated files in .ali, .st .hdf format of the aligned subtiltseries.""",
)
parser.add_argument(
"--inputstem",
type=str,
default="",
help="""Alternative to supplying --input. This is a string common to multiple files to be processed in the CURERENT directory. The common string doesn't need to be at a particular location in the filenames. For example, a series of files "tiltA.hdf, tiltB.hdf, tiltC.hdf" could have either 'hdf', '.hdf', 't,','ti', 'til', 'tilt', etc., as a common string. The key is to choose a string shared ONLY by the files of interest. The files should be multiple subtiltseries in .hdf format; each file should correspond to an individual subtiltseries for a different particle: That is, each file should be a subtiltseries corresponding to an individual subtomogram, as extracted by e2spt_subtilt.py, or as simulated by e2spt_simulation.py""",
)
parser.add_argument(
"--inputdir",
type=str,
default="",
help="""Alternative to --input and --inputstem. Path to a directory containing individual subtiltseries stacks.""",
)
parser.add_argument(
"--path",
type=str,
default="",
help="Directory to store results in. The default is a numbered series of directories containing the prefix 'sptisr'; for example, sptisr02 will be the directory by default if 'sptisr_01' already exists.",
)
parser.add_argument(
"--ppid",
type=int,
help="Default=1. Set the PID of the parent process, used for cross platform PPID",
default=-1,
)
parser.add_argument(
"--verbose",
"-v",
type=int,
default=0,
help="Default 0. Verbose level [0-9], higner number means higher level of verboseness",
dest="verbose",
action="store",
metavar="n",
)
parser.add_argument(
"--reconstructor",
type=str,
default="fourier:mode=gauss_2",
help="""Default=fourier:mode=gauss_2. The reconstructor to use to reconstruct the tilt series into a tomogram. Type 'e2help.py reconstructors' at the command line to see all options and parameters available. To specify the interpolation scheme for the fourier reconstructor, specify 'mode'. Options are 'nearest_neighbor', 'gauss_2', 'gauss_3', 'gauss_5'. For example --reconstructor=fourier:mode=gauss_5 """,
)
parser.add_argument("--iter", type=int, default=1, help="""Number of iterations to run algorithm for.""")
parser.add_argument(
"--tltfile",
type=str,
default="",
help="""IMOD-like .tlt file with tilt angles for the aligned tiltseries (or set of subtiltseries).""",
)
parser.add_argument(
"--tiltaxis",
type=str,
default="y",
help="""Axis to produce projections about. Default is 'y'; the only other valid option is 'x'.""",
)
parser.add_argument(
"--pad2d",
type=float,
default=0.0,
help="""Default=0.0. Padding factor (e.g., 2.0, to make the box twice as big) to zero-pad the 2d images in the tilt series for reconstruction purposes (the final reconstructed subvolumes will be cropped back to the original size though).""",
)
parser.add_argument(
"--pad3d",
type=float,
default=0.0,
help="""Default=0.0. Padding factor (e.g., 2.0, to make the box twice as big) to zero-pad the volumes for reconstruction purposes (the final reconstructed subvolumes will be cropped back to the original size though).""",
)
parser.add_argument(
"--savevols",
action="store_true",
default=False,
help="""This option will save the reconstructed volumes at each iteration.""",
)
parser.add_argument(
"--outxsize",
type=int,
default=0,
help="""Clip the output volume in x to this size. The default size is the nx size of the input images.""",
)
parser.add_argument(
"--outysize",
type=int,
default=0,
help="""Clip the output volume in y to this size. The default size is the ny size of the input images.""",
)
parser.add_argument(
"--outzsize",
type=int,
default=0,
help="""Clip the output volume in z to this size. The default size is the nx size of the input images.""",
)
parser.add_argument(
"--mask",
type=str,
help="""Default=None. Masking processor (see e2help.py --verbose=10) applied to the images to aid alignment. Default=None.""",
)
parser.add_argument(
"--preprocess",
type=str,
help="""Default=None. Any processor (see e2help.py --verbose=10) applied to the images to aid alignment.""",
)
parser.add_argument(
"--lowpass",
type=str,
default="",
help="""Default=None. A lowpass filtering processor (see e2help.py --verbose=10) applied to each volume prior to reprojection generation..""",
)
parser.add_argument(
"--highpass",
type=str,
default="",
help="""Default=None. A highpass filtering processor (see e2help.py --verbose=10) applied to each volume prior to reprojection generation.""",
)
parser.add_argument(
"--threshold",
type=str,
default="",
help="""Default=None. A threshold processor (see e2help.py --verbose=10) applied to each volume prior to reprojection generation.""",
)
parser.add_argument(
"--saveali",
action="store_true",
default=False,
help="""Default=False. If set, will save the recentered subtiltseries after each iteration.""",
)
(options, args) = parser.parse_args()
logger = E2init(sys.argv, options.ppid)
if not options.input and not options.inputdir and not options.inputstem:
print "ERROR: Either of the following required: --input, --inputstemp, --inputdir"
sys.exit()
if options.input and options.inputstem:
print "ERROR: Cannot provide --input and --inputstem simultaneously"
sys.exit()
if options.inputstem and options.inputdir:
print "ERROR: Cannot provide --inputstem and --inputdir simultaneously"
sys.exit()
if options.input and options.inputdir:
print "ERROR: Cannot provide --input and --inputdir simultaneously"
sys.exit()
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser(options)
# print "Options have been parsed, for example, mask is", options.mask, type(options.mask), len(options.mask)
from e2spt_classaverage import sptmakepath
options = sptmakepath(options, "sptisr")
originalpath = options.path
if options.verbose > 9:
print "\n(e2spt_isr.py) I've read the options"
inputfiles = {} # C:Figure out whether there's a single HDF stack to process,
# C:or a directory with many HDF stacks
c = os.getcwd()
if options.inputdir:
c = os.getcwd() + "/" + options.inputdir
findir = os.listdir(c)
if options.inputstem:
for f in findir:
if ".hdf" in f and options.inputstem in f:
if options.verbose > 8:
print "\nFound tiltseries!", f
inputfiles.update(
{f: [f, None]}
) # C:The input files are put into a dictionary in the format {originalseriesfile:[originalseriesfile,volumefile]}
elif options.inputdir:
for f in findir:
if ".hdf" in f:
if options.verbose > 8:
print "\nFound tiltseries!", f
inputfiles.update(
{f: [f, None]}
) # C:The input files are put into a dictionary in the format {originalseriesfile:[originalseriesfile,volumefile]}
elif options.input:
inputfiles.update({options.input: [options.input, None]})
tiltstep = 0
newvol = vol = None
newfiles = {}
firstiterdir = originalpath
print "\n\nThere are these many iterations", options.iter
fstats = {}
convergedfs = []
fmeanscores2d = {}
itermeanscores2d = {}
fmeanerrors = {}
itermeanerrors = {}
fmeanfscs = {-1: {}} # Compute initial FSC before any iterations of correction or alignment have occurred
itermeanfscs = {}
fmeanscores3d = {-1: {}}
itermeanscores3d = {}
errorswitch = 0
for i in range(options.iter): # Iterate over options.iter
itermeanscores2d.update({i: 0})
itermeanerrors.update({i: 0})
itermeanfscs.update({i: 0})
itermeanscores3d.update({i: 0})
print "***************\nStarting iteration number", i
print "****************"
previouspath = ""
if options.iter > 1:
iterdir = "iter_" + str(i + 1).zfill(len(str(options.iter)))
os.system(
"mkdir " + originalpath + "/" + iterdir
) # C:Make a new subdirectory within options.path only if options.iter > 0
options.path = originalpath + "/" + iterdir # C:Update path to include the newly created subdirectory
previouspath = originalpath + "/iter_" + str(i).zfill(len(str(options.iter)))
if i == 0:
firstiterdir = options.path
kk = 0
statsfile = options.path + "/stats.txt"
statslines = []
if len(convergedfs) == len(inputfiles):
print "\nAll files have converged. Terminating. Line 193"
break
fmeanscores2d.update({i: {}})
fmeanerrors.update({i: {}})
fmeanfscs.update({i: {}})
fmeanscores3d.update({i: {}})
for f in inputfiles:
fmeanscores2d[i].update({f: [0]})
fmeanerrors[i].update({f: [0]})
fmeanfscs[i].update({f: [0]})
fmeanscores3d[i].update({f: [0]}) # C:Iterate over files
# originalseries = f
if i == 0:
fstats.update({f: list([])})
print "fmeanfscs is", fmeanfscs
fmeanfscs[-1].update(
{f: [0]}
) # Compute initial FSC before any iterations of correction or alignment have occurred
print "fmeanscores3d is", fmeanscores3d
fmeanscores3d[-1].update({f: [0]})
print "set initial -1 fmeanscores3d!!"
if "converged" not in fstats[f]:
if i == 0:
if options.tltfile:
originalangles = getangles(options) # C:Read angles from tlt file if available
else:
originalangles = calcangles(
f
) # C:Get the angles of the actual data tilts from the header or calculate from input parameters
stackfile = options.path + "/" + os.path.basename(f).replace(".hdf", "_ISR.hdf")
print "\nWill refine center for file", f
if i == 0:
hdr = EMData(
f, 0, True
) # See if angles are in the header of the data, for each file; if not, write them
# print "\nRead header and its type", hdr, type(hdr)
# print "\nAnd the dictionary", hdr.get_attr_dict()
size = hdr["nx"]
aux = 0
if "spt_tiltangle" not in hdr.get_attr_dict():
print "\nspt_tiltangle not in header, therefore will write it by calling writeparamtoheader"
aux = writeparamtoheader(f, originalangles, "spt_tiltangle")
print "\naux returned is", aux
else:
aux = 1
if (
"sptisrtx" not in hdr.get_attr_dict()
or "sptisrty" not in hdr.get_attr_dict()
or "sptisrdr" not in hdr.get_attr_dict()
):
tvals = [0.0 for tt in range(len(originalangles))]
auxx = auxy = auxr = 0
if "sptisrtx" not in hdr.get_attr_dict():
print "\nsptisrtx not in header, therefore will write it by calling writeparamtoheader"
auxx = writeparamtoheader(f, tvals, "sptisrtx")
else:
auxx = 1
if "sptisrty" not in hdr.get_attr_dict():
print "\nsptisrty not in header, therefore will write it by calling writeparamtoheader"
auxy = writeparamtoheader(f, tvals, "sptisrty")
else:
auxy = 1
if "sptisrdr" not in hdr.get_attr_dict():
print "\nsptisrdr not in header, therefore will write it by calling writeparamtoheader"
auxr = writeparamtoheader(f, tvals, "sptisrdr")
else:
auxr = 1
if auxx and auxy and auxr:
aux = 1
else:
aux = 1
if aux:
series = {}
nimgs = EMUtil.get_image_count(f)
for ii in range(nimgs):
img = EMData(f, ii)
try:
angle = img["spt_tiltangle"]
series.update({angle: img})
except:
print "ERROR: spt_tiltangle not found in image", ii
print "\nHeader is", img.get_attr_dict()
sys.exit()
retm = makevol(
options, f, series, i, originalangles, size, writevols=1, initialfsc=1
) # C:In the first iteration, first reconstruct the tilt series into a 3D volume
vol = retm[0]
# newvolfile = retm[1]
fscarea = retm[2]
score3d = retm[3]
# fsc = retm[3]
# initialfscfilename = options.path + '/' + os.path.basename( f ).replace('.hdf', '_initial_evenOddFSC.txt')
fmeanfscs[-1][f].append(fscarea)
fmeanscores3d[-1][f].append(score3d)
print "\nVol and its type are", vol, type(vol)
else:
print "ERROR: Something went wrong. spt_tiltangle found in image headers, but somehow is unusuable"
sys.exit()
elif i > 0:
# vol = EMData( inputfiles[f][1] ) #C:For iterations > 0 (after the first iteration), the updated volumes
# C:for each tilt series should exist from the previous iteration
# previousstackfile = previouspath + '/' + os.path.basename(f).replace('.hdf','_IPET.hdf')
# print "\n\n\previousstackfile is", previousstackfile
# vol = EMData( previousstackfile, 0 )
vol = newvol
# C:Make reprojections from all known angles
reprojections = reprojectvolume(options, vol, f, originalangles)
originalsize = vol["nx"]
retrct = recentertilts(options, reprojections, f, originalangles, i)
newseries = retrct[0]
statsline = retrct[1]
fscores2d = retrct[2]
ferrors = retrct[3]
if ferrors:
errorswitch = 1
# print "fscores2d received", fscores2d
# print "ferrors received", ferrors
if ferrors == fscores2d:
print "ERROR: errors and scores2d are the same"
sys.exit()
fmeanscore2d = sum(fscores2d) / len(fscores2d)
fmeanscores2d[i][f].append(fmeanscore2d)
if ferrors and errorswitch:
fmeanerror = sum(ferrors) / len(ferrors)
fmeanerrors[i][f].append(fmeanerror)
# fmeanscores2d.update( i:{ f:fmeanscore2d } )
statslines.append(statsline)
line2append = statsline
# print "fstats[f] and type are", fstats[f], type(fstats[f])
# sys.exit()
if statsline in fstats[f]:
line2append = "converged"
convergedfs.append(f)
fstats[f].append(line2append)
fstats.update({f: fstats[f]})
retmkvol = makevol(options, f, newseries, i, originalangles, originalsize, writevols=1)
newvol = retmkvol[0]
newvolfile = retmkvol[1]
fscarea = retmkvol[2]
fmeanfscs[i][f].append(fscarea)
fmeanscore3d = retmkvol[3]
fmeanscores3d[i][f].append(fmeanscore3d)
if i == 0:
pass
# print "\nEEEEEEEEEEEEEEE\n\n\n\n\nNewvolfile returned is", newvolfile
newfiles.update({f: [newseries, newvolfile]})
# newvol.write_image( stackfile, 0 )
kk += 1
# firstrawvolfile = firstiterdir + '/' + os.path.basename(f).replace('.hdf','_3D.hdf')
# fscfile = options.path + '/' + os.path.basename(f).replace('.hdf','FSC.txt')
# cmdfsc = 'e2proc3d.py ' + firstrawvolfile + ' ' + fscfile + ' --calcfsc=' + newvolfile
# os.popen( cmdfsc )
else:
statsline = fstats[f][-2]
statslines.append(statsline)
# Write mean tx, ty and dr of all files to a text file, for each iteration
f = open(statsfile, "w")
f.writelines(statslines)
f.close()
inputfiles = newfiles
iterscores2d = []
# print "in iteration", i
# print "fmeanscores2d are",fmeanscores2d
for ff in fmeanscores2d[i]:
# print "therefore ff in fmeanscores2d[i] is", ff
iterscores2d.append(fmeanscores2d[i][ff][-1])
itermeanscore2d = sum(iterscores2d) / len(iterscores2d)
itermeanscores2d[i] = itermeanscore2d
ys = []
lines = []
for s in itermeanscores2d:
y = itermeanscores2d[s]
ys.append(y)
# print "appending this y", itermeanscores2d[s]
line = str(s) + "\t" + str(y) + "\n"
lines.append(line)
# if plots:
fs = open(options.path + "/scores2d_" + str(i + 1) + ".txt", "w")
fs.writelines(lines)
fs.close()
try:
if errorswitch:
itererrors = []
# print "in iteration", i
# print "fmeanerrors are",fmeanerrors
for fscores2df in fmeanerrors[i]:
# print "therefore ff in fmeanerrors[i] is", ff
itererrors.append(fmeanerrors[i][ff][-1])
itermeanerror = sum(itererrors) / len(itererrors)
itermeanerrors[i] = itermeanerror
yse = []
linese = []
for s in itermeanerrors:
ye = itermeanerrors[s]
yse.append(ye)
# print "appending this error", itermeanerrors[s]
linee = str(s) + "\t" + str(ye) + "\n"
linese.append(linee)
# if plots:
fse = open(options.path + "/error_" + str(i + 1) + ".txt", "w")
fse.writelines(linese)
fse.close()
except:
pass
if i == 0:
iterfscs = []
# print "in iteration", i
# print "APPENDING INITIAL fmeanfscs are", fmeanfscs
for ff in fmeanfscs[-1]:
# print "therefore INITIAL ff in fmeanfscs[-1] are", ff
iterfscs.append(fmeanfscs[-1][ff][-1])
itermeanfsc = sum(iterfscs) / len(iterfscs)
itermeanfscs[-1] = itermeanfsc
ysf = []
linesf = []
for s in [-1, 0]:
yf = itermeanfscs[s]
ysf.append(yf)
# print "for s", s
# print "appending this fscarea", itermeanfscs[s]
linef = str(s + 1) + "\t" + str(yf) + "\n"
linesf.append(linef)
# if plots:
fsf = open(options.path + "/fscsareas_" + str(i + 1) + ".txt", "w")
fsf.writelines(linesf)
fsf.close()
iterscores3d = []
# print "in iteration", i
# print "fmeanscores2d are",fmeanscores2d
for ff in fmeanscores3d[-1]:
# print "therefore ff in fmeanscores2d[i] is", ff
iterscores3d.append(fmeanscores3d[-1][ff][-1])
itermeanscore3d = sum(iterscores3d) / len(iterscores3d)
itermeanscores3d[-1] = itermeanscore3d
ys3 = []
lines3 = []
for s in [-1, 0]:
y3 = itermeanscores3d[s]
ys3.append(y)
# print "appending this y", itermeanscores2d[s]
line3 = str(s + 1) + "\t" + str(y3) + "\n"
lines3.append(line3)
# if plots:
fs = open(options.path + "/scores3d_" + str(i + 1) + ".txt", "w")
fs.writelines(lines3)
fs.close()
iterfscs = []
# print "in iteration", i
# print "fmeanfscs are",fmeanfscs
for ff in fmeanfscs[i]:
# print "therefore ff in fmeanfscs[i] are", ff
iterfscs.append(fmeanfscs[i][ff][-1])
itermeanfsc = sum(iterfscs) / len(iterfscs)
itermeanfscs[i] = itermeanfsc
ysf = []
linesf = []
fscsindxs = [s for s in itermeanfscs]
fscsindxs.sort()
for s in fscsindxs:
yf = itermeanfscs[s]
ysf.append(yf)
# print "for s", s
# print "appending this fscarea", itermeanfscs[s]
linef = str(s + 1) + "\t" + str(yf) + "\n"
linesf.append(linef)
# if plots:
fsf = open(options.path + "/fscsareas_" + str(i + 1) + ".txt", "w")
fsf.writelines(linesf)
fsf.close()
iterscores3d = []
# print "in iteration", i
# print "fmeanscores2d are",fmeanscores2d
for ff in fmeanscores3d[i]:
# print "therefore ff in fmeanscores2d[i] is", ff
iterscores3d.append(fmeanscores3d[i][ff][-1])
itermeanscore3d = sum(iterscores3d) / len(iterscores3d)
itermeanscores3d[i] = itermeanscore3d
ys3 = []
lines3 = []
score3dindxs = [s for s in itermeanscores3d]
score3dindxs.sort()
for s in score3dindxs:
y3 = itermeanscores3d[s]
ys3.append(y)
# print "appending this y", itermeanscores2d[s]
line3 = str(s + 1) + "\t" + str(y3) + "\n"
lines3.append(line3)
# if plots:
fs = open(options.path + "/scores3d_" + str(i + 1) + ".txt", "w")
fs.writelines(lines3)
fs.close()
if i > 0:
if itermeanscores2d[i] == itermeanscores2d[i - 1]:
print "The meanscore2d for two consecutive iterations is the same, suggesting the algorithm has converged."
sys.exit()
try:
if itermeanerrors[i] == itermeanerrors[i - 1]:
print "The meanerror for two consecutive iterations is the same, suggesting the algorithm has converged."
sys.exit()
except:
pass
if itermeanfscs[i] == itermeanfscs[i - 1]:
print "The fscsarea for two consecutive iterations is the same, suggesting the algorithm has converged."
sys.exit()
if itermeanscores3d[i] == itermeanscores3d[i - 1]:
print "The meanscore3d for two consecutive iterations is the same, suggesting the algorithm has converged."
sys.exit()
difscore = math.fabs(float(itermeanscores2d[i]) - float(itermeanscores2d[i - 1]))
if float(difscore) < 0.000001:
print "In iter %d/%d global score difference with previous iteration is smaller than 0.000001, see: %f \nExiting program." % (
i + 1,
options.iter,
difscore,
)
sys.exit()
try:
diferror = math.fabs(float(itermeanerrors[i]) - float(itermeanerrors[i - 1]))
if float(difscore) < 0.000001:
print "In iter %d/%d global error difference with previous iteration is smaller than 0.000001, see: %f \nExiting program." % (
i + 1,
options.iter,
diferror,
)
sys.exit()
except:
pass
diffscarea = math.fabs(float(itermeanfscs[i]) - float(itermeanfscs[i - 1]))
if float(difscore) < 0.000001:
print "In iter %d/%d global fscarea difference with previous iteration is smaller than 0.000001, see: %f \nExiting program." % (
i + 1,
options.iter,
diffscarea,
)
sys.exit()
difscore3d = math.fabs(float(itermeanscores3d[i]) - float(itermeanscores3d[i - 1]))
if float(difscore3d) < 0.000001:
print "In iter %d/%d global score3dipetResult difference with previous iteration is smaller than 0.000001, see: %f \nExiting program." % (
i + 1,
options.iter,
difscore,
)
sys.exit()
E2end(logger)
return ()
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options]
This program takes aligned stacks produced by e2spt_classaverage.py and raw tomograms
to recompute averages with different options (such as extracting recentered particles,
and possibly with a larger or smaller boxisze).
All the necessary files (aligned stacks in HDF format and raw tomograms in MRC format
ending in .rec, as produced by IMOD) should be in the running directory.
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--stacks", default='',type=str, help="""Comma separated list of HDF image stacks to process.""")
parser.add_argument("--tomograms", default='',type=str, help="""Comma separated list of tomograms with REC extension from which all particles in --stacks came from.""")
#parser.add_argument("--output", default="avg.hdf",type=str, help="The name of the output average volume.")
#parser.add_argument("--rotationtype", default="eman",type=str, help="Valid options are: eman,imagic,mrc,spider,quaternion,sgirot,spi,xyz")
#parser.add_argument("--averager",type=str,help="The type of averager used to produce the class average. Default=mean",default="mean")
parser.add_argument("--sym", type=str, default='', help = "Symmetry to impose - choices are: c<n>, d<n>, h<n>, tet, oct, icos")
parser.add_argument("--path",default='',type=str,help="Name of directory where to save the output file.")
#parser.add_argument("--alifile",default='',type=str,help=".json file with alingment parameters, if raw stack supplied via --input.")
parser.add_argument("--boxsize","-B",type=int,default=0,help="""Default=0 (option not used). Provide a value for the boxsize of the output average in pixels. If not provided, the boxsize of --stacks will be used.""")
parser.add_argument("--normproc",type=str,default='normalize.edgemean',help="""Normalization processor applied to particles before extraction. Default=normalize.edgemean. If using the latter, you must provide --masknorm, otherwise, a default --masknorm=mask.sharp:outer_radius=-2 will be used.""")
parser.add_argument("--threshold",type=str,help="""Threshold processor to apply to particles before writing them out to get rid of too high and/or too low pixel values.""", default='')
parser.add_argument("--usetomograms",action='store_true',default=False,help=""""Re-extract particles from the original tomogram.""")
parser.add_argument("--useinverseali",action='store_true',default=False,help=""""Use the inverse of the value stored in xform.align3d in the header of each particle.""")
parser.add_argument('--shrink', type=int, default=1, help="""Shrink factor to shrink particles before averaging. Default=1, which means no shrinking.""")
parser.add_argument("--lowpass",type=str,help="""Lowpass filtering processor to apply to particles before averaging. Default=None.""",default='')
parser.add_argument("--preprocess",type=str,help="""A processor (as in e2proc3d.py) to be applied to the tomogram before opening it. \nFor example, a specific filter with specific parameters you might like. \nType 'e2proc3d.py --processors' at the commandline to see a list of the available processors and their usage""",default=None)
parser.add_argument('--invert', action="store_true", default=False, help='''Default=False. This parameer indicates you want the contrast to me inverted while boxing, AND for the extracted sub-volumes. Remember that EMAN2 **MUST** work with "white" protein. You can very easily figure out what the original color\nof the protein is in your data by looking at the gold fiducials or the edge of the carbon hole in your tomogram. If they look black you MUST specify this option''')
parser.add_argument("--averager",type=str,help="""The type of averager used to produce the class average. Default=mean.tomo""", default='mean.tomo')
parser.add_argument("--keep",type=float,help="""The fraction of particles to keep in each class. Default=1.0""",default=1.0)
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n",type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness.")
#parser.add_argument("--saveali",action="store_true", default=False,help="""If set, will save the
# aligned particle volumes in class_ptcl.hdf. Overwrites existing file.""")
(options, args) = parser.parse_args()
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser( options )
'''
if options.normproc:
options.normproc=parsemodopt(options.normproc)
if options.threshold:
options.threshold=parsemodopt(options.threshold)
if options.lowpass:
options.lowpass=parsemodopt(options.lowpass)
if options.preprocess:
options.preprocess=parsemodopt(options.preprocess)
if options.averager:
options.averager=parsemodopt(options.averager)
'''
#if not options.boxsize:
# print "\n(e2spt_recompute.py) (main) ERROR: Boxsize must be greater than zero. It is:", options.boxsize
# sys.exit()
logid=E2init(sys.argv,options.ppid)
from e2spt_classaverage import sptmakepath
options = sptmakepath(options,'sptrecompute')
c = os.getcwd() #Get current directory
findir = os.listdir( c ) #Get list of files in current directory
stacks = set([]) #Make list set to store particle stacks
if options.stacks:
stacks = set( options.stacks.split(',') )
if not options.boxsize:
boxesdict = {}
boxes = []
for sta in stacks:
box = EMData( sta, 0 )['nx']
boxesdict.update({sta:box})
boxes.append( box )
boxes = set(boxes)
if len(boxes) >1:
print "ERROR: Your stacks are not all the same boxsize. There are %d many different boxsizes" %(len(boxes))
print "which are", boxes
print "all input stacks in --stacks must have the same boxsize; otherwise, specify a unique boxsize through --boxsize."
sys.exit()
else:
for f in findir:
if '.hdf' in f:
stacks.add(f)
tomograms = set([])
tomogramsbases = set([])
if options.usetomograms: #Make list set to store tomograms
if options.tomograms:
tomograms = set( options.tomograms.split(',') )
for tom in tomograms:
tombase = os.path.basename(tom)
tomogramsbases.append(tombase)
else:
for f in findir:
if '.rec' in f:
tomograms.add(f)
for stack in stacks: #Loop over stacks
n = EMUtil.get_image_count( stack ) #Determine number of particles in stack
avgr = Averagers.get(options.averager[0], options.averager[1]) #initialize averager
print "\n(e2spt_recompute.py) (main) Processing stack",stack
for i in range(n): #Loop over particles in stack
hdr = EMData( stack, i, True) #Load particle header by providing 'True' flag
a = None
box = hdr['nx']
if options.boxsize:
box = options.boxsize
if options.usetomograms and tomograms:
tomogram = hdr['ptcl_source_image'] #Determine what tomogram a particle comes from
if tomogram not in tomogramsbases and tomogram not in tomograms:
print "\n(e2spt_recompute.py) (main) ERROR: Tomogram %s not found" %( tomogram )
sys.exit()
print "\n(e2spt_recompute.py) (main) Processing particle %d in stack %s which should come from tomogram %s" %(i, stack, tomogram )
coords = hdr['ptcl_source_coord'] #And from what location exactly
x = coords[0] #Parse coordinates
y = coords[1]
z = coords[2]
r = Region((2*x-box)/2,(2*y-box)/2, (2*z-box)/2, box, box, box) #Define extraction region based on boxsize
a = EMData()
a.read_image(tomogram,0,False,r) #Actually 'read'/extract particle data
#Preprocess as needed
else:
a = EMData( stack, i )
if a:
if options.normproc:
a.process_inplace(options.normproc[0],options.normproc[1])
if options.invert:
a.mult(-1)
t = None
try:
t = hdr['xform.align3d'] #Rotate particle into aligned orientation
except:
print "WARNING: 'xform.align3d' not found in header of particle %d" % (i)
try:
t = hdr['sptsim_randT']
except:
print "ERROR: 'sptsim_randT also not found in header or particle %d" %(i)
if t:
tf = t
if options.useinverseali:
tf = t.inverse()
print "t is", t
print "and its inverse is", tf
#print "Applied this transform",tf
a.transform(tf)
if options.threshold:
a.process_inplace(options.threshold[0],options.threshold[1])
if options.lowpass:
a.process_inplace(options.lowpass[0],options.lowpass[1])
if options.preprocess:
a.process_inplace(options.preprocess[0],options.preprocess[1])
if options.shrink and int(options.shrink) > 1:
shrinkfactor = options.shrink
a.process_inplace('math.meanshrink',{'n':shrinkfactor})
#a['origin_x'] = 0
#a['origin_y'] = 0
#a['origin_z'] = 0
#a['ptcl_source_image'] = hdr['ptcl_source_image']
#a['ptcl_source_coord'] = hdr['ptcl_source_coord']
avgr.add_image(a)
else:
print "skipping particle", i
avg = avgr.finish()
avg.process_inplace('normalize')
if options.sym and not breaksym:
avg.process_inplace('xform.applysym',{'sym':options.sym})
outname = options.path + '/' + stack.replace('.hdf','_AVG.hdf')
avg.write_image(outname,0)
E2end(logid)
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog <output> [options]
Program to build an initial subtomogram average by averaging pairs from the largest subset
in --input that is a power of 2. For example, if you supply an input stack with 100 subtomograms,
this program will build an initial reference using 64, since 64 is the largest power of 2 contained in 100.
In the first iteration, particle 1 will be averaged with 2, 3 with 4, 5 with 6... etc.
32 new averages (each an average of 2 subtomograms) will be used for the second iteration.
Again, 1 will be averaged with 2, 3 with 4, etc... yielding 16 new averages.
The algorithm continues until the entire subset (64) has been merged into 1 average.
This program depends on e2spt_classaverage.py because it imports the preprocessing
and alignment functions from it.
--mask=mask.sharp:outer_radius=<safe radius>
--preprocess=filter.lowpass.gauss:cutoff_freq=<1/resolution in A>
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_header(name="sptbtheader",
help="""Options below this label are specific to
sptbinarytree""",
title="### sptbinarytree options ###",
row=6,
col=0,
rowspan=1,
colspan=3,
mode="align")
parser.add_header(
name="caheader",
help="""Options below this label are specific to sptclassaverage""",
title="### sptclassaverage options ###",
row=3,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--path",
type=str,
default='spt',
help=
"""Default=spt. Directory to store results in. The default is a numbered series of directories containing the prefix 'spt'; for example, spt_02 will be the directory by default if 'spt_01' already exists."""
)
parser.add_argument(
"--input",
type=str,
default='',
help=
"""Default=None. The name of the input volume stack. MUST be HDF since volume stack support is required.""",
guitype='filebox',
browser='EMSubTomosTable(withmodal=True,multiselect=False)',
row=0,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--output",
type=str,
default='avg.hdf',
help=
"""Default=avg.hdf. The name of the output class-average stack. MUST be HDF since volume stack support is required.""",
guitype='strbox',
row=2,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
#parser.add_argument("--classmx", type=str, default='', help="""Default=None. The name of the classification matrix specifying how particles in 'input' should be grouped. If omitted, all particles will be averaged.""")
#parser.add_argument("--ref", type=str, default='', help="""Default=None. Reference image(s). Used as an initial alignment reference and for final orientation adjustment if present. This is typically the projections that were used for classification.""", guitype='filebox', browser='EMBrowserWidget(withmodal=True,multiselect=True)', filecheck=False, row=1, col=0, rowspan=1, colspan=3, mode='alignment')
#parser.add_argument("--refpreprocess",action="store_true",default=False,help="""Default=False. This will preprocess the reference identically to the particles. It is off by default, but it is internally turned on when no reference is supplied.""")
#parser.add_argument("--resultmx",type=str,default=None,help="""Default=Npone. Specify an output image to store the result matrix. This is in the same format as the classification matrix. http://blake.bcm.edu/emanwiki/EMAN2/ClassmxFiles""")
#parser.add_argument("--refinemultireftag", type=str, default='', help="""Default=''. DO NOT USE THIS PARAMETER. It is passed on from e2spt_refinemulti.py if needed.""")
parser.add_argument(
"--radius",
type=float,
default=0,
help=
"""Default=0 (which means it's not used by default). Hydrodynamic radius of the particle in Angstroms. This will be used to automatically calculate the angular steps to use in search of the best alignment. Make sure the apix is correct on the particles' headers, sine the radius will be converted from Angstroms to pixels. Then, the fine angular step is equal to 360/(2*pi*radius), and the coarse angular step 4 times that."""
)
parser.add_argument(
"--precision",
type=float,
default=1.0,
help=
"""Default=1.0. Precision in pixels to use when figuring out alignment parameters automatically using --radius. Precision would be the number of pixels that the the edge of the specimen is moved (rotationally) during the finest sampling, --falign. If precision is 1, then the precision of alignment will be that of the sampling (apix of your images) times the --shrinkfine factor specified."""
)
parser.add_argument(
"--search",
type=int,
default=8,
help=
""""Default=8. During COARSE alignment translational search in X, Y and Z, in pixels. Default=8. This WILL overwrite any search: provided through --align, EXCEPT if you provide --search=8, which is the default. In general, just avoid providing search twice (through here and through the aligner, --align). If you do, just be careful to make them consistent to minimize misinterpretation and error."""
)
parser.add_argument(
"--searchfine",
type=int,
default=2,
help=
""""Default=2. During FINE alignment translational search in X, Y and Z, in pixels. Default=2. This WILL overwrite any search: provided through --falign, EXCEPT if you provide --searchfine=2, which is the default. In general, just avoid providing search twice (through here and through the fine aligner --falign). If you do, just be careful to make them consistent to minimize misinterpretation and error."""
)
#parser.add_argument("--donotaverage",action="store_true", help="""If e2spt_refinemulti.py is calling e2spt_classaverage.py, the latter need not average any particles, but rather only yield the alignment results.""", default=False)
parser.add_argument(
"--iterstop",
type=int,
default=0,
help=
"""Default=0. (Not used). The program is called to convergence by default (all particles merge into one final average). To stop at an intermediate iteration, provide this parameter. For example, --iterstop=1, will only allow the algorithm to complete 1 iteration; --iterstop=2 will allow it to go through 2, etc."""
)
parser.add_argument(
"--savesteps",
action="store_true",
default=False,
help=
"""Default=False. If set, will save the average after each iteration to class_#.hdf. Each class in a separate file. Appends to existing files.""",
guitype='boolbox',
row=4,
col=0,
rowspan=1,
colspan=1,
mode='alignment,breaksym')
parser.add_argument(
"--saveali",
action="store_true",
default=False,
help=
"""Default=False. If set, will save the aligned particle volumes in class_ptcl.hdf. Overwrites existing file.""",
guitype='boolbox',
row=4,
col=1,
rowspan=1,
colspan=1,
mode='alignment,breaksym')
parser.add_argument(
"--saveallalign",
action="store_true",
default=False,
help=
"""Default=False. If set, will save the alignment parameters after each iteration""",
guitype='boolbox',
row=4,
col=2,
rowspan=1,
colspan=1,
mode='alignment,breaksym')
parser.add_argument(
"--sym",
dest="sym",
default='',
help=
"""Default=None (equivalent to c1). Symmetry to impose -choices are: c<n>, d<n>, h<n>, tet, oct, icos""",
guitype='symbox',
row=9,
col=1,
rowspan=1,
colspan=2,
mode='alignment,breaksym')
parser.add_argument(
"--mask",
type=str,
default="mask.sharp:outer_radius=-2",
help=
"""Default is mask.sharp:outer_radius=-2. Masking processor applied to particles before alignment. IF using --clipali, make sure to express outer mask radii as negative pixels from the edge.""",
returnNone=True,
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'mask\')',
row=11,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--maskfile",
type=str,
default='',
help=
"""Default=None. Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None."""
)
parser.add_argument(
"--normproc",
type=str,
default='normalize.edgemean',
help=
"""Default is 'normalize.edgemean' (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to particles before alignment. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'"""
)
parser.add_argument(
"--threshold",
type=str,
default='',
help=
"""Default=None. A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=10,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--preprocess",
type=str,
default='',
help=
"""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=10,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--preprocessfine",
type=str,
default='',
help=
"""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--lowpass",
type=str,
default='',
help=
"""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=17,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--lowpassfine",
type=str,
default='',
help=
"""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--highpass",
type=str,
default='',
help=
"""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=18,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--highpassfine",
type=str,
default='',
help=
"""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--shrink",
type=int,
default=1,
help=
"""Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount for coarse alignment.""",
guitype='shrinkbox',
row=5,
col=1,
rowspan=1,
colspan=1,
mode='alignment,breaksym')
parser.add_argument(
"--shrinkfine",
type=int,
default=1,
help=
"""Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount for refine alignment.""",
guitype='intbox',
row=5,
col=2,
rowspan=1,
colspan=1,
mode='alignment')
parser.add_argument(
"--clipali",
type=int,
default=0,
help=
"""Default=0 (which means it's not used). Boxsize to clip particles as part of preprocessing to speed up alignment. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary; still, there are some benefits from 'oversampling' the data during averaging; so you might still want an average of size 2x, but perhaps particles in a box of 1.5x are sufficiently good for alignment. In this case, you would supply --clipali=75"""
)
parser.add_argument(
"--postprocess",
type=str,
default='',
help=
"""A processor to be applied to the FINAL volume after averaging the raw volumes in their FINAL orientations, after all iterations are done.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=16,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--procfinelikecoarse",
action='store_true',
default=False,
help=
"""If you supply this parameters, particles for fine alignment will be preprocessed identically to particles for coarse alignment by default. If you supply this, but want specific parameters for preprocessing particles for also supply: fine alignment, nd supply fine alignment parameters, such as --lowpassfine, --highpassfine, etc; to preprocess the particles for FINE alignment differently than for COARSE alignment."""
)
parser.add_argument(
"--npeakstorefine",
type=int,
help=
"""Default=1. The number of best coarse alignments to refine in search of the best final alignment. Default=1.""",
default=4,
guitype='intbox',
row=9,
col=0,
rowspan=1,
colspan=1,
nosharedb=True,
mode='alignment,breaksym[1]')
parser.add_argument(
"--align",
type=str,
default="rotate_translate_3d:search=8:delta=12:dphi=12",
help=
"""This is the aligner used to align particles to the previous class average. Default is rotate_translate_3d:search=8:delta=12:dphi=12, specify 'None' (with capital N) to disable.""",
returnNone=True,
guitype='comboparambox',
choicelist='re_filter_list(dump_aligners_list(),\'3d\')',
row=12,
col=0,
rowspan=1,
colspan=3,
nosharedb=True,
mode="alignment,breaksym['rotate_symmetry_3d']")
parser.add_argument(
"--aligncmp",
type=str,
default="ccc.tomo",
help=
"""Default=ccc.tomo. The comparator used for the --align aligner. Do not specify unless you need to use anotherspecific aligner.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_cmps_list(),\'tomo\')',
row=13,
col=0,
rowspan=1,
colspan=3,
mode="alignment,breaksym")
parser.add_argument(
"--falign",
type=str,
default="refine_3d_grid:delta=3:range=15:search=2",
help=
"""Default="refine_3d_grid:delta=3:range=15:search=2". This is the second stage aligner used to fine-tune the first alignment. Specify 'None' to disable.""",
returnNone=True,
guitype='comboparambox',
choicelist='re_filter_list(dump_aligners_list(),\'refine.*3d\')',
row=14,
col=0,
rowspan=1,
colspan=3,
nosharedb=True,
mode='alignment,breaksym[None]')
parser.add_argument(
"--faligncmp",
type=str,
default="ccc.tomo",
help=
"""Default=ccc.tomo. The comparator used by the second stage aligner.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_cmps_list(),\'tomo\')',
row=15,
col=0,
rowspan=1,
colspan=3,
mode="alignment,breaksym")
parser.add_argument(
"--averager",
type=str,
default="mean.tomo",
help=
"""Default=mean.tomo. The type of averager used to produce the class average. Default=mean.tomo."""
)
#parser.add_argument("--keep",type=float,default=1.0,help="""Default=1.0 (all particles kept). The fraction of particles to keep in each class.""", guitype='floatbox', row=6, col=0, rowspan=1, colspan=1, mode='alignment,breaksym')
#parser.add_argument("--keepsig", action="store_true", default=False,help="""Default=False. Causes the keep argument to be interpreted in standard deviations.""", guitype='boolbox', row=6, col=1, rowspan=1, colspan=1, mode='alignment,breaksym')
#parser.add_argument("--inixforms",type=str,default="",help="""Default=None. .json file containing a dict of transforms to apply to 'pre-align' the particles.""", guitype='dirbox', dirbasename='spt_|sptsym_', row=7, col=0,rowspan=1, colspan=2, nosharedb=True, mode='breaksym')
parser.add_argument(
"--breaksym",
action="store_true",
default=False,
help=
"""Default=False. Break symmetry. Do not apply symmetrization after averaging, even if searching the asymmetric unit provided through --sym only for alignment. Default=False""",
guitype='boolbox',
row=7,
col=2,
rowspan=1,
colspan=1,
nosharedb=True,
mode=',breaksym[True]')
#parser.add_argument("--groups",type=int,default=0,help="""Default=0 (not used; data not split). This parameter will split the data into a user defined number of groups. For purposes of gold-standard FSC computation later, select --group=2.""")
parser.add_argument(
"--randomizewedge",
action="store_true",
default=False,
help=
"""Default=False. This parameter is EXPERIMENTAL. It randomizes the position of the particles BEFORE alignment, to minimize missing wedge bias and artifacts during symmetric alignment where only a fraction of space is scanned"""
)
parser.add_argument(
"--savepreprocessed",
action="store_true",
default=False,
help=
"""Default=False. Will save stacks of preprocessed particles (one for coarse alignment and one for fine alignment if preprocessing options are different)."""
)
parser.add_argument(
"--autocenter",
type=str,
default='',
help=
"""Default=None. Autocenters each averaged pair during initial average generation with --btref and --hacref. Will also autocenter the average of all particles after each iteration of iterative refinement. Options are --autocenter=xform.centerofmass (self descriptive), or --autocenter=xform.centeracf, which applies auto-convolution on the average."""
)
parser.add_argument(
"--autocentermask",
type=str,
default='',
help=
"""Default=None. Masking processor to apply before autocentering. See 'e2help.py processors -v 10' at the command line."""
)
parser.add_argument(
"--autocenterpreprocess",
action='store_true',
default=False,
help=
"""Default=False. This will apply a highpass filter at a frequency of half the box size times the apix, shrink by 2, and apply a low pass filter at half nyquist frequency to any computed average for autocentering purposes if --autocenter is provided. Default=False."""
)
parser.add_argument(
"--parallel",
default="thread:1",
help=
"""default=thread:1. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel""",
guitype='strbox',
row=19,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--ppid",
type=int,
help=
"""Default=-1. Set the PID of the parent process, used for cross platform PPID""",
default=-1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"""Default=0. Verbose level [0-9], higner number means higher level of verboseness"""
)
#parser.add_argument("--resume",type=str,default='',help="""(Not working currently). tomo_fxorms.json file that contains alignment information for the particles in the set. If the information is incomplete (i.e., there are less elements in the file than particles in the stack), on the first iteration the program will complete the file by working ONLY on particle indexes that are missing. For subsequent iterations, all the particles will be used.""")
parser.add_argument(
"--plots",
action='store_true',
default=False,
help=
"""Default=False. Turn this option on to generatea plot of the ccc scores during each iteration. Running on a cluster or via ssh remotely might not support plotting."""
)
parser.add_argument(
"--subset",
type=int,
default=0,
help=
"""Default=0 (not used). Refine only this substet of particles from the stack provided through --input"""
)
parser.add_argument(
"--notmatchimgs",
action='store_true',
default=False,
help=
"""Default=True. This option prevents applying filter.match.to to one image so that it matches the other's spectral profile during preprocessing for alignment purposes."""
)
parser.add_argument(
"--preavgproc1",
type=str,
default='',
help=
"""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)"""
)
parser.add_argument(
"--preavgproc2",
type=str,
default='',
help=
"""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)"""
)
parser.add_argument(
"--weighbytiltaxis",
type=str,
default='',
help=
"""Default=None. A,B, where A is an integer number and B a decimal. A represents the location of the tilt axis in the tomogram in pixels (eg.g, for a 4096x4096xZ tomogram, this value should be 2048), and B is the weight of the particles furthest from the tomogram. For example, --weighbytiltaxis=2048,0.5 means that praticles at the tilt axis (with an x coordinate of 2048) will have a weight of 1.0 during averaging, while the distance in the x coordinates of particles not-on the tilt axis will be used to weigh their contribution to the average, with particles at the edge(0+radius or 4096-radius) weighing 0.5, as specified by the value provided for B."""
)
parser.add_argument(
"--weighbyscore",
action='store_true',
default=False,
help=
"""Default=False. This option will weigh the contribution of each subtomogram to the average by score/bestscore."""
)
parser.add_argument(
"--tweak",
action='store_true',
default=False,
help=
"""WARNING: BUGGY. This will perform a final alignment with no downsampling [without using --shrink or --shrinkfine] if --shrinkfine > 1."""
)
'''
BT SPECIFIC PARAMETERS
'''
parser.add_argument("--nseedlimit",
type=int,
default=0,
help="""Maximum number of particles
to use. For example, if you supply a stack with 150 subtomograms, the program will
automatically select 128 as the limit to use because it's the largest power of 2 that is
smaller than 150. But if you provide, say --nseedlimit=100, then the number of particles
used will be 64, because it's the largest power of 2 that is still smaller than 100."""
)
(options, args) = parser.parse_args()
'''
Make the directory where to create the database where the results will be stored
'''
from e2spt_classaverage import sptmakepath
options = sptmakepath(options, 'spt_bt')
rootpath = os.getcwd()
if rootpath not in options.path:
options.path = rootpath + '/' + options.path
if not options.input:
parser.print_help()
exit(0)
elif options.subset:
subsetStack = options.path + '/subset' + str(options.subset).zfill(
len(str(options.subset))) + '.hdf'
print "\nSubset to be written to", subsetStack
subsetcmd = 'e2proc3d.py ' + options.input + ' ' + subsetStack + ' --first=0 --last=' + str(
options.subset - 1)
print "Subset cmd is", subsetcmd
p = subprocess.Popen(subsetcmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
text = p.communicate()
p.stdout.close()
options.input = subsetStack
from e2spt_classaverage import sptParseAligner
options = sptParseAligner(options)
'''
If --radius of the particle is provided, we calculate the optimal alignment steps for
coarse and fine alignment rounds using --shrink and --shrinkfine options and apix info
'''
if options.shrink < options.shrinkfine:
options.shrink = options.shrinkfine
print "It makes no sense for shrinkfine to be larger than shrink; therefore, shrink will be made to match shrinkfine"
if options.radius:
from e2spt_classaverage import calcAliStep
options = calcAliStep(options)
'''
Parse parameters such that "None" or "none" are adequately interpreted to turn of an option
'''
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser(options)
from e2spt_classaverage import writeParameters
writeParameters(options, 'e2spt_binarytree.py', 'bt')
hdr = EMData(options.input, 0, True)
nx = hdr["nx"]
ny = hdr["ny"]
nz = hdr["nz"]
if nx != ny or ny != nz:
print "ERROR, input volumes are not cubes"
sys.exit(1)
logger = E2init(sys.argv, options.ppid)
'''
Initialize parallelism if being used
'''
if options.parallel:
if options.parallel == 'none' or options.parallel == 'None' or options.parallel == 'NONE':
options.parallel = ''
etc = ''
else:
print "\n\n(e2spt_classaverage.py) INITIALIZING PARALLELISM!"
print "\n\n"
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel)
pclist = [options.input]
etc.precache(pclist)
else:
etc = ''
nptcl = EMUtil.get_image_count(options.input)
if nptcl < 1:
print "ERROR : at least 2 particles required in input stack"
sys.exit(1)
ptclnums = range(nptcl)
nptclForRef = len(ptclnums)
nseed = 2**int(
floor(log(len(ptclnums), 2))
) # we stick with powers of 2 for this to make the tree easier to collapse
if options.nseedlimit:
nseed = 2**int(floor(log(options.nseedlimit, 2)))
binaryTreeRef(options, nptclForRef, nseed, -1, etc)
print "Will end logger"
E2end(logger)
print "logger ended"
sys.stdout.flush()
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options]
This program aligns a paricle to its symmetry axis. There are two algorithmic modes.
A coarse search followed by simplex minimization (not yet implimented) OR monte carlo course
search followed by simplex minimization. The Goal is to align the paricle to its
symmetry axis so symmetry can be applied for avergaing and for alignment speed up
(it is only necessary to search over one asymmetric unit!
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_header(
name="symsearch3dheader",
help="""Options below this label are specific to e2symsearch3d""",
title="### e2symsearch3d options ###",
row=3,
col=0,
rowspan=1,
colspan=2,
)
parser.add_argument(
"--input",
dest="input",
default="",
type=str,
help="""The name of input volume or hdf stack of volumes""",
guitype="filebox",
browser="EMBrowserWidget(withmodal=True,multiselect=False)",
row=0,
col=0,
rowspan=1,
colspan=2,
)
# parser.add_argument("--output", dest="output", default="""e2symsearch3d_OUTPUT.hdf""", type=str, help="The name of the output volume", guitype='strbox', filecheck=False, row=1, col=0, rowspan=1, colspan=2)
parser.add_argument(
"--ref",
type=str,
default="",
help="""Default=None. If provided and --average is also provided and --keep < 1.0 or --keepsig is specified, 'good particles' will be determined by correlation to --ref.""",
)
parser.add_argument(
"--mirror",
type=str,
default="",
help="""Axis across of which to generate a mirrored copy of --ref. All particles will be compared to it in addition to the unmirrored image in --ref if --keepsig is provided or if --keep < 1.0.""",
)
parser.add_argument(
"--path",
type=str,
default="",
help="""Name of path for output file""",
guitype="strbox",
row=2,
col=0,
rowspan=1,
colspan=2,
)
parser.add_argument(
"--plots",
action="store_true",
default=False,
help="""Default=False. Turn this option on to generate a plot of the ccc scores if --average is supplied. Running on a cluster or via ssh remotely might not support plotting.""",
)
parser.add_argument(
"--sym",
dest="sym",
default="c1",
help="""Specify symmetry -choices are: c<n>, d<n>, h<n>, tet, oct, icos. For asymmetric reconstruction ommit this option or specify c1.""",
guitype="symbox",
row=4,
col=0,
rowspan=1,
colspan=2,
)
parser.add_argument(
"--shrink",
dest="shrink",
type=int,
default=0,
help="""Optionally shrink the input particles by an integer amount prior to computing similarity scores. For speed purposes. Default=0, no shrinking""",
guitype="shrinkbox",
row=5,
col=0,
rowspan=1,
colspan=1,
)
parser.add_argument(
"--mask",
type=str,
help="""Mask processor applied to particles before alignment. Default is mask.sharp:outer_radius=-2. IF using --clipali, make sure to express outer mask radii as negative pixels from the edge.""",
returnNone=True,
default="mask.sharp:outer_radius=-2",
guitype="comboparambox",
choicelist="re_filter_list(dump_processors_list(),'mask')",
row=11,
col=0,
rowspan=1,
colspan=3,
)
parser.add_argument(
"--maskfile",
type=str,
default="",
help="""Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None.""",
)
parser.add_argument(
"--normproc",
type=str,
default="",
help="""Normalization processor applied to particles before alignment. Default is to use normalize. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'""",
)
parser.add_argument(
"--nopreprocprefft",
action="store_true",
default=False,
help="""Turns off all preprocessing that happens only once before alignment (--normproc, --mask, --maskfile, --clipali, --threshold; i.e., all preprocessing excepting filters --highpass, --lowpass, --preprocess, and --shrink.""",
)
parser.add_argument(
"--threshold",
default="",
type=str,
help="""A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""",
guitype="comboparambox",
choicelist="re_filter_list(dump_processors_list(),'filter')",
row=10,
col=0,
rowspan=1,
colspan=3,
)
parser.add_argument(
"--preprocess",
default="",
type=str,
help="""Any processor (as in e2proc3d.py) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype="comboparambox",
choicelist="re_filter_list(dump_processors_list(),'filter')",
row=10,
col=0,
rowspan=1,
colspan=3,
)
parser.add_argument(
"--lowpass",
type=str,
default="",
help="""A lowpass filtering processor (from e2proc3d.py; see e2help.py processors) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype="comboparambox",
choicelist="re_filter_list(dump_processors_list(),'filter')",
row=17,
col=0,
rowspan=1,
colspan=3,
)
parser.add_argument(
"--highpass",
type=str,
default="",
help="""A highpass filtering processor (from e2proc3d.py, see e2help.py processors) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype="comboparambox",
choicelist="re_filter_list(dump_processors_list(),'filter')",
row=18,
col=0,
rowspan=1,
colspan=3,
)
parser.add_argument(
"--clipali",
type=int,
default=0,
help="""Boxsize to clip particles as part of preprocessing to speed up alignment. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary; still, there are some benefits from 'oversampling' the data during averaging; so you might still want an average of size 2x, but perhaps particles in a box of 1.5x are sufficiently good for alignment. In this case, you would supply --clipali=75""",
)
parser.add_argument(
"--savepreproc",
action="store_true",
default=False,
help="""Default=False. Will save stacks of preprocessed particles (one for coarse alignment and one for fine alignment if preprocessing options are different).""",
)
parser.add_argument(
"--average",
action="store_true",
default=False,
help="""Default=False. If supplied and a stack is provided through --input, the average of the aligned and/or symmetrized stack will also be saved.""",
)
parser.add_argument(
"--averager",
type=str,
default="mean.tomo",
help="""Default=mean.tomo. The type of averager used to produce the class average. Default=mean.tomo.""",
)
parser.add_argument(
"--keep",
type=float,
default=1.0,
help="""Fraction of particles to include if --average is on, after correlating the particles with the average.""",
)
parser.add_argument(
"--keepsig",
action="store_true",
default=False,
help="""Default=False. Causes theoptions.keep argument to be interpreted in standard deviations.""",
guitype="boolbox",
row=6,
col=1,
rowspan=1,
colspan=1,
mode="alignment,breaksym",
)
parser.add_argument(
"--avgiter",
type=int,
default=1,
help="""Default=1. If --keep is different from 1.0 and --average is on, the initial average will include all the particles, but then the percent specified byoptions.keep will be kept (the rest thrown away) and a new average will be computed. If --avgiter > 1, this new average will be compared again against all the particles. The procedure will be repeated for however many iterations --avgiter is given, or the process will stop automatically if in two consecutive rounds exactly the same particles are kept""",
)
parser.add_argument(
"--subset",
type=int,
default=0,
help="""Number of particles in a subset of particles from the --input stack of particles to run the alignments on.""",
)
parser.add_argument(
"--steps",
dest="steps",
type=int,
default=10,
help="""Number of steps (for the MC). Default=10.""",
guitype="intbox",
row=5,
col=1,
rowspan=1,
colspan=1,
)
parser.add_argument(
"--symmetrize",
default=False,
action="store_true",
help="""Symmetrize volume after alignment.""",
guitype="boolbox",
row=6,
col=0,
rowspan=1,
colspan=1,
)
parser.add_argument(
"--cmp",
type=str,
help="""The name of a 'cmp' to be used in comparing the symmtrized object to unsymmetrized""",
default="ccc",
guitype="comboparambox",
choicelist="re_filter_list(dump_cmps_list(),'tomo', True)",
row=7,
col=0,
rowspan=1,
colspan=2,
)
parser.add_argument(
"--parallel",
"-P",
type=str,
help="""Run in parallel, specify type:<option>=<value>:<option>:<value>""",
default=None,
guitype="strbox",
row=8,
col=0,
rowspan=1,
colspan=2,
)
parser.add_argument(
"--ppid", type=int, help="""Set the PID of the parent process, used for cross platform PPID.""", default=-1
)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help="""verbose level [0-9], higner number means higher level ofoptions.verboseness.""",
)
parser.add_argument(
"--nopath",
action="store_true",
default=False,
help="""If supplied, this option will save results in the directory where the command is run. A directory to store the results will not be made.""",
)
parser.add_argument(
"--nolog",
action="store_true",
default=False,
help="""If supplied, this option will prevent logging the command run in .eman2log.txt.""",
)
parser.add_argument(
"--saveali", action="store_true", default=False, help="""Save the stack of aligned/symmetrized particles."""
)
parser.add_argument(
"--savesteps",
action="store_true",
default=False,
help="""If --avgiter > 1, save all intermediate averages and intermediate aligned kept stacks.""",
)
parser.add_argument(
"--notmatchimgs",
action="store_true",
default=False,
help="""Default=True. This option prevents applying filter.match.to to one image so that it matches the other's spectral profile during preprocessing for alignment purposes.""",
)
parser.add_argument(
"--preavgproc1",
type=str,
default="",
help="""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)""",
)
parser.add_argument(
"--preavgproc2",
type=str,
default="",
help="""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)""",
)
parser.add_argument(
"--weighbytiltaxis",
type=str,
default="",
help="""Default=None. A,B, where A is an integer number and B a decimal. A represents the location of the tilt axis in the tomogram in pixels (eg.g, for a 4096x4096xZ tomogram, this value should be 2048), and B is the weight of the particles furthest from the tomogram. For example, --weighbytiltaxis=2048,0.5 means that praticles at the tilt axis (with an x coordinate of 2048) will have a weight of 1.0 during averaging, while the distance in the x coordinates of particles not-on the tilt axis will be used to weigh their contribution to the average, with particles at the edge(0+radius or 4096-radius) weighing 0.5, as specified by the value provided for B.""",
)
parser.add_argument(
"--weighbyscore",
action="store_true",
default=False,
help="""Default=False. This option will weigh the contribution of each subtomogram to the average by score/bestscore.""",
)
parser.add_argument(
"--align",
type=str,
default="symalignquat",
help="""Default=symalignquat. WARNING: The aligner cannot be changed for this program currently. Option ignored.""",
)
parser.add_argument(
"--tweak",
action="store_true",
default=False,
help="""WARNING: Not used for anything yet. This will perform a final alignment with no downsampling [without using --shrink or --shrinkfine] if --shrinkfine > 1.""",
)
(options, args) = parser.parse_args()
if not options.input:
parser.print_help()
sys.exit(0)
# If no failures up until now, initialize logger
log = 0
if not options.nolog:
logid = E2init(sys.argv, options.ppid)
log = 1
# inimodeldir = os.path.join(".",options.path)
# if not os.access(inimodeldir, os.R_OK):
# os.mkdir(options.path)
# Make directory to save results
from e2spt_classaverage import (
sptmakepath,
preprocessingprefft,
Preprocprefft3DTask,
get_results_preproc,
preprocfilter,
sptOptionsParser,
)
options = sptmakepath(options, "symsearch")
if options.nopath:
options.path = "."
rootpath = os.getcwd()
if rootpath not in options.path:
options.path = rootpath + "/" + options.path
if options.parallel:
from EMAN2PAR import EMTaskCustomer
options = sptOptionsParser(options)
avgr = Averagers.get(options.averager[0], options.averager[1])
resultsdict = {}
scores = []
outputstack = options.path + "/all_ptcls_ali.hdf"
# Determine number of particles in the stack
n = EMUtil.get_image_count(options.input)
if options.subset and options.subset < n:
n = options.subset
options.raw = options.input
if not options.nopreprocprefft:
if options.mask or options.normproc or options.threshold or options.clipali:
preprocprefftstack = options.path + "/" + os.path.basename(options.input).replace(".hdf", "_preproc.hdf")
# save "dummy" images for preproc images
for i in range(n):
dimg = EMData(8, 8, 8)
dimg.to_one()
dimg.write_image(preprocprefftstack, i)
originalsavepreproc = options.savepreproc
options.savepreproc = True
print "\n(e2spt_hac.py) (allvsall) Initializing parallelism for preprocessing"
if options.parallel: # Initialize parallelism if being used
# from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel)
pclist = [options.input]
etc.precache(pclist)
tasks = []
results = []
# preprocprefftstack = options.path + '/' + options.input.replace('.hdf','_preproc.hdf')
for i in range(n):
img = EMData(options.input, i)
if options.parallel:
task = Preprocprefft3DTask(["cache", options.input, i], options, i, preprocprefftstack)
tasks.append(task)
else:
pimg = preprocessingprefft(img, options)
pimg.write_image(preprocprefftstack, i)
print "\nthere are these many tasks to send", len(tasks)
if options.parallel and tasks:
tids = etc.send_tasks(tasks)
print "therefore these many tids", len(tids)
if options.verbose:
print "%d preprocessing tasks queued" % (len(tids))
results = get_results_preproc(etc, tids, options.verbose)
print "results are", results
options.input = preprocprefftstack
options.savepreproc = originalsavepreproc
for i in range(n):
print "\nI'll look for symmetry in particle number", i
# Load particle and make a copy to modify if preprocessing options are specified
volume = EMData(options.input, i)
preprocvol = volume.copy()
# Preprocess volume if any preprocessing options are specified
preprocprefftstack = options.path + "/" + os.path.basename(options.input).replace(".hdf", "_preproc.hdf")
if (
(options.shrink and options.shrink > 1)
or options.lowpass
or options.highpass
or options.normproc
or options.preprocess
or options.threshold
or options.clipali
):
print "\nHowever, I will first preprocess particle number", i
print "\nWill call preprocessing on ptcl", i
preprocvol = preprocfilter(preprocvol, options, i)
if options.savepreproc:
preprocvol.write_image(preprocprefftstack, i)
# preprocessing(s2image,options, ptclindx, savetagp ,'no',round)
print "\nDone preprocessing on ptcl", i
if options.parallel:
etc = EMTaskCustomer(options.parallel)
else:
etc = EMTaskCustomer("thread:1")
symalgorithm = SymALignStrategy(preprocvol, options.sym, options.steps, options.cmp, etc)
ret = symalgorithm.execute()
symxform = ret[0]
score = ret[1]
scores.append(score)
resultsdict.update({score: [symxform, i]})
print "\nWriting output for best alignment found for particle number", i
if options.shrink and options.shrink > 1:
trans = symxform.get_trans()
symxform.set_trans(trans[0] * options.shrink, trans[1] * options.shrink, trans[2] * options.shrink)
print "\nWrittng to output ptcl", i
# Rotate volume to the best orientation found, set the orientation in the header, apply symmetry if specified and write out the aligned (and symmetrized) particle to the output stack
output = volume.process("xform", {"transform": symxform})
output.set_attr("symxform", symxform)
print "\nApplying this transform to particle", symxform
if options.symmetrize:
output = output.process("xform.applysym", {"sym": options.sym})
output["spt_score"] = score
output.write_image(outputstack, -1)
# Averaging here only makes sense if all particles are going to be kept. Otherwise, different code is needed (below)
if options.average:
avgr.add_image(output)
# Finalize average of all particles if non were set to be excluded. Otherwise, determine the discrimination threshold and then average the particles that pass it.
if options.average:
final_avg = avgr.finish()
final_avg["origin_x"] = 0
final_avg["origin_y"] = 0 # The origin needs to be reset to ZERO to avoid display issues in Chimera
final_avg["origin_z"] = 0
final_avg["xform.align3d"] = Transform()
if options.keep == 1.0 and not options.keepsig:
final_avg.write_image(options.path + "/final_avg.hdf", 0)
if options.avgiter > 1:
print """WARNING: --avgiter > 1 must be accompanied by --keepsig, or by --keep < 1.0"""
elif options.keep < 1.0 or options.keepsig:
if options.ref:
ref = EMData(options.ref, 0)
refComp(options, outputstack, ref, resultsdict, "")
if options.mirror:
ref.process_inplace("xform.mirror", {"axis": options.mirror})
refComp(options, outputstack, ref, results, "_vs_mirror")
else:
ref2compare = final_avg
refComp(options, outputstack, final_avg, resultsdict, "")
del final_avg
if log:
E2end(logid)
return
def main():
#import pylab
#import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
progname = os.path.basename(sys.argv[0])
usage = """Produces mean intensity histograms of stack of sub-volumes"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--input",type=str,default='',help="""Default=None. Comma-separated stacks of images whose mean intensity distribution you want to plot.""")
parser.add_argument("--subset",type=int,default=0,help="""Default=0 (not used). N > 2 number of particles to from each stack provided through --input to consider.""")
parser.add_argument("--path",type=str,default='',help="Directory to store results in. The default is a numbered series of directories containing the prefix 'sptsim'; for example, sptsim_02 will be the directory by default if 'sptsim_01' already exists.")
#parser.add_argument("--output",type=str,default='',help="""Name of output plot if comparing two populations or more.""")
parser.add_argument("--shrink", type=int,default=1,help="Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount n > 1.")
parser.add_argument("--bins", type=int,default=0,help="""Default=0 (not used). Number of bins for histogram. If not provided, the optimal bin number will be automatically calculated based on bin-width, computed using Scott's normal reference rule, width = (3.5*std)/cuberoot(n), where 'std' is the standard deviation of the mean intensity distribution of population and n is the number of mean intensity values considered (this is affected by --removesigma). Then, bins will be nbins = (max(intensities) - min(intensities)) / width.""")
#parser.add_argument("--sym", type=str, default='c1', help = "Symmetry to enforce before computing mean intensity in the box. Note that this should only be used if the particles are properly aligned to the symmetry axis.")
parser.add_argument("--mask",type=str,default="mask.sharp:outer_radius=-2",help="Default=mask.sharp:outer_radius=-2. Mask processor applied to the particles before alignment. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py).")
parser.add_argument("--maskfile",type=str,default='',help="""Default=None. An image file containing an additional mask to apply besides --mask.""")
parser.add_argument("--clip",type=int,default=0,help="""Default=0 (not used). Boxsize to clip particles to before computing mean and standard deviation values for each image. (This can act as a mask, as you'd want to clip the boxes to a smaller size than their current, original size, excluding neighboring particles and background pixels/voxels).""")
parser.add_argument("--preprocess",type=str,default='',help="""Any processor to be applied to each image before computing mean and standard deviation values. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py).""")
parser.add_argument("--lowpass",type=str,default='',help="""Default=None. A lowpass filtering processor to be applied before computing mean and standard deviation values for each image. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py).""")
parser.add_argument("--highpass",type=str,default='',help="""Default=None. A highpass filtering processor to be applied before computing mean and standard deviation values for each image. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py).""")
parser.add_argument("--threshold",type=str,default='',help="""A thresholding processor to be applied before computing mean and standard deviation values for each image. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py).""")
parser.add_argument("--normproc",type=str,default="normalize.edgemean",help="""Default=normalize.edgemean. Normalization processor applied to particles before computing mean and standard deviation values for each iamge. If normalize.mask is used, --mask will be passed in automatically. If you want to turn normalization off specify \'None\'. (See 'e2help.py processors' at the command line for a list of processors that can be applied through e2proc3d.py).""")
parser.add_argument("--savepreprocessed",action="store_true",default=False,help="""Default=False. If provided, this option will save the image stacks in --input after all preprocessing options (lowpass, highpass, preprocess, masking, etc.) have been applied.""")
parser.add_argument("--normalizeplot",action="store_true",default=False,help="""Default=False. This will normalize the intensity values of the distribution to be between 0 and 1""")
parser.add_argument("--removesigma",type=int,default=0,help="""Default=0. Provide a value for the number of standard deviations away from the mean to consider values to exclude. For example, if --removesigma=3, values further than 3 standard deviations away from the mean will be excluded.""")
parser.add_argument("--ppid", type=int, help="Default=1. Set the PID of the parent process, used for cross platform PPID",default=-1)
parser.add_argument("--verbose", "-v", type=int, default=0, help="Default 0. Verbose level [0-9], higner number means higher level of verboseness",dest="verbose", action="store", metavar="n")
(options, args) = parser.parse_args()
logger = E2init(sys.argv, options.ppid)
'''
if options.mask:
options.mask=parsemodopt(options.mask)
if options.preprocess:
options.preprocess=parsemodopt(options.preprocess)
if options.lowpass:
options.lowpass=parsemodopt(options.lowpass)
if options.highpass:
options.highpass=parsemodopt(options.highpass)
if options.threshold:
options.threshold=parsemodopt(options.threshold)
if options.normproc:
options.normproc=parsemodopt(options.normproc)
'''
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser( options )
datafiles = options.input.split(',')
from e2spt_classaverage import sptmakepath
options = sptmakepath( options, 'meanintensityplots')
intensitiesSeveral = []
iwzSeveral = []
iminsSeveral = []
imaxsSeveral = []
istdsSeveral = []
means = []
stds = []
from e2spt_classaverage import writeParameters
cmdwp = writeParameters(options,'e2spt_meanintensityplot.py', 'sptmeanintensity')
for datafile in datafiles:
n = EMUtil.get_image_count(datafile)
if options.subset:
if options.subset < 3:
print "ERROR:Subset must be > 2."
sys.exit(1)
n = options.subset
if n < 3:
print "ERROR: All stacks must have at least 3 particles in them. This one doesn't:", datafile
sys.exit(1)
for datafile in datafiles:
ret = calcintensities( options, datafile )
intensitiesSingle = ret[0]
iwz = ret[1]
imins = ret[2]
imaxs = ret[3]
istds = ret[4]
intensitiesSeveral.append( [ datafile, list( intensitiesSingle ) ] )
iwzSeveral.append( [ datafile, list( iwz ) ] )
iminsSeveral.append( [ datafile, list( imins ) ] )
imaxsSeveral.append( [ datafile, list( imaxs ) ] )
istdsSeveral.append( [ datafile, list( istds ) ] )
intensitiesSingleNorm = intensitiesSingle
if options.normalizeplot:
intensitiesSingleNorm = normintensities( intensitiesSingle, 0, 0 )
#print "\]n\\n\nIntensities before plotting are", intensitiesSingleNorm
#print "\n\n\n\n\n"
ret = plotintensities( intensitiesSingleNorm, options, datafile )
mean = ret[0]
std = ret[1]
means.append(mean)
stds.append(std)
ret = plotintensities( iwz, options, datafile,'wz' )
ret = plotintensities( imins, options, datafile,'mins' )
ret = plotintensities( imaxs, options, datafile,'maxs' )
ret = plotintensities( istds, options, datafile,'stds' )
#print "\nIntensities several len is", len( intensitiesSeveral )
if len( intensitiesSeveral ) > 1:
datafile1 = intensitiesSeveral[0][0]
datafile2 = intensitiesSeveral[1][0]
intensities1 = intensitiesSeveral[0][1]
intensities2 = intensitiesSeveral[1][1]
n1 = len( intensities1 )
n2 = len( intensities2 )
zscore = ( means[0]-means[1] )/ np.sqrt( (stds[0]*stds[0])/n1 + (stds[1]*stds[1])/n2 )
g = open(options.path + '/MIboth_INFO.txt','w')
zscoreline = 'zscore=' + str(zscore)+' for ' + datafile1 + ' vs ' + datafile2 + ' \n'
lines=[ zscoreline ]
g.writelines(lines)
g.close()
print "\nzzzzzzz\n%s" %( zscoreline )
absmax = absmin = 0
if options.normalizeplot:
minses = []
maxes = []
for intenS in intensitiesSeveral:
minS = float(min( intenS[1] ))
maxS = float(max( intenS[1] ))
minses.append( minS )
maxes.append( maxS )
absmin = min( minses )
absmax = max( maxes ) - absmin
for intensities in intensitiesSeveral:
print "Type and len of intensities is", type(intensities[1]), len(intensities[1])
intensitiesNorm = intensities[1]
if options.normalizeplot:
print "Normalizeplot on"
intensitiesNorm = normintensities( intensities[1], absmin, absmax )
plotintensities( intensitiesNorm, options, datafile, 'no' )
plt.savefig(options.path + '/MIbothPlot.png')
plt.clf()
E2end(logger)