def __init__(self, options, fsp, sym, start, modeln=0, logger=None):
'''
@param options the options produced by (options, args) = parser.parse_args()
@param args the options produced by (options, args) = parser.parse_args()
@param logger and EMAN2 logger, i.e. logger=E2init(sys.argv)
assumes you have already called the check function.
'''
self.options = options
self.args = fsp
self.sym = sym
self.logger = logger
self.start = start
self.modeln = modeln
from EMAN2PAR import EMTaskCustomer
self.etc = EMTaskCustomer(options.parallel)
print "Precache ", fsp
self.etc.precache([fsp])
self.num_cpus = self.etc.cpu_est()
print self.num_cpus, " total CPUs available"
if self.num_cpus > 64: # upper limit
self.num_cpus = 64
self.__task_options = None
def __init__(self,options,args,logger=None): ''' @param options the options produced by (options, args) = parser.parse_args() @param args the options produced by (options, args) = parser.parse_args() @param logger and EMAN2 logger, i.e. logger=E2init(sys.argv) assumes you have already called the check function. ''' self.options = options self.args = args self.logger = logger from EMAN2PAR import EMTaskCustomer self.etc=EMTaskCustomer(options.parallel) if options.colmasks!=None : self.etc.precache([args[0],args[1],options.colmasks]) else : self.etc.precache([args[0],args[1]]) self.num_cpus = self.etc.cpu_est() if self.num_cpus < 32: # lower limit self.num_cpus = 32 self.__task_options = None
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_average.py [options]
Note that this program is not part of the original e2spt hierarchy, but is part of an experimental refactoring.
Will read metadata from the specified spt_XX directory, as produced by e2spt_align.py, and average a selected subset of subtomograms in the predetermined orientation.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--threads",
default=4,
type=int,
help=
"Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present."
)
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=-1)
parser.add_argument(
"--simthr",
default=-0.1,
type=float,
help=
"Similarity is smaller for better 'quality' particles. Specify the highest value to include from e2spt_hist.py. Default -0.1"
)
parser.add_argument(
"--keep",
default=-1,
type=float,
help="fraction of particles to keep. will overwrite simthr if set.")
parser.add_argument(
"--replace",
type=str,
default=None,
help=
"Replace the input subtomograms used for alignment with the specified file (used when the aligned particles were masked or filtered)"
)
parser.add_argument(
"--outfile",
type=str,
default=None,
help=
"Normally even/odd and overall outputs automatically generated. If specified, only overall file written with this specified filename. Suppresses postprocessing."
)
parser.add_argument(
"--wedgesigma",
type=float,
help=
"Threshold for identifying missing data in Fourier space in terms of standard deviation of each Fourier shell. Default 3.0",
default=3.0)
parser.add_argument(
"--minalt",
type=float,
help="Minimum alignment altitude to include. Default=0",
default=0)
parser.add_argument(
"--maxalt",
type=float,
help="Maximum alignment altitude to include. Deafult=180",
default=180)
parser.add_argument(
"--maxtilt",
type=float,
help=
"Explicitly zeroes data beyond specified tilt angle. Assumes tilt axis exactly on Y and zero tilt in X-Y plane. Default 90 (no limit).",
default=90.0)
parser.add_argument(
"--listfile",
type=str,
help=
"Specify a filename containing a list of integer particle numbers to include in the average, one per line, first is 0. Additional exclusions may apply.",
default=None)
parser.add_argument(
"--automaskexpand",
default=-1,
type=int,
help=
"Default=boxsize/20. Specify number of voxels to expand mask before soft edge. Use this if low density peripheral features are cut off by the mask.",
guitype='intbox',
row=12,
col=1,
rowspan=1,
colspan=1,
mode="refinement[-1]")
parser.add_argument(
"--symalimasked",
type=str,
default=None,
help=
"This will translationally realign each asymmetric unit to the specified (usually masked) reference "
)
parser.add_argument(
"--sym",
type=str,
default="c1",
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly."
)
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder containing current results (default = highest spt_XX)"
)
parser.add_argument("--skippostp",
action="store_true",
default=False,
help="Skip post process steps (fsc, mask and filters)")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default=None)
(options, args) = parser.parse_args()
if options.path == None:
fls = [
int(i[-2:]) for i in os.listdir(".")
if i[:4] == "spt_" and len(i) == 6 and str.isdigit(i[-2:])
]
if len(fls) == 0:
print("Error, cannot find any spt_XX folders")
sys.exit(2)
options.path = "spt_{:02d}".format(max(fls))
if options.verbose: print("Working in : ", options.path)
if options.iter < 0:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0:
print("Cannot find a {}/particle_parms* file".format(options.path))
sys.exit(2)
options.iter = max(fls)
if options.verbose: print("Using iteration ", options.iter)
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, options.iter))
else:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0:
print("Cannot find a {}/particle_parms* file".format(options.path))
sys.exit(2)
mit = max(fls)
if options.iter > mit:
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, mit))
print(
"WARNING: no particle_parms found for iter {}, using parms from {}"
.format(options.iter, mit))
else:
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, options.iter))
if options.listfile != None:
plist = set([int(i) for i in open(options.listfile, "r")])
NTHREADS = max(options.threads + 1,
2) # we have one thread just writing results
logid = E2init(sys.argv, options.ppid)
# jsd=Queue.Queue(0)
# filter the list of particles to include
keys = list(angs.keys())
if options.listfile != None:
keys = [i for i in keys if eval(i)[1] in plist]
if options.verbose:
print("{}/{} particles based on list file".format(
len(keys), len(list(angs.keys()))))
newkey = []
newang = {}
if options.keep > 0 and options.keep <= 1:
score = [float(angs[k]["score"]) for k in keys]
options.simthr = np.sort(score)[int(len(score) * options.keep) - 1]
print("Keeping {:.0f}% particles with score below {:.2f}".format(
options.keep * 100, options.simthr))
for k in keys:
val = angs[k]
if type(val) == list:
val = val[0]
if val["score"] <= options.simthr and inrange(
options.minalt, val["xform.align3d"].get_params("eman")["alt"],
options.maxalt):
newkey.append(k)
newang[k] = val
if options.verbose:
print("{}/{} particles after filters".format(len(newkey),
len(list(angs.keys()))))
keys = newkey
angs = newang
if options.parallel and options.symalimasked == None:
#print("running in mpi mode. This is experimental, so please switch back to threading if anything goes wrong...")
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_average.SptavgTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
data = [[], []] ## even/odd
if "eo" in angs[keys[0]]:
print("Reading even/odd subset from json file...")
for i, k in enumerate(keys):
src, ii = eval(k)[0], eval(k)[1]
eo = int(angs[k]["eo"])
data[eo].append([src, ii, angs[k]["xform.align3d"]])
else:
for i, k in enumerate(keys):
src, ii = eval(k)[0], eval(k)[1]
data[ii % 2].append([src, ii, angs[k]["xform.align3d"]])
#### check and save size of particle
fsp, i, xf = data[0][0]
b = EMData(fsp, i, True)
sz = options.boxsz = b["ny"]
avgs = []
for ieo, eo in enumerate(["even", "odd"]):
print("Averaging {}...".format(eo))
nbatch = min(len(data[ieo]) // 4, num_cpus)
tasks = [data[ieo][i::nbatch] for i in range(nbatch)]
print("{} particles in {} jobs".format(len(data[ieo]), len(tasks)))
tids = []
for t in tasks:
task = SptavgTask(t, options)
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if np.min(st_vals) == 100: break
time.sleep(5)
output = EMData(sz, sz, sz)
normvol = EMData((sz // 2 + 1) * 2, sz, sz)
output.to_zero()
output.do_fft_inplace()
avg = Averagers.get("mean")
normvol.to_zero()
for i in tids:
threed, norm = etc.get_results(i)[1]
#print(i, threed["mean"], threed["sigma"], norm["mean"], norm["sigma"])
#norm.div(len(tids))
threed.process_inplace("math.multamplitude", {"amp": norm})
avg.add_image(threed)
normvol.add(norm)
output = avg.finish()
normvol.process_inplace("math.reciprocal")
output.process_inplace("math.multamplitude", {"amp": normvol})
output.process_inplace("xform.phaseorigin.tocenter")
output.do_ift_inplace()
output.depad()
avgs.append(output)
ave, avo = avgs
else:
avg = [0, 0]
avg[0] = Averagers.get(
"mean.tomo",
{"thresh_sigma": options.wedgesigma}) #,{"save_norm":1})
avg[1] = Averagers.get("mean.tomo",
{"thresh_sigma": options.wedgesigma})
# Rotation and insertion are slow, so we do it with threads.
if options.symalimasked != None:
if options.replace != None:
print("Error: --replace cannot be used with --symalimasked")
sys.exit(1)
alimask = EMData(options.symalimasked)
thrds = [
threading.Thread(target=rotfnsym,
args=(avg[eval(k)[1] % 2], eval(k)[0],
eval(k)[1], angs[k]["xform.align3d"],
options.sym, alimask, options.maxtilt,
options.verbose))
for i, k in enumerate(keys)
]
else:
# Averager isn't strictly threadsafe, so possibility of slight numerical errors with a lot of threads
if options.replace != None:
thrds = [
threading.Thread(target=rotfn,
args=(avg[eval(k)[1] % 2],
options.replace, eval(k)[1],
angs[k]["xform.align3d"],
options.maxtilt, options.verbose))
for i, k in enumerate(keys)
]
else:
thrds = [
threading.Thread(target=rotfn,
args=(avg[eval(k)[1] % 2], eval(k)[0],
eval(k)[1],
angs[k]["xform.align3d"],
options.maxtilt, options.verbose))
for i, k in enumerate(keys)
]
print(len(thrds), " threads")
thrtolaunch = 0
while thrtolaunch < len(thrds) or threading.active_count() > 1:
# If we haven't launched all threads yet, then we wait for an empty slot, and launch another
# note that it's ok that we wait here forever, since there can't be new results if an existing
# thread hasn't finished.
if thrtolaunch < len(thrds):
while (threading.active_count() == NTHREADS):
time.sleep(.1)
if options.verbose:
print("Starting thread {}/{}".format(
thrtolaunch, len(thrds)))
thrds[thrtolaunch].start()
thrtolaunch += 1
else:
time.sleep(1)
#while not jsd.empty():
#fsp,n,ptcl=jsd.get()
#avg[n%2].add_image(ptcl)
for t in thrds:
t.join()
ave = avg[0].finish() #.process("xform.phaseorigin.tocenter").do_ift()
avo = avg[1].finish() #.process("xform.phaseorigin.tocenter").do_ift()
# impose symmetry on even and odd halves if appropriate
if options.sym != None and options.sym.lower(
) != "c1" and options.symalimasked == None:
ave.process_inplace("xform.applysym", {
"averager": "mean.tomo",
"sym": options.sym
})
avo.process_inplace("xform.applysym", {
"averager": "mean.tomo",
"sym": options.sym
})
av = ave + avo
av.mult(0.5)
if options.outfile:
av.write_image(options.outfile)
sys.exit(0)
evenfile = "{}/threed_{:02d}_even.hdf".format(options.path, options.iter)
oddfile = "{}/threed_{:02d}_odd.hdf".format(options.path, options.iter)
combfile = "{}/threed_{:02d}.hdf".format(options.path, options.iter)
ave.write_image(evenfile, 0)
avo.write_image(oddfile, 0)
av.write_image(combfile, 0)
cmd = "e2proc3d.py {evenfile} {path}/fsc_unmasked_{itr:02d}.txt --calcfsc={oddfile}".format(
path=options.path,
itr=options.iter,
evenfile=evenfile,
oddfile=oddfile)
launch_childprocess(cmd)
#### skip post process in case we want to do this elsewhere...
if options.skippostp:
E2end(logid)
return
# final volume at this point is Wiener filtered
launch_childprocess(
"e2proc3d.py {combfile} {combfile} --process=filter.wiener.byfsc:fscfile={path}/fsc_unmasked_{itr:02d}.txt:snrmult=2"
.format(path=options.path, itr=options.iter, combfile=combfile))
# New version of automasking based on a more intelligent interrogation of the volume
vol = EMData(combfile)
nx = vol["nx"]
apix = vol["apix_x"]
md = vol.calc_radial_dist(old_div(nx, 2), 0, 1,
3) # radial max value per shell in real space
rmax = int(old_div(nx, 2.2)) # we demand at least 10% padding
vmax = max(md[:rmax]) # max value within permitted radius
# this finds the first radius where the max value @ r falls below overall max/4
# this becomes the new maximum mask radius
act = 0
mv = 0, 0
for i in range(rmax):
if md[i] > mv[0]:
mv = md[i], i # find the radius of the max val in range
if not act and md[i] < 0.9 * vmax: continue
act = True
if md[i] < 0.2 * vmax:
rmax = i
break
rmaxval = mv[1]
vmax = mv[0]
# excludes any spurious high values at large radius
vol.process_inplace("mask.sharp", {"outer_radius": rmax})
# automask
mask = vol.process(
"mask.auto3d", {
"threshold": vmax * .15,
"radius": 0,
"nshells":
int(nx * 0.05 + 0.5 + old_div(20, apix)) + options.automaskexpand,
"nmaxseed": 24,
"return_mask": 1
})
mask.process_inplace("filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, (40.0))})
mask.write_image("{path}/mask.hdf".format(path=options.path), 0)
# compute masked fsc and refilter
ave.mult(mask)
ave.write_image("{path}/tmp_even.hdf".format(path=options.path), 0)
avo.mult(mask)
avo.write_image("{path}/tmp_odd.hdf".format(path=options.path), 0)
av.mult(mask)
av.write_image(combfile, 0)
cmd = "e2proc3d.py {path}/tmp_even.hdf {path}/fsc_masked_{itr:02d}.txt --calcfsc={path}/tmp_odd.hdf".format(
path=options.path, itr=options.iter)
launch_childprocess(cmd)
# final volume is premasked and Wiener filtered based on the masked FSC
launch_childprocess(
"e2proc3d.py {combfile} {combfile} --process=filter.wiener.byfsc:fscfile={path}/fsc_masked_{itr:02d}.txt:snrmult=2"
.format(path=options.path, itr=options.iter, combfile=combfile))
E2end(logid)
def main():
usage = """e2classifytree.py <projection> <particle> [options]
Classify particles using a binary tree. Can be used as an alternative for e2simmx2stage.py + e2classify.py.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--threads", type=int, help="", default=12)
parser.add_argument("--nodes", type=str, help="", default="nodes.hdf")
#parser.add_argument("--clsmx", type=str,help="", default="clsmx.hdf")
parser.add_argument("--output", type=str, help="", default="clsmx.hdf")
parser.add_argument(
"--align",
type=str,
help="The name of an 'aligner' to use prior to comparing the images",
default=None)
parser.add_argument(
"--aligncmp",
type=str,
help="Name of the aligner along with its construction arguments",
default="dot")
parser.add_argument(
"--ralign",
type=str,
help=
"The name and parameters of the second stage aligner which refines the results of the first alignment",
default=None)
parser.add_argument(
"--raligncmp",
type=str,
help=
"The name and parameters of the comparitor used by the second stage aligner. Default is dot.",
default="dot")
parser.add_argument(
"--cmp",
type=str,
help="The name of a 'cmp' to be used in comparing the aligned images",
default="dot:normalize=1")
parser.add_argument(
"--cmpdiff",
action="store_true",
default=False,
help="Compare using the difference of the two children")
parser.add_argument(
"--incomplete",
type=int,
help="The degree of incomplete allowed in the tree on each level",
default=0)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument("--parallel",
default=None,
help="parallelism argument")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
(options, args) = parser.parse_args()
E2n = E2init(sys.argv, options.ppid)
options.align = parsemodopt(options.align)
options.aligncmp = parsemodopt(options.aligncmp)
options.ralign = parsemodopt(options.ralign)
options.raligncmp = parsemodopt(options.raligncmp)
options.cmp = parsemodopt(options.cmp)
projs = args[0]
#projsimmx=args[1]
ptcl = args[1]
npj = EMUtil.get_image_count(projs)
npt = EMUtil.get_image_count(ptcl)
if options.parallel == None:
par = "thread:{:d}".format(options.threads)
else:
par = options.parallel
### Build tree
### always overwrite the tree here now
#if not os.path.isfile(options.nodes):
print("Building binary tree...")
buildtree(projs, par, options.nodes, options.incomplete, options.verbose)
#else:
#print "Using existing tree..."
## Generate children pairs for comparison
print("Generating children pairs for comparison...")
if options.cmpdiff:
nodepath = os.path.dirname(options.nodes)
masktmp = '/'.join([nodepath, "tmp_msk.hdf"])
if os.path.isfile(masktmp): os.remove(masktmp)
cmptmp = '/'.join([nodepath, "tmp_cmp.hdf"])
if os.path.isfile(cmptmp):
os.remove(cmptmp)
makechildpair(options.nodes, cmptmp, masktmp)
else:
masktmp = None
cmptmp = None
E2progress(E2n, 0.5)
#exit()
print("Starting classification...")
### Classify particles
clsmx = [EMData(1, npt) for i in range(7)]
nnod = EMUtil.get_image_count(options.nodes)
if options.parallel:
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2classifytree.TreeClassifyTask")
tasks = []
step = 50
tt = [list(range(i, i + step)) for i in range(0, npt - step, step)]
tt.append(list(range(tt[-1][-1] + 1, npt)))
for it in tt:
tasks.append(
TreeClassifyTask(ptcl, it, options.nodes, options.align,
options.aligncmp, options.cmp, options.ralign,
options.raligncmp, cmptmp, masktmp))
taskids = etc.send_tasks(tasks)
ptclpernode = [0 for i in range(nnod)]
nfinished = 0
while len(taskids) > 0:
haveprogress = False
time.sleep(3)
curstat = etc.check_task(taskids)
for i, j in enumerate(curstat):
if j == 100:
haveprogress = True
rslt = etc.get_results(taskids[i])
rslt = rslt[1]
for r in rslt:
nfinished += 1
if options.verbose > 0:
print("Particle:", r["id"], "\tnodes:",
r["choice"])
for c in r["choice"]:
ptclpernode[c] += 1
clsmx[0].set_value_at(0, r["id"], r["cls"])
for nt in range(1, 7):
clsmx[nt].set_value_at(0, r["id"], r["simmx"][nt])
taskids = [j for i, j in enumerate(taskids) if curstat[i] != 100]
if haveprogress: print("{:d}/{:d} finished".format(nfinished, npt))
E2progress(E2n, 0.5 + old_div(float(nfinished), npt))
for i in range(nnod):
ndtmp = EMData(options.nodes, i, True)
ndtmp["tree_nptls"] = ptclpernode[i]
ndtmp.write_image(options.nodes, i)
else:
### To record the number of particles in each branch of the tree
for i in range(nnod):
ndtmp = EMData(options.nodes, i, True)
ndtmp["tree_nptls"] = 0
ndtmp.write_image(options.nodes, i)
t = {}
clsmx = [EMData(1, npt) for i in range(7)]
for i in range(options.threads):
ai = [x for x in range(npt) if x % options.threads == i]
t[i] = threading.Thread(target=classify,
args=(ptcl, ai, options.nodes, clsmx,
options.align, options.aligncmp,
options.cmp, options.ralign,
options.raligncmp, cmptmp, masktmp))
t[i].start()
for i in range(options.threads):
t[i].join()
if os.path.isfile(options.output):
os.remove(options.output)
for i in clsmx:
i.write_image(options.output, -1)
if options.cmpdiff:
os.remove(cmptmp)
os.remove(masktmp)
print("Finished~")
E2progress(E2n, 1.0)
E2end(E2n)
def main():
parser = EMArgumentParser(usage=get_usage())
parser.add_argument("--output", default="threed.hdf", help="Output reconstructed volume file name.")
parser.add_argument("--input", default=None, help="The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion")
parser.add_argument("--input_model", default=None, help="If the class-averages have the model_id parameter (produced by e2refinemulti.py), this will use only class-averages with the specified model_id for the reconstruction.")
parser.add_argument("--tlt", help="An imod tlt file containing alignment angles. If specified slices will be inserted using these angles in the IMOD convention", type=str, default=None)
parser.add_argument("--sym", dest="sym", default="c1", help="Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct.")
parser.add_argument("--fillangle", type=float, dest="fillangle", help="An angular range used for both alt & az over which the projection should be averaged. Generally the angular step used when making projections.",default=0)
parser.add_argument("--pad", metavar="x or x,y", default=None,type=str, help="Will zero-pad images to the specifed size (x,y) or (x,x) prior to reconstruction. If not specified or 0 no padding occurs. If a negative value is specified automatic padding is performed. ")
parser.add_argument("--padvol", metavar="x or x,y,z", default=None,type=str, help="Defines the dimensions (x,y,z) or (x,x,x) of the reconstructed volume. If ommitted, implied value based on padded 2D images is used.")
parser.add_argument("--outsize", metavar="x or x,y,z", default=None, type=str, help="Defines the dimensions (x,y,z) or (x,x,x) of the final volume written to disk, if ommitted, size will be based on unpadded input size")
#parser.add_argument("--clipz", default=None, type=int, help="Extract a specified number of central z-slices. Option disabled by default.")
parser.add_argument("--savenorm", default=None, type=str, help="If set, will save the normalization volume showing Fourier space filling to the specified file")
parser.add_argument("--keep", type=float, dest="keep", help="The fraction of slices to keep, based on quality scores (1.0 = use all slices). See keepsig.",default=1.0)
parser.add_argument("--keepsig",action="store_true",default=False, dest="keepsig", help="If set, keep will be interpreted as a standard deviation coefficient instead of as a percentage.")
parser.add_argument("--keepabs",action="store_true",default=False, dest="keepabs", help="If set, keep will refer to the absolute quality of the class-average, not a local quality relative to other similar sized classes.")
parser.add_argument("--altedgemask",action="store_true",default=False, dest="altedgemask", help="If set, assumes tomographic data with a thin specimen, and masks out the +-X edges based on the alt Euler angle.")
parser.add_argument("--no_wt", action="store_true", dest="no_wt", default=False, help="This argument turns automatic weighting off causing all images to be weighted by 1. If this argument is not specified images inserted into the reconstructed volume are weighted by the number of particles that contributed to them (i.e. as in class averages), which is extracted from the image header (as the ptcl_repr attribute).")
parser.add_argument("--sqrt_wt", action="store_true", default=False, help="Normally class-averages are weighted into the reconstruction based on the number of particles in the average. This option causes the sqrt of the number of particles to be used instead.")
parser.add_argument("--iterative", action="store_true", default=False, help="Uses iterative interpolation in Fourier space instead of single step gridding or interpolation. --mode and --usessnr are ignored with this option.")
parser.add_argument("--itermask",type=str, default = None, help="Used in iterative mode to define a mask to apply between iterations. The resulting reconstruction won't be masked per-se, but will be biased towards lower densities outside the mask.")
parser.add_argument("--usessnr", action="store_true", default=False, help="Makes use of the class_ssnr header data to weight each slice during insertion, instead of the default behavior of just using the number of particles in the average as a global weight.")
parser.add_argument("--mode", type=str, default="gauss_var", help="Fourier reconstruction 'mode' to use. The default should not normally be changed. default='gauss_var'")
parser.add_argument("--noradcor", action="store_true",default=False, help="Normally a radial correction will be applied based on the --mode used. This option disables that correction.")
parser.add_argument("--seedmap",type=str, default = None, help="If specified this volume will be used as a starting point for the reconstruction, filling any missing values in Fourier space. experimental.")
parser.add_argument("--seedweight", type=float, default=1.0, help="If seedmap specified, this is how strongly the seedmap will bias existing values. 1 is default, and is equivalent to a one particle weight.")
parser.add_argument("--seedweightmap", type=str, default=None, help="Specify a full map of weights for the seed. This must be in the same format as the --savenorm output map.")
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--threads", default=4,type=int,help="Number of threads to run in parallel on a single computer. This is the only parallelism supported by e2make3dpar", guitype='intbox', row=24, col=2, rowspan=1, colspan=1, mode="refinement")
parser.add_argument("--preprocess", metavar="processor_name(param1=value1:param2=value2)", type=str, action="append", help="preprocessor to be applied to the projections prior to 3D insertion. There can be more than one preprocessor and they are applied in the order in which they are specifed. Applied before padding occurs. See e2help.py processors for a complete list of available processors.")
parser.add_argument("--setsf",type=str,help="Force the structure factor to match a 'known' curve prior to postprocessing (<filename>, auto or none). default=none",default="none")
parser.add_argument("--postprocess", metavar="processor_name(param1=value1:param2=value2)", type=str, action="append", help="postprocessor to be applied to the 3D volume once the reconstruction is completed. There can be more than one postprocessor, and they are applied in the order in which they are specified. See e2help.py processors for a complete list of available processors.")
parser.add_argument("--apix",metavar="A/pix",type=float,help="A/pix value for output, overrides automatic values",default=None)
parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
# Database Metadata storage
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
(options, args) = parser.parse_args()
# options.fillangle=options.fillangle*pi/180.0
# make sure that the user has atleast specified an input file
# if len(args) <1:
# parser.error("No input file specified")
# options.input = args[0]
if (not options.keep and not options.keepsig):
print("Warning, neither the keep nor the keepsig argument was specified. Setting keep=1 (keeping 100% of inserted slices)")
options.keep=1
if options.input_model!=None : options.input_model=int(options.input_model)
print("e2make3dpar.py")
logger=E2init(sys.argv,options.ppid)
# update the no_wt parameter
if options.sqrt_wt : options.no_wt=2
elif options.no_wt : options.no_wt=1
else : options.no_wt=0
# get basic image parameters
tmp=EMData()
tmp.read_image(options.input,0,True)
if options.no_wt!=1 :
try:
n=1
while tmp["ptcl_repr"]==0 :
tmp.read_image(options.input,n,True)
n+=1
except: pass
nx=tmp["nx"]
ny=tmp["ny"]
nslice=tmp["nz"]
if options.apix!=None : apix=options.apix
else : apix=tmp["apix_x"]
if options.verbose>0: print("Image dimensions %d x %d"%(nx,ny))
# parse the padding options, to make sure we have a 2 or 3 tuple for each
try :
if options.pad==None or options.pad==0: options.pad=(max(nx,ny),max(nx,ny))
elif options.pad[0]=='-':
sz=max(nx,ny)
sz=good_size(sz*1.25)
options.pad=(sz,sz)
print("padding to {}x{}".format(sz,sz))
elif "," in options.pad :
s=options.pad.split(",")
options.pad=(int(s[0]),int(s[1]))
else : options.pad=(int(options.pad),int(options.pad))
if options.verbose>0 : print("Pad to %d x %d"%(options.pad[0],options.pad[1]))
except:
print("Couldn't parse pad option :",options.pad)
exit(1)
try :
if options.padvol==None : padvol=(max(options.pad),max(options.pad),max(options.pad))
elif "," in options.padvol :
s=options.padvol.split(",")
padvol=(int(s[0]),int(s[1]),int(s[2]))
else : padvol=(int(options.padvol),int(options.padvol),int(options.padvol))
if options.verbose>0 : print("Padded volume to reconstruct %d x %d x %d"%(padvol[0],padvol[1],padvol[2]))
except:
print("Couldn't parse padvol option :",options.padvol)
exit(1)
try :
if options.outsize==None : outsize=(max(nx,ny),max(nx,ny),max(nx,ny))
elif "," in options.outsize :
s=options.outsize.split(",")
outsize=(int(s[0]),int(s[1]),int(s[2]))
else : outsize=(int(options.outsize),int(options.outsize),int(options.outsize))
if options.verbose>0 : print("Final output volume %d x %d x %d"%(outsize[0],outsize[1],outsize[2]))
except:
print("Couldn't parse outsize option :",options.outsize)
exit(1)
data=initialize_data(options.input,options.input_model,options.tlt,options.pad,options.no_wt,options.preprocess)
# Filter out averages/images which aren't good enough
if options.verbose: print("Filtering data, %d images ->"%len(data), end=' ')
# we have an absolute threshold
if options.keepabs: thr=options.keep
else :
quals=np.array(sorted([i["quality"] for i in data]))
if np.std(quals)==0 : thr=max(quals)+1.0
else :
if options.keepsig:
thr=np.mean(quals)+options.keep*np.std(quals)
else:
try:
thr=quals[int(floor(len(quals)*options.keep))]
except: thr=max(quals)+1.0
excluded=[max(i["fileslice"],i["filenum"]) for i in data if i["quality"]>thr]
included=[max(i["fileslice"],i["filenum"]) for i in data if i["quality"]<=thr]
data=[i for i in data if i["quality"]<=thr]
ptclcount=sum([i["weight"] for i in data])
if options.verbose: print("After filter, %d images"%len(data))
if options.itermask:
itmask=EMData(options.itermask)
itmask=itmask.get_clip(Region(old_div((itmask["nx"]-options.pad[1]),2),old_div((itmask["ny"]-options.pad[1]),2),old_div((itmask["nz"]-options.pad[1]),2),options.pad[1],options.pad[1],options.pad[1]))
# Get the reconstructor and initialize it correctly
if options.iterative :
a = {"size":padvol,"sym":options.sym,"verbose":options.verbose-1}
if options.savenorm!=None : a["savenorm"]=options.savenorm
recon=Reconstructors.get("fourier_iter", a)
niter=4
else :
a = {"size":padvol,"sym":options.sym,"mode":options.mode,"usessnr":options.usessnr,"verbose":options.verbose-1}
if options.savenorm!=None : a["savenorm"]=options.savenorm
recon=Reconstructors.get("fourier", a)
niter=1
#########################################################
# The actual reconstruction
options.padvol3=padvol
if options.parallel!=None:
par=options.parallel.split(':')
if par[0].startswith("thread"):
#options.parallel=None
options.threads=int(par[1])
elif par[0]=="mpi":
nthr=int(par[1])
ppt=len(data)/nthr
if ppt>16:
print("Inserting {:.1f} images per thread...".format(ppt))
### prepare some options
else:
print("Too few images in input ({:.1f} images per thread). Switching back to threading".format(ppt))
options.parallel=None
options.threads=nthr
else:
options.parallel=None
if options.seedmap!=None :
seed=EMData(options.seedmap)
seed.clip_inplace(Region(old_div((nx-padvol[0]),2),old_div((ny-padvol[1]),2),old_div((nslice-padvol[2]),2),padvol[0],padvol[1],padvol[2]))
seed.do_fft_inplace()
else:
seed=None
for it in range(niter):
if options.parallel:
print("running in mpi mode. This is experimental, so please switch back to threading if anything goes wrong...")
if it>0:
seed=output
from EMAN2PAR import EMTaskCustomer
etc=EMTaskCustomer(options.parallel, module="e2make3dpar.Make3dTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
tasks=[data[i::num_cpus] for i in range(num_cpus)]
print("{} jobs".format(len(tasks)))
tids=[]
for t in tasks:
task = Make3dTask(t, seed, options)
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
#print(tids)
if np.min(st_vals) == 100: break
time.sleep(5)
#dics=[0]*nptcl
output=EMData(padvol[0], padvol[1], padvol[2])
normvol=EMData(padvol[0]//2+1, padvol[1], padvol[2])
output.to_zero()
output.do_fft_inplace()
normvol.to_zero()
for i in tids:
threed, norm=etc.get_results(i)[1]
threed.process_inplace("math.multamplitude", {"amp":norm})
output.add(threed)
normvol.add(norm)
normvol.process_inplace("math.reciprocal")
output.process_inplace("math.multamplitude", {"amp":normvol})
output.do_ift_inplace()
output.depad()
output.process_inplace("xform.phaseorigin.tocenter")
del etc
else:
threads=[threading.Thread(target=reconstruct,args=(data[i::options.threads],recon,options.preprocess,options.pad,
options.fillangle,options.altedgemask,max(options.verbose-1,0),options.input.endswith(".lst"))) for i in range(options.threads)]
if it==0:
if options.seedmap!=None :
#seed=EMData(options.seedmap)
# seed.process_inplace("normalize.edgemean")
#seed.clip_inplace(Region(old_div((nx-padvol[0]),2),old_div((ny-padvol[1]),2),old_div((nslice-padvol[2]),2),padvol[0],padvol[1],padvol[2]))
#seed.do_fft_inplace()
if options.seedweightmap==None:
recon.setup_seed(seed,options.seedweight)
else:
seedweightmap=EMData(seedweightmap,0)
recon.setup_seedandweights(seed,seedweightmap)
else : recon.setup()
else:
recon.setup_seed(output,1.0)
for i,t in enumerate(threads):
if options.verbose>1: print("started thread ",i)
t.start()
for t in threads: t.join()
# output = recon.finish(it==niter-1) # only return real-space on the final pass
output = recon.finish(True)
if options.verbose:
print("Iteration ",it)
if options.verbose>2: output.write_image("it{}.hdf".format(it))
#if options.verbose>2 and it!=niter-1:
#output.process("xform.phaseorigin.tocenter").do_ift().write_image("it{}.hdf".format(it))
if it<niter-1 :
if options.itermask:
output.mult(itmask)
else:
output.process_inplace("threshold.compress",{"range":output["sigma"],"value":output["mean"],"clamponly":True})
if options.verbose:
print("Iteration ",it)
output.write_image("it{}.hdf".format(it))
#if options.verbose>2 and it!=niter-1:
#output.process("xform.phaseorigin.tocenter").do_ift().write_image("it{}.hdf".format(it))
if options.verbose>0 : print("Finished Reconstruction")
try:
output.set_attr("ptcl_repr",ptclcount)
if len(included)>0 : output.set_attr("threed_ptcl_idxs",included)
if len(excluded)>0 : output.set_attr("threed_excl_ptcl_idxs",excluded)
output.set_attr("threed_ptcl_src",data[0]["filename"])
except:
print("Warning, error setting reconstruction attributes")
# progress += 10
# E2progress(logid,float(progress)/total_progress)
#
########################################################3
# Correct for Gaussian falloff
if not options.noradcor :
cor={"gauss_2":4.0,"gauss_3":6.4,"gauss_4":8.8,"gauss_5":10.4,"gauss_5_slow":10.4} # Gaussian widths for different reconstruction modes
if options.mode in cor :
output.process_inplace("math.gausskernelfix",{"gauss_width":cor[options.mode]})
elif options.mode in ["gridding_5","gridding_7"]:
output.process_inplace("math.gridkernelfix",{"mode":options.mode})
else:
print("Warning: no radial correction applied for this mode")
# clip to the requested final dimensions
if output["nx"]!=outsize[0] or output["ny"]!=outsize[1] or output["nz"]!=outsize[2] :
output.clip_inplace(Region(old_div((output["nx"]-outsize[0]),2),old_div((output["ny"]-outsize[1]),2),old_div((output["nz"]-outsize[2]),2), outsize[0],outsize[1],outsize[2]))
#if options.clipz != None:
# output.clip_inplace(Region(0,0,(output["nz"]-options.clipz)//2, output["nx"], output["ny"], options.clipz))
if options.apix!=None : apix=options.apix
output["apix_x"]=apix
output["apix_y"]=apix
output["apix_z"]=apix
# Structure factor setting
if options.setsf.lower() != "none" :
if options.setsf.lower()!="auto" :
try:
sfcurve=XYData()
sfcurve.read_file(options.setsf)
for i in range(sfcurve.get_size()):
v=sfcurve.get_y(i)
if v<=0 :
print("Warning values <=0 found in structure factor file. Please remove.")
sfcurve.update()
except:
print("ERROR: Specified structure factor ({}) not found.".format(options.setsf))
sys.exit(1)
else:
try:
sfcurve=XYData()
sfcurve.read_file("strucfac.txt")
for i in range(sfcurve.get_size()):
v=sfcurve.get_y(i)
if v<=0 :
print("Warning values <=0 found in structure factor file. Please remove.")
sfcurve.update()
except : sfcurve=None
if sfcurve==None:
print("ERROR : Structure factor read failed. Not applying structure factor")
else:
# need to be really careful about the corners
# for i in range(sfcurve.get_size()):
# if sfcurve.get_x(i)>1.0/(2.0*apix) : sfcurve.set_y(i,0.0)
output.process_inplace("filter.setstrucfac",{"apix":apix,"strucfac":sfcurve})
# output.process_inplace("filter.lowpass.tophat",{"apix":apix,"cutoff_abs":0.49})
if options.postprocess != None:
for p in options.postprocess:
try:
(processorname, param_dict) = parsemodopt(p)
if not param_dict : param_dict={}
output.process_inplace(str(processorname), param_dict)
except:
print("warning - application of the post processor",p," failed. Continuing anyway")
# output.process_inplace("normalize.circlemean")
# just remove the output file
if file_exists(options.output):
remove_file(options.output)
# write the reconstruction to disk
output.write_image(options.output,0)
if options.verbose>0:
print("Output File: "+options.output)
E2end(logger)
print("Exiting")
def main():
parser = EMArgumentParser(usage=get_usage())
parser.add_argument("--tiltseries",
default=None,
help="""The input projections.
Project should usually have the xform.projection header attribute, which is
used for slice insertion""")
parser.add_argument("--tltfile",
type=str,
default=None,
help="""An IMOD-like .tlt file containing
alignment angles. If specified slices will be inserted using these angles in the
IMOD convention""")
parser.add_argument("--output",
default="threed.hdf",
help="""Output reconstructed
tomogram file name.""")
parser.add_argument("--path",
type=str,
default='tvrecon_3d',
help="""Directory in which
results will be stored.""")
parser.add_argument("--iter",
default=10,
type=int,
help="""Specify the number of
iterative reconstructions to complete before returning the final reconstructed volume.
The default number is 50.""")
parser.add_argument("--beta",
default=1.0,
type=float,
help="""Specify the total-variation
penalization/regularization weight parameter 'beta'. The default is 5.0.""")
parser.add_argument("--subpix",
default=1,
type=int,
help="""Specify the number of linear
subdivisions used to compute the projection of one image pixel onto a detector pixel."""
)
parser.add_argument("--savesinograms",
action="store_true",
default=False,
help="""If provided,
this option will save the sinogram for each 2-D slice (along Y) in the reconstruction
to disk.""")
parser.add_argument("--inmemory",
action='store_true',
default=False,
help="""If provided,
this option will keep certain files open in memory instead of writing them and
reading from disk every time. While this can be faster, it is very memory-intensive."""
)
parser.add_argument("--saveslices",
action="store_true",
default=False,
help="""If provided,
this option will save each reconstructed 2-D slice (along Y) to disk.""")
parser.add_argument("--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help="""
verbose level [0-9], higher number means higher level of verboseness.""")
parser.add_argument("--parallel",
type=str,
default='thread:1',
help="""Default=thread:1.
See http://blake.bcm.edu/emanwiki/EMAN2/Parallel""")
parser.add_argument("--ppid",
type=int,
help="""Set the PID of the parent process,
used for cross platform PPID.""",
default=-1)
(options, args) = parser.parse_args()
#Check that the minimum data required are available and sane, otherwise exit
if not options.tiltseries:
print("\nERROR: You must specficy --tiltseries")
sys.exit(1)
if not options.tltfile:
print("\nERROR: You must specficy --tlt")
sys.exit(1)
if options.beta < 0.0:
print("\nERROR: Parameter beta must be a positive, real number.")
sys.exit(1)
#Parse and count tilt angles
tiltangles = np.asarray([float(i) for i in open(options.tltfile, "r")])
tiltangles = tiltangles.tolist()
nimgs = EMUtil.get_image_count(options.tiltseries)
nangles = len(tiltangles)
if nimgs != nangles:
print(
"""\nERROR: The number of images in the tiltseries, %d, does not match
the number of angles in the tlt file, %d""" % (nimgs, nangles))
sys.exit(1)
#Read essential info from image header
hdr = EMData(options.tiltseries, 0, True)
apix = hdr['apix_x']
xsize = hdr['nx']
ysize = hdr['ny']
#Once all parameters and data have passed wholesomeness checks, initialize logging
logger = E2init(sys.argv, options.ppid)
#Create new output directory for this run of the program
options = makepath(options, options.path)
if options.verbose > 2:
print("\nGenerating this new directory to save results to:",
options.path)
options.path = os.getcwd() + "/" + options.path
#Generate one projection operator for all 2D slice reconstructions
if options.verbose:
print("\nBuilding projection operator...")
projection_operator = build_projection_operator(options, tiltangles, xsize,
nimgs, None, 0, None)
#Initialize parallelism
if options.verbose:
print("\n\n(e2tvrecon.py) INITIALIZING PARALLELISM\n\n")
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel, "e2tvrecon.TVReconTask")
tasks = []
nimgs = len(tiltangles)
for y in range(ysize):
task = TVReconTask(options, xsize, ysize, y, projection_operator,
tiltangles, nimgs)
tasks.append(task)
tids = etc.send_tasks(tasks)
results = get_results(etc, tids, options)
if options.verbose:
print(
"\nThese many results %d were computed because there were these many tasks %d"
% (len(results), len(tasks)))
results.sort()
np_recons = []
for i in range(len(results)):
recon = results[i][-1]
# Store 2D reconstructions in options.path if requested
if options.saveslices:
twodpath = options.path + "/slices.hdf"
from_numpy(recon).write_image(twodpath, i)
np_recons.append(recon)
reconstack = np.dstack(np_recons)
threed_recon = from_numpy(reconstack)
threed_recon['apix_x'] = apix
threed_recon['apix_y'] = apix
threed_recon['apix_z'] = apix
threed_recon.rotate(0, -90, -90)
threed_recon.write_image(options.path + '/' + options.output, 0)
E2end(logger)
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options]
This program will take a set of reference-free class-averages (or other projections) and generate a set of possible
3-D initial models. It does this by heavily downsampling the data, then running a number of very fast, full iterative
refinements, each seeded with a random starting model. The results are sorted in order of apparent agreement with the
data, such that at the end, the first numbered model should be the best result. Ideally the top few answers will all
qualtitatively agree on the overall structure. If they do not, the results should be thoroughly assessed manually to
insure a sensible result. By default this routine will generate 10 initial models, but this may be fewer or more than
is strictly necessary depending on a number of factors. If the data is highly structurally heterogeneous, particularly
if combined with a strongly preferred orientation, a correct solution using this technique may not be possible, but
for most situations it will work well. For other situations, single particle tomography presents a good alternative
for generating initial models."""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_header(
name="initialmodelheader",
help='Options below this label are specific to e2initialmodel',
title="### e2initialmodel options ###",
row=1,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--input",
dest="input",
default=None,
type=str,
help=
"This file should contain good class-averages to use in constructing the initial model",
browser='EMBrowserWidget(withmodal=True,multiselect=False)',
guitype='filebox',
row=0,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--iter",
type=int,
default=8,
help=
"The total number of refinement iterations to perform, typically 5-10",
guitype='intbox',
row=2,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--tries",
type=int,
default=10,
help=
"The number of different initial models to generate in search of a good one",
guitype='intbox',
row=2,
col=1,
rowspan=1,
colspan=1)
parser.add_argument(
"--shrink",
dest="shrink",
type=int,
default=0,
help=
"Optionally shrink the input particles by an integer factor prior to reconstruction. Default=0, no shrinking",
guitype='shrinkbox',
row=2,
col=2,
rowspan=1,
colspan=1)
parser.add_argument(
"--sym",
dest="sym",
help="Specify symmetry - choices are: c<n>, d<n>, h<n>, tet, oct, icos",
default="c1",
guitype='symbox',
row=4,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--automaskexpand",
default=-1,
type=int,
help=
"Number of voxels of post-threshold expansion in the mask, for use when peripheral features are truncated. (default=shrunk boxsize/20)",
guitype='intbox',
row=6,
col=2,
rowspan=1,
colspan=1)
parser.add_argument(
"--randorient",
action="store_true",
help=
"Instead of seeding with a random volume, seeds by randomizing input orientations",
default=False,
guitype='boolbox',
row=4,
col=2,
rowspan=1,
colspan=1)
parser.add_argument(
"--maskproc",
default=None,
type=str,
help=
"Default=none. If specified, this mask will be performed after the built-in automask, eg - mask.soft to remove the core of a virus",
)
# parser.add_argument("--savemore",action="store_true",help="Will cause intermediate results to be written to flat files",default=False, guitype='boolbox', expert=True, row=5, col=0, rowspan=1, colspan=1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--orientgen",
type=str,
default="eman:delta=9.0:inc_mirror=0:perturb=1",
help=
"The type of orientation generator. Default is eman:delta=9.0:inc_mirror=0:perturb=1. See e2help.py orientgens",
guitype='strbox',
expert=True,
row=4,
col=2,
rowspan=1,
colspan=1)
parser.add_argument(
"--parallel",
"-P",
type=str,
help=
"Run in parallel, specify type:<option>=<value>:<option>=<value>. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel",
default="thread:1",
guitype='strbox',
row=6,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
# Database Metadata storage
#parser.add_argument("--dbls",type=str,default=None,help="data base list storage, used by the workflow. You can ignore this argument.")
(options, args) = parser.parse_args()
verbose = options.verbose
try:
ptcls = EMData.read_images(options.input)
except:
print("Error: bad input file")
exit(1)
apix = ptcls[0]["apix_x"]
if options.shrink > 1: apix *= options.shrink
if options.tries < 10:
print(
"Warning: suggest using --tries >=10. The first 8 starting maps are generated deterministically, and it is good to have several random seeds as well to increase the liklihood of a good outcome."
)
for i in range(len(ptcls)):
ptcls[i].process_inplace("normalize.edgemean", {})
if options.shrink > 1:
ptcls[i] = ptcls[i].process("math.meanshrink",
{"n": options.shrink})
if ptcls[0]["nx"] > 160:
print(
"WARNING: using a large box size may be slow. Suggest trying --shrink="
)
if not ptcls or len(ptcls) == 0: parser.error("Bad input file")
boxsize = ptcls[0].get_xsize()
if verbose > 0:
print("%d particles %dx%d" % (len(ptcls), boxsize, boxsize))
print("Models will be %1.3f A/pix" % apix)
[og_name, og_args] = parsemodopt(options.orientgen)
try:
sfcurve = XYData()
sfcurve.read_file("strucfac.txt")
sfcurve.update()
except:
sfcurve = None
if options.maskproc != None:
mask2 = EMData(boxsize, boxsize, boxsize)
mask2.to_one()
parms = parsemodopt(options.maskproc)
if parms[0] == "mask.auto3d":
print(
"Error, maskproc may not be mask.auto3d, it must be a processor that does not rely on the input map density to function"
)
sys.exit(1)
mask2.process_inplace(parms[0], parms[1])
else:
mask2 = None
# angles to use for refinement
sym_object = parsesym(options.sym)
orts = sym_object.gen_orientations(og_name, og_args)
logid = E2init(sys.argv, options.ppid)
results = []
try:
os.mkdir("initial_models")
except:
pass
iters = [
int(i[10:12]) for i in os.listdir("initial_models")
if i[:10] == "particles_"
]
try:
newiter = max(iters) + 1
except:
newiter = 0
results_name = "initial_models/model_%02d" % newiter
particles_name = "initial_models/particles_%02d.hdf" % newiter
# we write the pre-processed "particles" (usually class-averages) to disk, both as a record and to prevent collisions
for i, p in enumerate(ptcls):
p.write_image(particles_name, i)
# parallelism
from EMAN2PAR import EMTaskCustomer # we need to put this here to avoid a circular reference
etc = EMTaskCustomer(options.parallel, module="e2initialmodel.InitMdlTask")
pclist = [particles_name]
etc.precache(
pclist
) # make sure the input particles are precached on the compute nodes
tasks = []
for t in range(options.tries):
tasks.append(
InitMdlTask(particles_name, len(ptcls), orts, t, sfcurve,
options.iter, options.sym, mask2, options.randorient,
options.automaskexpand, options.verbose))
taskids = etc.send_tasks(tasks)
alltaskids = taskids[:] # we keep a copy for monitoring progress
# This loop runs until all subtasks are complete (via the parallelism system
ltime = 0
while len(taskids) > 0:
time.sleep(0.1)
curstat = etc.check_task(
taskids) # a list of the progress on each task
if options.verbose > 1:
if time.time() - ltime > 1:
print("progress: ", curstat)
ltime = time.time()
for i, j in enumerate(curstat):
if j == 100:
rslt = etc.get_results(
taskids[i]
) # read the results back from a completed task as a one item dict
results.append(rslt[1]["result"])
if options.verbose == 1:
print("Task {} ({}) complete".format(i, taskids[i]))
# filter out completed tasks. We can't do this until after the previous loop completes
taskids = [
taskids[i] for i in range(len(taskids)) if curstat[i] != 100
]
# Write out the final results
results.sort()
for i, j in enumerate(results):
out_name = results_name + "_%02d.hdf" % (i + 1)
j[1].write_image(out_name, 0)
j[4].write_image(results_name + "_%02d_init.hdf" % (i + 1), 0)
print(out_name, j[1]["quality"], j[0], j[1]["apix_x"])
for k, l in enumerate(j[3]):
l[0].write_image(results_name + "_%02d_proj.hdf" % (i + 1),
k) # set of projection images
for k, l in enumerate(j[2]):
l.process("normalize").write_image(
results_name + "_%02d_aptcl.hdf" % (i + 1),
k * 2) # set of aligned particles
j[3][l["match_n"]][0].process("normalize").write_image(
results_name + "_%02d_aptcl.hdf" % (i + 1),
k * 2 + 1) # set of projections matching aligned particles
E2end(logid)
def main():
usage = """
Extract 2D subtilt particles from the tilt series, and reconstruct 3D subvolumes.
There are a number of modes for the program and it is probably easier to get the right commands from the GUI.
To extract particles of a certain label from one tomogram
e2spt_extract.py tomograms/tomox__bin4.hdf --boxsz_unbin 128 --label <label from particle picking>
To extract the particles of the same label from all tomograms
e2spt_extract.py --alltomograms --boxsz_unbin 128 --label <label from particle picking>
To extract particles along curves drawn using e2tomo_drawcurve.py. The direction of particles along the curve will be written to the header of particles which can be used in downstream refinement. The spacing of particles along the curve is controlled by the box size and the overlap between adjacent boxes.
e2spt_extract.py --curves <curve ID> --curves_overlap 0.75 --alltomograms --boxsz_unbin 128 --newlabel <new particle label>
To extract particls from an existing refinement (either the particle_param_xx.json from e2spt_refine, or aliptcls3d_xx.lst from e2spt_refine_new)
e2spt_extract.py --jsonali <json/lst from existing alignment> --boxsz_unbin 128 --newlabel <new particle label>
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_pos_argument(
name="tomograms",
help=
"Specify tomograms from which you wish to extract boxed particles.",
default="",
guitype='filebox',
browser="EMTomoBoxesTable(withmodal=True,multiselect=True)",
row=0,
col=0,
rowspan=1,
colspan=2,
mode="extract")
parser.add_argument("--boxsz_unbin",
type=int,
help="box size in unbinned tomogram",
default=-1,
guitype='intbox',
row=2,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--label",
type=str,
help=
"Only extract particle with this name. Leave blank to extract all particles.",
default=None,
guitype='strbox',
row=2,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--newlabel",
type=str,
help=
"Label of output particles. Same as original particle label by default.",
default="",
guitype='strbox',
row=6,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock1",
help='Just a visual separation',
title="Options",
row=3,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--threads",
type=int,
help="threads",
default=12,
guitype='intbox',
row=4,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--maxtilt",
type=int,
help="max tilt",
default=100,
guitype='intbox',
row=4,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--padtwod",
type=float,
help="padding factor",
default=2.0,
guitype='floatbox',
row=5,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--noctf",
action="store_true",
default=False,
help="skip ctf correction.")
parser.add_argument(
"--wiener",
action="store_true",
default=False,
help=
"wiener filter the particles using ctf information. Not very useful..."
)
parser.add_argument("--alltomograms",
action="store_true",
default=False,
help="use all tomograms.",
guitype='boolbox',
row=1,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--dotest",
action="store_true",
default=False,
help="only make 1 batch of subtomograms for testing")
parser.add_argument(
"--shrink",
type=float,
help=
"Shrinking factor for output particles. 1.5 or integers allowed. Default is 1 (no shrink).",
default=1,
guitype='floatbox',
row=8,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--tltkeep",
type=float,
help=
"keep a fraction of tilt images with good score determined from tomogram reconstruction",
default=1.0,
guitype='floatbox',
row=8,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--rmbeadthr",
type=float,
help=
"remove 2d particles with high contrast object beyond N sigma at 100A. Note that this may result in generating fewer particles than selected. Default is -1 (include all particles). 0.5 might be a good choice for removing gold beads but may need some testing...",
default=-1,
guitype='floatbox',
row=9,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock2",
help='Just a visual separation',
title="Extract from curves",
row=10,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--curves",
type=int,
default=-1,
help="specify curve id to extract particles from saved curves. ",
guitype='intbox',
row=11,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--curves_overlap",
type=float,
help=
"fraction of overlap when generating particle along curves. default is 0.5",
default=0.5,
guitype='floatbox',
row=12,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock3",
help='Just a visual separation',
title="Re-extraction from spt",
row=13,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--jsonali",
type=str,
help=
"re-extract particles using a particle_param_xx json file from a spt alignment",
default="",
guitype='filebox',
browser="EMBrowserWidget(withmodal=True,multiselect=False)",
row=14,
col=0,
rowspan=1,
colspan=2,
mode="extract")
parser.add_argument(
"--mindist",
type=float,
help="minimum distance between particles in A. for reextraction only",
default=10)
parser.add_argument(
"--keep",
type=float,
help=
"fraction of particles to keep fron previous alignment. for reextraction only.",
default=1)
parser.add_argument("--postproc",
type=str,
help="processor after 3d particle reconstruction",
default="")
parser.add_argument(
"--postmask",
type=str,
help=
"masking after 3d particle reconstruction. The mask is transformed if json ",
default="")
#parser.add_argument("--textin", type=str,help="text file for particle coordinates. do not use..", default=None)
parser.add_argument(
"--compressbits",
type=int,
help=
"Bits to keep for compression. default is -1 meaning uncompressed floating point. 8 bit seems fine...",
default=8,
guitype='intbox',
row=5,
col=1,
rowspan=1,
colspan=1,
mode="extract[8]")
parser.add_argument("--norewrite",
action="store_true",
default=False,
help="skip existing files. do not rewrite.")
parser.add_argument("--append",
action="store_true",
default=False,
help="append to existing files.")
parser.add_argument("--parallel", type=str, help="parallel", default="")
parser.add_argument(
"--postxf",
type=str,
help=
"a file listing post transforms (see http://eman2.org/e2tomo_more), or for simple symmetry, <sym>,<cx>,<cy>,<cz> where the coordinates specify the center of a single subunit",
default=None)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-2)
parser.add_argument(
"--skip3d",
action="store_true",
default=False,
help=
"do not make 3d particles. only generate 2d particles and 3d header. ")
#parser.add_argument("--shrink3d", type=int, help="Only shrink 3d particles by x factor",default=-1)
parser.add_argument("--verbose", type=int, help="verbose", default=1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
allxfs = {}
allinfo = {}
if len(options.jsonali) > 0:
allxfs, allinfo = parse_json(options)
if options.alltomograms:
fld = "tomograms/"
args = [fld + f for f in os.listdir(fld) if f.endswith(".hdf")]
### do not count a tomogram twice when multiple versions exist
uq, uid = np.unique([base_name(a) for a in args], return_index=True)
args = [args[i] for i in uid]
if len(args) == 0 and len(allxfs) == 0:
print("No input. Exit.")
return
if options.parallel != "":
#### in theory we allow doing this via MPI in addition to threading, and it is possible to spawn one job per node, and each worker will do reconstruction with multiple threads using shared memory
## in practice it is not well tested and it is not quite clear how well it performs on different clusters.
## since the major speed limiting factor of the program is still in disk I/O, it is not much slowerer with just --threads
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_extract.SptExtractTask")
num_cpus = etc.cpu_est()
options.nowtime = time.time()
print(
"Running in parallel mode with {} workers, each running {} threads"
.format(num_cpus, options.threads))
njob = num_cpus
options.verbose = 0
if len(allxfs) > 0:
fnames = list(allxfs.keys())
elif len(args) > 0:
fnames = args
tids = []
for i in range(njob):
task = SptExtractTask(fnames[i::njob], options, allxfs, allinfo)
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if np.min(st_vals) == 100: break
#print(st_vals)
time.sleep(1)
else:
#### do extraction sequentially for each tomogram
if len(allxfs) > 0:
for fname in allxfs.keys():
#### it seems options is changed inplace somewhere...
(options, args) = parser.parse_args()
do_extraction(fname, options, allxfs[fname], allinfo[fname])
elif len(args) > 0:
for a in args:
do_extraction(a, options)
return
print("Particle extraction finished.")
E2end(logid)
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--path",
type=str,
help=
"Path to a folder where results should be stored, following standard naming conventions",
default="subtlt_00")
parser.add_argument("--ref", type=str, help="reference map", default=None)
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=0)
parser.add_argument(
"--goldcontinue",
action="store_true",
help=
"Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
default=False)
parser.add_argument(
"--maxres",
type=float,
help="Maximum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument(
"--minres",
type=float,
help="Minimum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument(
"--smooth",
type=float,
help=
"smooth local motion by this factor. smoother local motion with larger numbers. default 100",
default=100)
parser.add_argument(
"--smoothN",
type=int,
help="number of neighboring particles used for smoothing. default 15",
default=15)
parser.add_argument("--maxshift",
type=float,
help="max shift in pixel. default default box size/6",
default=-1)
parser.add_argument("--refine_trans",
action="store_true",
help="do translational alignment.",
default=False)
parser.add_argument("--refine_trans_ccf",
action="store_true",
help="do translational alignment using simple ccf.",
default=False)
parser.add_argument(
"--refine_rot",
action="store_true",
help=
"do translational-rotational alignment. better to start from an existing translational alignment.",
default=False)
parser.add_argument(
"--refine_defocus",
action="store_true",
help=
"do defocus refinement. need aliptcls input. doesn't work with refine_trans or rot yet..",
default=False)
parser.add_argument(
"--use3d",
action="store_true",
help="use projection of 3d particles instead of 2d ones..",
default=False)
parser.add_argument(
"--preprocess",
metavar="processor_name:param1=value1:param2=value2",
type=str,
default=None,
help="Preprocess each 2-D subtilt while loading (alignment only)")
parser.add_argument(
"--aliptcls2d",
type=str,
help=
"optional aliptcls input. the program can start search from the position from last run.",
default="")
parser.add_argument("--aliptcls3d",
type=str,
help="optional aliptcls input.",
default="")
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default="thread:4")
parser.add_argument("--debug",
action="store_true",
help="for testing.",
default=False)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv, options.ppid)
options.info2dname = "{}/particle_info_2d.lst".format(options.path)
options.info3dname = "{}/particle_info_3d.lst".format(options.path)
n = EMUtil.get_image_count(options.info2dname)
tasks = list(range(n))
if options.preprocess != None:
options.preprocess = parsemodopt(options.preprocess)
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_subtlt_local.SptAlignTask")
num_cpus = etc.cpu_est()
if options.debug:
tasks = tasks[:num_cpus * 4]
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob = num_cpus
tids = []
for i in range(njob):
t = tasks[i::njob]
task = SptAlignTask(t, options)
if options.debug:
ret = task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output = [None] * len(tasks)
for i in tids:
rets = etc.get_results(i)[1]
for r in rets:
output[r[0]] = r[1]
del etc
fm = "{}/aliptcls2d_{:02d}.lst".format(options.path, options.iter)
save_lst_params(output, fm)
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
This program is part of the 'new' hierarchy of e2spt_ programs. It performs one iteration of a classical subtomogram refinement, ie - aligning particles with missing wedge to a reference in 3-D
The reference may be <volume> or <volume>,<n>
If --goldstandard is specified, even and odd variants of the alignment reference must be provided, and even and odd particles will be aligned separately"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--threads", default=4,type=int,help="Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.", guitype='intbox', row=24, col=2, rowspan=1, colspan=1, mode="refinement")
parser.add_argument("--iter",type=int,help="Iteration number within path. Default = start a new iteration",default=0)
parser.add_argument("--goldstandard",type=float,help="If specified, will phase randomize the even and odd references past the specified resolution (in A, not 1/A)",default=0)
parser.add_argument("--goldcontinue",action="store_true",help="Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",default=False)
#parser.add_argument("--saveali",action="store_true",help="Save a stack file (aliptcls.hdf) containing the aligned subtomograms.",default=False)
#parser.add_argument("--savealibin",type=int,help="shrink aligned particles before saving",default=1)
parser.add_argument("--mask",type=str,default=None,help="Mask file aligned to the input reference. Alignment occurs under this mask.")
parser.add_argument("--path",type=str,default=None,help="Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)")
parser.add_argument("--sym",type=str,default="c1",help="Symmetry of the input. Must be aligned in standard orientation to work properly.")
parser.add_argument("--maxres",type=float,help="Maximum resolution (the smaller number) to consider in alignment (in A, not 1/A)",default=0)
parser.add_argument("--minres",type=float,help="Minimum resolution (the larger number) to consider in alignment (in A, not 1/A)",default=0)
#parser.add_argument("--wtori",type=float,help="Weight for using the prior orientation in the particle header. default is -1, i.e. not used.",default=-1)
parser.add_argument("--nsoln",type=int,help="number of solutions to keep at low resolution for the aligner",default=1)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
parser.add_argument("--transonly",action="store_true",help="translational alignment only, for prealigned particles",default=False)
parser.add_argument("--refine",action="store_true",help="local refinement from xform.align3d in header.",default=False)
parser.add_argument("--flcf",action="store_true",help="use slower aligner (experimental)",default=False)
parser.add_argument("--refinentry", type=int, help="number of tests for refine mode. default is 8",default=8)
parser.add_argument("--randphi",action="store_true",help="randomize phi during refine alignment",default=False)
parser.add_argument("--breaksym",action="store_true",help="symmetry breaking.",default=False)
parser.add_argument("--breaksymsym",type=str,help="the symmetry to use for breaksym. setting sym to c6 and this to c2 results in a c3 structure. default is the same as sym",default=None)
parser.add_argument("--rand180",action="store_true",help="randomly add a 180 degree rotation during refine alignment",default=False)
parser.add_argument("--test180",action="store_true",help="Test for improved alignment with 180 degree rotations even during refine alignment",default=False)
parser.add_argument("--skipali",action="store_true",help="skip alignment. the program will do nothing. mostly for testing...",default=False)
parser.add_argument("--maxang",type=float,help="Maximum angular difference for the refine mode. default is 30",default=30)
parser.add_argument("--maxshift",type=float,help="Maximum shift for the refine mode. default is 16",default=-1)
parser.add_argument("--scipytest",action="store_true",help="test scipy optimizer.",default=False)
parser.add_argument("--debug",action="store_true",help=".",default=False)
(options, args) = parser.parse_args()
if options.path == None:
options.path=num_path_new("spt_")
if options.iter<=0 :
fls=[int(i[15:17]) for i in os.listdir(options.path) if i[:15]=="particle_parms_" and str.isdigit(i[15:17])]
if len(fls)==0 : options.iter=1
else: options.iter=max(fls)+1
if options.parallel==None:
options.parallel="thread:{}".format(options.threads)
if options.breaksym:
if options.breaksymsym==None:
if options.sym=="c1":
print("cannot break a c1 symmetry. breaksym disabled.")
options.breaksym=False
else:
options.breaksymsym=options.sym
# file may be "name" or "name,#"
reffile=args[1].split(",")[0]
try: refn=int(args[1].split(",")[1])
except: refn=0
NTHREADS=max(options.threads+1,2) # we have one thread just writing results
logid=E2init(sys.argv, options.ppid)
refnames=[reffile,reffile]
if options.goldcontinue or options.goldstandard>0:
ref=[]
try:
refnames=[reffile[:-4]+"_even.hdf", reffile[:-4]+"_odd.hdf"]
ref.append(EMData(refnames[0],0))
ref.append(EMData(refnames[1],0))
except:
print("Error: cannot find one of reference files, eg: ",EMData(reffile[:-4]+"_even.hdf",0))
# else:
# ref=[]
# ref.append(EMData(reffile,refn))
# ref.append(EMData(reffile,refn))
#
# if options.goldstandard>0 :
# ref[0].process_inplace("filter.lowpass.randomphase",{"cutoff_freq":old_div(1.0,options.goldstandard)})
# ref[0].process_inplace("filter.lowpass.gauss",{"cutoff_freq":old_div(1.0,options.goldstandard)})
# ref[1].process_inplace("filter.lowpass.randomphase",{"cutoff_freq":old_div(1.0,options.goldstandard)})
# ref[1].process_inplace("filter.lowpass.gauss",{"cutoff_freq":old_div(1.0,options.goldstandard)})
# refnames=["{}/align_ref_even.hdf".format(options.path), "{}/align_ref_odd.hdf".format(options.path)]
# ref[0].write_image(refnames[0],0)
# ref[1].write_image(refnames[1],0)
#
# else:
# refnames=[reffile, reffile]
n=-1
tasks=[]
readjson=False
if args[0].endswith(".lst") or args[0].endswith(".hdf"):
#### check if even/odd split exists
fsps=[args[0][:-4]+"__even.lst",args[0][:-4]+"__odd.lst"]
if os.path.isfile(fsps[0]) and os.path.isfile(fsps[1]):
print("Using particle list: \n\t {} \n\t {}".format(fsps[0], fsps[1]))
for eo, f in enumerate(fsps):
N=EMUtil.get_image_count(f)
tasks.extend([(f,i,refnames, eo) for i in range(N)])
#### split by even/odd by default
else:
N=EMUtil.get_image_count(args[0])
tasks.extend([(args[0],i,refnames, i%2) for i in range(N)])
#thrds=[threading.Thread(target=alifn,args=(jsd,args[0],i,ref[i%2],options)) for i in range(N)]
elif args[0].endswith(".json"):
#print("Reading particles from json. This is experimental...")
js=js_open_dict(args[0])
readjson=True
jsinput=dict(js)
keys=sorted(js.keys())
for k in keys:
src, ii=eval(k)
dic=js[k]
xf=dic["xform.align3d"]
tasks.append([src, ii, refnames, ii%2, xf])
from EMAN2PAR import EMTaskCustomer
if options.scipytest:
etc=EMTaskCustomer(options.parallel, module="e2spt_align.ScipySptAlignTask")
else:
etc=EMTaskCustomer(options.parallel, module="e2spt_align.SptAlignTask")
num_cpus = etc.cpu_est()
options.nowtime=time.time()
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob=num_cpus#*4
tids=[]
for i in range(njob):
t=tasks[i::njob]
if options.scipytest:
task=ScipySptAlignTask(t, options)
else:
task=SptAlignTask(t, options)
if options.debug:
ret=task.execute(print)
print(ret)
return
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print(st_vals)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals)/100.)
if np.min(st_vals) == 100: break
time.sleep(5)
#dics=[0]*nptcl
angs={}
for i in tids:
rets=etc.get_results(i)[1]
for ret in rets:
fsp,n,dic=ret
if len(dic)==1:
dic=dic[0]
if readjson:
k=str((fsp,n))
if "eo" in js[k]:
dic["eo"]=jsinput[k]["eo"]
angs[(fsp,n)]=dic
out="{}/particle_parms_{:02d}.json".format(options.path,options.iter)
if os.path.isfile(out):
os.remove(out)
js=js_open_dict(out)
js.update(angs)
js.close()
del etc
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
Note that this program is not part of the original e2spt hierarchy, but is part of an experimental refactoring.
This program will take an input stack of subtomograms and a reference volume, and perform a missing-wedge aware alignment of each particle to the reference. If --goldstandard is specified, then even and odd particles will be aligned to different perturbed versions of the reference volume, phase-randomized past the specified resolution."""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--threads",
default=4,
type=int,
help=
"Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.",
guitype='intbox',
row=24,
col=2,
rowspan=1,
colspan=1,
mode="refinement")
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=0)
parser.add_argument(
"--goldstandard",
type=float,
help=
"If specified, will phase randomize the even and odd references past the specified resolution (in A, not 1/A)",
default=0)
parser.add_argument(
"--goldcontinue",
action="store_true",
help=
"Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
default=False)
parser.add_argument(
"--saveali",
action="store_true",
help=
"Save a stack file (aliptcls.hdf) containing the aligned subtomograms.",
default=False)
parser.add_argument("--savealibin",
type=int,
help="shrink aligned particles before saving",
default=1)
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
)
parser.add_argument(
"--sym",
type=str,
default="c1",
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly."
)
parser.add_argument(
"--maxres",
type=float,
help="Maximum resolution to consider in alignment (in A, not 1/A)",
default=0)
#parser.add_argument("--wtori",type=float,help="Weight for using the prior orientation in the particle header. default is -1, i.e. not used.",default=-1)
parser.add_argument(
"--nsoln",
type=int,
help="number of solutions to keep at low resolution for the aligner",
default=1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default=None)
parser.add_argument("--refine",
action="store_true",
help="local refinement from xform.init in header.",
default=False)
parser.add_argument(
"--maxang",
type=float,
help="Maximum angular difference for the refine mode. default is 30",
default=30)
(options, args) = parser.parse_args()
#task=SptAlignTask(0,1,2,options)
#from pickle import dumps,loads,dump,load
#f=open("task.tmp",'w')
#dump(task,f)
#f.close()
#print(task)
#return
if options.path == None:
fls = [
int(i[-2:]) for i in os.listdir(".")
if i[:4] == "spt_" and len(i) == 6 and str.isdigit(i[-2:])
]
if len(fls) == 0: fls = [0]
options.path = "spt_{:02d}".format(max(fls) + 1)
try:
os.mkdir(options.path)
except:
pass
if options.iter <= 0:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0: options.iter = 1
else: options.iter = max(fls) + 1
if options.parallel == None:
options.parallel = "thread:{}".format(options.threads)
reffile = args[1]
NTHREADS = max(options.threads + 1,
2) # we have one thread just writing results
logid = E2init(sys.argv, options.ppid)
if options.goldcontinue:
ref = []
try:
ref.append(EMData(reffile[:-4] + "_even.hdf", 0))
ref.append(EMData(reffile[:-4] + "_odd.hdf", 0))
except:
print("Error: cannot find one of reference files, eg: ",
EMData(reffile[:-4] + "_even.hdf", 0))
else:
ref = []
ref.append(EMData(reffile, 0))
ref.append(EMData(reffile, 0))
if options.goldstandard > 0:
ref[0].process_inplace(
"filter.lowpass.randomphase",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[0].process_inplace(
"filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[1].process_inplace(
"filter.lowpass.randomphase",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[1].process_inplace(
"filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[0].write_image("{}/align_ref.hdf".format(options.path), 0)
ref[1].write_image("{}/align_ref.hdf".format(options.path), 1)
ref[0] = ref[0].do_fft()
ref[0].process_inplace("xform.phaseorigin.tocorner")
ref[1] = ref[1].do_fft()
ref[1].process_inplace("xform.phaseorigin.tocorner")
jsd = queue.Queue(0)
n = -1
#### check if even/odd split exists
fsps = [args[0][:-4] + "__even.lst", args[0][:-4] + "__odd.lst"]
tasks = []
if os.path.isfile(fsps[0]) and os.path.isfile(fsps[1]):
print("Using particle list: \n\t {} \n\t {}".format(fsps[0], fsps[1]))
for eo, f in enumerate(fsps):
N = EMUtil.get_image_count(f)
tasks.extend([(f, i, ref[eo]) for i in range(N)])
#### split by even/odd by default
else:
N = EMUtil.get_image_count(args[0])
tasks.extend([(args[0], i, ref[i % 2]) for i in range(N)])
#thrds=[threading.Thread(target=alifn,args=(jsd,args[0],i,ref[i%2],options)) for i in range(N)]
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel, module="e2spt_align.SptAlignTask")
num_cpus = etc.cpu_est()
#tasks=tasks[:24]
print("{} total CPUs available".format(num_cpus))
print("{} jobs".format(len(tasks)))
tids = []
for t in tasks:
task = SptAlignTask(t[0], t[1], t[2], options)
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
#print(tids)
if np.min(st_vals) == 100: break
time.sleep(5)
#dics=[0]*nptcl
angs = {}
for i in tids:
ret = etc.get_results(i)[1]
fsp, n, dic = ret
if len(dic) == 1:
angs[(fsp, n)] = dic[0]
else:
angs[(fsp, n)] = dic
js = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, options.iter))
js.update(angs)
del etc
## here we run the threads and save the results, no actual alignment done here
#print(len(thrds)," threads")
#thrtolaunch=0
#while thrtolaunch<len(thrds) or threading.active_count()>1:
## If we haven't launched all threads yet, then we wait for an empty slot, and launch another
## note that it's ok that we wait here forever, since there can't be new results if an existing
## thread hasn't finished.
#if thrtolaunch<len(thrds) :
#while (threading.active_count()==NTHREADS ) : time.sleep(.1)
#if options.verbose : print("Starting thread {}/{}".format(thrtolaunch,len(thrds)))
#thrds[thrtolaunch].start()
#thrtolaunch+=1
#else: time.sleep(1)
#while not jsd.empty():
#fsp,n,d=jsd.get()
#angs[(fsp,n)]=d
#if options.saveali:
#v=EMData(fsp,n)
#v.transform(d["xform.align3d"])
#if options.savealibin>1:
#v.process_inplace("math.meanshrink",{"n":options.savealibin})
#v.write_image("{}/aliptcls_{:02d}.hdf".format(options.path, options.iter),n)
#for t in thrds:
#t.join()
E2end(logid)
def execute(self, alignment_jobs, files, caller):
'''
The main function
@param alignment_jobs a list of alignment pair indices like this [[0,1],[2,1],[2,3],[0,5],...] etc the indices pair represent images to be aligned and correspond to the order of the files argument
@param files a list of filenames - used to read image based on the indices present in alignment_jobs
@param caller - the calling object - it needs to have a function called process_output that takes a dictionary as the argument
'''
options = self.options
align_data = EMAN2.parsemodopt(options.align)
align_cmp_data = EMAN2.parsemodopt(options.aligncmp)
cmp_data = EMAN2.parsemodopt(options.cmp)
ralign_data = None
if options.ralign != None:
ralign_data = EMAN2.parsemodopt(options.ralign)
ralign_cmp_data = EMAN2.parsemodopt(options.raligncmp)
data = {}
data["align"] = align_data
data["aligncmp"] = align_cmp_data
data["cmp"] = cmp_data
if ralign_data:
data["ralign"] = ralign_data
data["raligncmp"] = ralign_cmp_data
data["using_cuda"] = self.using_cuda
data["nsoln"] = self.nsoln
if self.options.parallel:
task_customers = []
tids = []
if options.shrink:
scratch_name_1 = numbered_bdb(
"bdb:tomo_scratch#scratch_shrink")
scratch_name_2 = numbered_bdb(
"bdb:tomo_scratch##scratch_shrink")
else:
print("no shrink")
for i, j in alignment_jobs:
if options.shrink or options.filter:
a = EMData(files[i], 0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0], filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",
{"n": options.shrink})
a.set_attr("src_image", files[i])
a.write_image(scratch_name_1, 0)
a = EMData(files[j], 0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0], filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",
{"n": options.shrink})
a.set_attr("src_image", files[j])
a.write_image(scratch_name_2, 0)
data["probe"] = ("cache", scratch_name_1, 0)
data["target"] = ("cache", scratch_name_2, 0)
else:
data["probe"] = ("cache", files[i], 0)
data["target"] = ("cache", files[j], 0)
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTaskDC(data=data)
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(self.options.parallel)
#print "Est %d CPUs"%etc.cpu_est()
tid = etc.send_task(task)
#print "Task submitted tid=",tid
task_customers.append(etc)
tids.append(tid)
self.dc_monitor(task_customers, tids, caller)
else:
n = len(alignment_jobs)
p = 0.0
for i, j in alignment_jobs:
probe = EMData(files[i], 0)
target = EMData(files[j], 0)
if options.filter:
print("filtered")
filter_params = EMAN2.parsemodopt(options.filter)
probe.process_inplace(filter_params[0], filter_params[1])
target.process_inplace(filter_params[0], filter_params[1])
if options.shrink:
probe.process_inplace("math.meanshrink",
{"n": options.shrink})
target.process_inplace("math.meanshrink",
{"n": options.shrink})
else:
print("no shrink")
data["target"] = target
data["probe"] = probe
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTask(data=data)
rslts = task.execute(self.progress_callback)
if options.shrink:
self.correction_translation(rslts, options.shrink)
caller.process_output(rslts)
p += 1.0
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_average.py <ref1> <ref2> ... [options]
Note that this program is not part of the original e2spt hierarchy, but is part of an experimental refactoring.
Will read metadata from the specified spt_XX directory, as produced by e2spt_align.py, and average a selected subset of subtomograms in the predetermined orientation.
This version of the program competes each particle against N reference volumes, and only averages it with the best match. Alignment parameters from a previous
e2spt_align run are used to define the coarse orientation, so the references must be similar and in the same orientation. Alignments are translationally adjusted only.
If --sym is specified, each possible symmetric orientation is tested starting with the exisiting alignment parameters, and only the best is kept.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=0)
parser.add_argument(
"--simthr",
default=-0.1,
type=float,
help=
"Similarity is smaller for better 'quality' particles. Specify the highest value to include from e2spt_hist.py. Default -0.1"
)
parser.add_argument(
"--simthr2",
default=0,
type=float,
help=
"Simlarity score for the best matching final alignment. Scaling may be different due to resolution limit. Default 0"
)
parser.add_argument(
"--replace",
type=str,
default=None,
help=
"Replace the input subtomograms used for alignment with the specified file (used when the aligned particles were masked or filtered)"
)
parser.add_argument(
"--wedgesigma",
type=float,
help=
"Threshold for identifying missing data in Fourier space in terms of standard deviation of each Fourier shell. Default 3.0",
default=3.0)
parser.add_argument(
"--minalt",
type=float,
help="Minimum alignment altitude to include. Default=0",
default=0)
parser.add_argument(
"--maxalt",
type=float,
help="Maximum alignment altitude to include. Deafult=180",
default=180)
parser.add_argument(
"--maxtilt",
type=float,
help=
"Explicitly zeroes data beyond specified tilt angle. Assumes tilt axis exactly on Y and zero tilt in X-Y plane. Default 90 (no limit).",
default=90.0)
parser.add_argument(
"--maxres",
type=float,
help=
"Lowpass filter applied to particles prior to alignment/averaging, resolution in A. Default disabled",
default=-1)
parser.add_argument(
"--listfile",
type=str,
help=
"Specify a filename containing a list of integer particle numbers to include in the average, one per line, first is 0. Additional exclusions may apply.",
default=None)
parser.add_argument(
"--shrinkcompare",
type=int,
help="Shrink factor for classification only (for speed)",
default=0)
parser.add_argument(
"--sym",
type=str,
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly. The structure will be expanded from this symmetry to c1.",
default="c1")
parser.add_argument(
"--applysym",
type=str,
help="Symmetry to apply to the structure after classification.",
default="c1")
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder containing current results (default = highest spt_XX)"
)
parser.add_argument(
"--parallel",
type=str,
default=None,
help="parallel mode. Not all functions are implemented yet..")
parser.add_argument(
"--threads",
default=4,
type=int,
help=
"Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present."
)
parser.add_argument("--maskclass",
type=str,
default=None,
help="Mask each reference before classification")
parser.add_argument("--mask",
type=str,
default=None,
help="Mask applied to final averages")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--noali",
action="store_true",
default=False,
help="Skip translational alignment.")
parser.add_argument(
"--symcopy",
action="store_true",
default=False,
help=
"Copy each particle for each asymetrical unit. need a maskclass to focus on one unit. do not work with applysym"
)
parser.add_argument("--nolstout",
action="store_true",
default=False,
help="Skip writting lst output.")
parser.add_argument("--sample",
type=int,
help="use only N samples.",
default=-1)
parser.add_argument("--randnclass",
type=int,
help="split into N random classes. ignore refs",
default=-1)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
if options.parallel != None and options.parallel[:6] == "thread":
options.threads = int(options.parallel.split(":")[1])
options.parallel = None
print("--parallel converted to --threads for efficiency")
if options.path == None:
options.path = num_path_last("spt_")
if options.verbose: print("Working in : ", options.path)
options.path = options.path.strip('/\\')
if options.iter <= 0:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0:
print("Cannot find a {}/particle_parms* file".format(options.path))
sys.exit(2)
options.iter = max(fls)
if options.verbose: print("Using iteration ", options.iter)
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, options.iter))
else:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0:
print("Cannot find a {}/particle_parms* file".format(options.path))
sys.exit(2)
mit = max(fls)
if options.iter > mit:
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, mit))
print(
"WARNING: no particle_parms found for iter {}, using parms from {}"
.format(options.iter, mit))
else:
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, options.iter))
if options.listfile != None:
plist = set([int(i) for i in open(options.listfile, "r")])
NTHREADS = max(options.threads + 1,
2) # we have one thread just writing results
logid = E2init(sys.argv, options.ppid)
# filter the list of particles to include
keys = list(angs.keys())
if options.listfile != None:
keys = [i for i in keys if eval(i)[1] in plist]
if options.verbose:
print("{}/{} particles based on list file".format(
len(keys), len(list(angs.keys()))))
keys = [
k for k in keys if angs[k]["score"] <= options.simthr
and inrange(options.minalt, angs[k]["xform.align3d"].get_params("eman")
["alt"], options.maxalt)
]
if options.verbose:
print("{}/{} particles after filters".format(len(keys),
len(list(angs.keys()))))
if options.sample > 0:
rnd = np.arange(len(keys))
np.random.shuffle(rnd)
rnd = rnd[:options.sample]
keys = [keys[i] for i in rnd.tolist()]
print("using {} samples...".format(len(keys)))
if options.parallel:
print(
"Running in parallel mode. Warning, memory consumption may be high!"
)
if options.randnclass > 0:
nref = options.randnclass
else:
nref = len(args)
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_average_multi.SptavgmultTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
data = [] ## even/odd
for i, k in enumerate(keys):
src, ii = eval(k)[0], eval(k)[1]
data.append([src, ii, angs[k]["xform.align3d"]])
#### check and save size of particle
fsp, i, xf = data[0]
b = EMData(fsp, i, True)
sz = options.boxsz = b["ny"]
avgs = []
tasks = [data[i::num_cpus] for i in range(num_cpus)]
print("{} particles in {} jobs".format(len(data), len(tasks)))
tids = []
for t in tasks:
task = SptavgmultTask(t, args, options)
#task.execute(print)
#exit()
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
#print(tids)
if np.min(st_vals) == 100: break
time.sleep(5)
#dics=[0]*nptcl
threeds = []
normvols = []
stats = []
print("collecting from workers...")
for i in tids:
threed, norm, stat = etc.get_results(i)[1]
#print(len(threed), len(norm), threed[0]["sigma"])
threeds.append(threed)
normvols.append(norm)
stats.append(stat)
stats = np.vstack(stats)
stats = stats[np.argsort(stats[:, 0]), :]
np.savetxt(
"{}/avg_multi_{:02d}.txt".format(options.path, options.iter),
stats)
if not options.nolstout:
lsts = [
LSXFile(f"sets/{options.path}_{options.iter:02d}_{i:02d}.lst")
for i in range(nref)
]
for n, score, cls, x in stats:
lsts[int(cls)].write(-1, int(n), data[0][0])
lsts = None
avs = []
for r in range(nref):
output = EMData(sz, sz, sz)
normvol = EMData((sz // 2 + 1) * 2, sz, sz)
output.to_zero()
output.do_fft_inplace()
avg = Averagers.get("mean")
normvol.to_zero()
for k, thd in enumerate(threeds):
threed = thd[r]
norm = normvols[k][r]
threed.process_inplace("math.multamplitude", {"amp": norm})
avg.add_image(threed)
normvol.add(norm)
output = avg.finish()
normvol.process_inplace("math.reciprocal")
output.process_inplace("math.multamplitude", {"amp": normvol})
output.process_inplace("xform.phaseorigin.tocenter")
output.do_ift_inplace()
output.depad()
avs.append(output)
else:
print("Running in thread mode")
n = len(args)
args = [comma(i) for i in args]
refs = [EMData(i[0], i[1]) for i in args]
if options.maxres > 0:
for r in refs:
r.process_inplace(
"filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, options.maxres)})
if options.maskclass != None:
mask = EMData(options.maskclass)
for r in refs:
r.mult(mask)
jsd = queue.Queue(0)
avgs = [
Averagers.get("mean.tomo", {"thresh_sigma": options.wedgesigma})
for i in range(n)
]
if options.shrinkcompare > 1:
shrinkrefs = [
r.process("math.meanshrink", {"n": options.shrinkcompare})
for r in refs
]
else:
shrinkrefs = refs
# Rotation and insertion are slow, so we do it with threads.
# Averager isn't strictly threadsafe, so possibility of slight numerical errors with a lot of threads
if options.replace != None:
thrds = [
threading.Thread(target=rotfncompete,
args=(jsd, avgs, options.replace, eval(k)[1],
angs[k]["xform.align3d"], options.sym,
refs, shrinkrefs, options.maxtilt,
options.wedgesigma,
options.shrinkcompare, options.maxres,
options.simthr2, options.verbose))
for i, k in enumerate(keys)
]
else:
#thrds=[threading.Thread(target=rotfncompete,args=(jsd,avgs,eval(k)[0],eval(k)[1],angs[k]["xform.align3d"],options.sym,refs,shrinkrefs,options.maxtilt,options.wedgesigma,options.shrinkcompare,options.maxres,options.simthr2,options.verbose)) for i,k in enumerate(keys)]
thrds = [
threading.Thread(target=rotfncompete,
args=(jsd, avgs, eval(k)[0], eval(k)[1],
angs[k]["xform.align3d"], refs,
shrinkrefs, options))
for i, k in enumerate(keys)
]
print(len(thrds), " threads")
thrtolaunch = 0
out = open(
"{}/avg_multi_{:02d}.txt".format(options.path, options.iter), "w")
while thrtolaunch < len(
thrds) or threading.active_count() > 1 or not jsd.empty():
# If we haven't launched all threads yet, then we wait for an empty slot, and launch another
# note that it's ok that we wait here forever, since there can't be new results if an existing
# thread hasn't finished.
if thrtolaunch < len(thrds):
while (threading.active_count() == NTHREADS):
time.sleep(.1)
if options.verbose:
print("Starting thread {}/{}".format(
thrtolaunch, len(thrds)))
thrds[thrtolaunch].start()
thrtolaunch += 1
else:
time.sleep(1)
while not jsd.empty():
fspn, score, ref, sym = jsd.get()
out.write(
"{}\t{}\t{}\t{}\n".format(fspn, score, ref, sym)
) # Output columns are img #, best score, # of best ref, # of best sym
#avg[n%2].add_image(ptcl)
out.flush()
for t in thrds:
t.join()
avs = [i.finish() for i in avgs]
if options.mask: mask = EMData(options.mask)
for i, v in enumerate(avs):
if options.mask: v.mult(mask)
# v.write_image("{}/threed_{:02d}_{:02d}.hdf".format(options.path,options.iter,i),0)
v.write_compressed(
"{}/threed_{:02d}_{:02d}.hdf".format(options.path, options.iter,
i), 0, 12)
print("Done")
E2end(logid)
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--ptclin",
type=str,
help="particle input",
default=None)
parser.add_argument("--ptclout",
type=str,
help="particle output",
default=None)
parser.add_argument("--ref",
type=str,
help="reference input",
default=None)
parser.add_argument("--threedout",
type=str,
help="map output",
default=None)
parser.add_argument("--keep",
type=float,
help="propotion of tilts to keep. default is 0.5",
default=0.5)
parser.add_argument("--threads",
type=int,
help="Number of CPU threads to use. Default is 12.",
default=12)
parser.add_argument(
"--parallel",
type=str,
help="Thread/mpi parallelism to use. Default is thread:12",
default="thread:12")
parser.add_argument(
"--debug",
action="store_true",
default=False,
help=
"Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)"
)
parser.add_argument("--transonly",
action="store_true",
default=False,
help="only refine translation")
parser.add_argument("--savepath",
action="store_true",
default=False,
help="save alignment path in a json file for testing.")
#parser.add_argument("--scipytest", action="store_true", default=False ,help="test scipy optimizer")
parser.add_argument(
"--fromscratch",
action="store_true",
default=False,
help=
"align from scratch and ignore previous particle transforms. for spt mostly. will include mirror"
)
parser.add_argument("--refineastep",
type=float,
help="Mean angular variation for refine alignment",
default=2.)
parser.add_argument("--refinentry",
type=int,
help="number of starting points for refine alignment",
default=4)
parser.add_argument("--maxshift",
type=int,
help="maximum shift allowed",
default=8)
parser.add_argument("--localrefine",
action="store_true",
default=False,
help="local refinement")
parser.add_argument("--defocus",
action="store_true",
default=False,
help="refine defocus. Still under development")
parser.add_argument("--seedmap",
action="store_true",
default=False,
help="seed")
parser.add_argument("--ctfweight",
action="store_true",
default=False,
help="weight by ctf")
parser.add_argument("--skipm3d",
action="store_true",
default=False,
help="skip make3d. only output aligned list")
parser.add_argument("--padby",
type=float,
default=2.0,
help="pad by factor. default is 2")
parser.add_argument("--maxres",
type=float,
default=-1,
help="max resolution for cmp")
parser.add_argument("--minres",
type=float,
default=-1,
help="min resolution for cmp")
parser.add_argument(
"--sym",
type=str,
help="symmetry. will use symmetry from spt refinement by default",
default="c1")
parser.add_argument(
"--smooth",
type=int,
help=
"Smooth trajectory per image based on nearby particles. Still under development",
default=-1)
parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
parser.add_argument("--nkeep", type=int, help="", default=1)
parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
lstname = options.ptclin
threedname = options.ref
lname = options.ptclout
threedout = options.threedout
lst = LSXFile(lstname, True)
m = EMData(threedname)
bxsz = m["nx"]
apix = m["apix_x"]
options.shrink = 1
pinfo = []
if options.debug: nptcl = options.threads * 8
else: nptcl = lst.n
for i in range(nptcl):
pinfo.append(lst.read(i))
lst = None
print("Initializing parallelism...")
etc = EMTaskCustomer(options.parallel,
module="e2spt_tiltrefine_oneiter.SptTltRefineTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
print("{} jobs".format(nptcl))
infos = [[] for i in range(num_cpus)]
for i, info in enumerate(pinfo):
infos[i % num_cpus].append([i, info])
tids = []
for info in infos:
task = SptTltRefineTask(info, threedname, options)
if options.debug:
task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
dics = [0] * nptcl
for i in tids:
ret = etc.get_results(i)[1]
for r in ret:
#print(r)
ii = r.pop("idx")
dics[ii] = r
del etc
allscr = [d["score"] for d in dics]
if options.smooth > 0 or options.defocus:
### need to add per tilt smoothing later...
s = np.array(allscr)
oname = lname.replace(".lst", "_score.hdf")
ss = from_numpy(s).copy()
ss.write_image(oname)
return
maxl = np.max([len(s) for s in allscr])
maxv = np.max(np.concatenate(allscr))
for s in allscr:
s.extend([maxv] * (maxl - len(s)))
allscr = np.array(allscr)
#print(np.min(allscr), np.mean(allscr), np.max(allscr), np.std(allscr))
if options.skipm3d:
pass
else:
allscr = 2 - allscr
allscr -= np.min(allscr)
allscr /= np.max(allscr)
if maxl > 1:
mx = np.max(allscr, axis=1)[:, None]
allscr = np.exp(allscr * 20)
allscr = allscr / np.sum(allscr, axis=1)[:, None]
allscr *= mx
try:
os.remove(lname)
except:
pass
lout = LSXFile(lname, False)
for i, dc in enumerate(dics):
lc = ""
if isinstance(dc["xform.align3d"], list):
alilist = dc["xform.align3d"]
scorelist = dc["score"]
else:
alilist = [dc["xform.align3d"]]
scorelist = [dc["score"]]
for j, xf in enumerate(alilist):
d = xf.get_params("eman")
d["score"] = float(allscr[i, j])
if d["score"] > .05 or j == 0:
lc = lc + str(d) + ';'
l = pinfo[i]
lout.write(-1, l[0], l[1], lc[:-1])
lout = None
pb = options.padby
if options.parallel.startswith("mpi") and len(dics) > 10000:
m3dpar = "--parallel {}".format(options.parallel)
else:
m3dpar = ""
if options.seedmap:
seed = "--seedmap " + threedname
else:
seed = ""
cmd = "e2make3dpar.py --input {inp} --output {out} --pad {pd} --padvol {pdv} --threads {trd} --outsize {bx} --apix {apx} --mode trilinear --keep {kp} --sym {sm} {seed} {par}".format(
inp=lname,
out=threedout,
bx=bxsz,
pd=int(bxsz * pb),
pdv=int(bxsz * pb),
apx=apix,
kp=options.keep,
sm=options.sym,
trd=options.threads,
par=m3dpar,
seed=seed)
if options.skipm3d:
print("Skipping 3D reconstruction")
else:
run(cmd)
E2end(logid)
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--ptclin",
type=str,
help="particle input",
default=None)
parser.add_argument("--ptclout",
type=str,
help="particle output",
default=None)
parser.add_argument("--ref",
type=str,
help="reference input",
default=None)
parser.add_argument("--threedout",
type=str,
help="map output",
default=None)
parser.add_argument("--keep",
type=float,
help="propotion of tilts to keep. default is 0.5",
default=0.5)
parser.add_argument("--threads",
type=int,
help="Number of CPU threads to use. Default is 12.",
default=12)
parser.add_argument(
"--parallel",
type=str,
help="Thread/mpi parallelism to use. Default is thread:12",
default="thread:12")
parser.add_argument(
"--debug",
action="store_true",
default=False,
help=
"Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)"
)
parser.add_argument("--transonly",
action="store_true",
default=False,
help="only refine translation")
parser.add_argument(
"--fromscratch",
action="store_true",
default=False,
help=
"align from scratch and ignore previous particle transforms. for spt mostly. will include mirror"
)
parser.add_argument("--refineastep",
type=float,
help="Mean angular variation for refine alignment",
default=2.)
parser.add_argument("--refinentry",
type=int,
help="number of starting points for refine alignment",
default=4)
parser.add_argument("--maxshift",
type=int,
help="maximum shift allowed",
default=8)
parser.add_argument("--padby",
type=float,
default=2.0,
help="pad by factor. default is 2")
parser.add_argument("--maxres",
type=float,
default=-1,
help="max resolution for cmp")
parser.add_argument(
"--sym",
type=str,
help="symmetry. will use symmetry from spt refinement by default",
default="c1")
parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
lstname = options.ptclin
threedname = options.ref
lname = options.ptclout
threedout = options.threedout
lst = LSXFile(lstname, True)
m = EMData(threedname)
bxsz = m["nx"]
apix = m["apix_x"]
if options.maxres > 0:
options.shrink = max(1, int(options.maxres / apix * .3))
options.shrink = min(options.shrink, bxsz // 48)
print("Will shrink by {} and filter to {:.0f} A. Box size {}".format(
options.shrink, options.maxres, bxsz // options.shrink))
else:
options.shrink = 1
#m.process_inplace('normalize.edgemean')
pinfo = []
if options.debug: nptcl = options.threads * 8
else: nptcl = lst.n
for i in range(nptcl):
pinfo.append(lst.read(i))
lst = None
print("Initializing parallelism...")
etc = EMTaskCustomer(options.parallel,
module="e2spt_tiltrefine_oneiter.SptTltRefineTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
print("{} jobs".format(nptcl))
infos = [[] for i in range(num_cpus)]
for i, info in enumerate(pinfo):
infos[i % num_cpus].append([i, info])
tids = []
for info in infos:
task = SptTltRefineTask(info, threedname, options)
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print(st_vals)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
dics = [0] * nptcl
for i in tids:
ret = etc.get_results(i)[1]
for r in ret:
#print(r)
ii = r.pop("idx")
dics[ii] = r
del etc
allscr = np.array([d["score"] for d in dics])
print(np.min(allscr), np.mean(allscr), np.max(allscr), np.std(allscr))
allscr = 2 - allscr
s = allscr.copy()
s -= np.mean(s)
s /= np.std(s)
clp = 2
ol = abs(s) > clp
print("Removing {} outliers from {} particles..".format(
np.sum(ol), len(s)))
s = (s + clp) / clp / 2
s[ol] = 0
allscr = s
allscr -= np.min(allscr) - 1e-5
allscr /= np.max(allscr)
try:
os.remove(lname)
except:
pass
lout = LSXFile(lname, False)
for i, dc in enumerate(dics):
d = dc["xform.align3d"].get_params("eman")
d["score"] = float(allscr[i])
l = pinfo[i]
lout.write(-1, l[0], l[1], str(d))
lout = None
pb = options.padby
if options.parallel.startswith("mpi") and len(dics) > 10000:
m3dpar = "--parallel {}".format(options.parallel)
else:
m3dpar = ""
cmd = "e2make3dpar.py --input {inp} --output {out} --pad {pd} --padvol {pdv} --threads {trd} --outsize {bx} --apix {apx} --mode gauss_2 --keep {kp} --sym {sm} {par}".format(
inp=lname,
out=threedout,
bx=bxsz,
pd=int(bxsz * pb),
pdv=int(bxsz * pb),
apx=apix,
kp=options.keep,
sm=options.sym,
trd=options.threads,
par=m3dpar)
run(cmd)
E2end(logid)
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--ptclin",
type=str,
help="particle input",
default=None)
parser.add_argument("--ptclout",
type=str,
help="particle output",
default=None)
parser.add_argument("--ref",
type=str,
help="reference input",
default=None)
parser.add_argument(
"--parallel",
type=str,
help="Thread/mpi parallelism to use. Default is thread:12",
default="thread:12")
parser.add_argument(
"--debug",
action="store_true",
default=False,
help=
"Turn on debug mode. This will only process a small subset of the data"
)
#parser.add_argument("--maxshift", type=int,help="maximum shift allowed", default=-1)
parser.add_argument(
"--localrefine",
type=int,
default=-1,
help="local refinement. larger value correspond to smaller local region"
)
parser.add_argument("--goldcontinue",
action="store_true",
default=False,
help="split even/odd subset and references.")
#parser.add_argument("--ctfweight", action="store_true", default=False ,help="weight by ctf. not used yet...")
#parser.add_argument("--slow", action="store_true", default=False ,help="slow but finer search")
parser.add_argument("--maxres",
type=float,
default=-1,
help="max resolution for cmp")
parser.add_argument("--minrespx",
type=int,
default=4,
help="skip the first x pixel in fourier space")
parser.add_argument("--sym", type=str, help="symmetry. ", default="c1")
parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
#m=EMData(options.ref)
#bxsz=m["nx"]
#apix=m["apix_x"]
options.shrink = 1
pinfo = load_lst_params(options.ptclin)
nptcl = len(pinfo)
#if options.maxshift<0:
#options.maxshift=bxsz//2
print("Initializing parallelism...")
etc = EMTaskCustomer(options.parallel, module="e2spa_align.SpaAlignTask")
num_cpus = etc.cpu_est()
print("{} particles".format(nptcl))
print("{} total CPUs available".format(num_cpus))
if options.debug:
nptcl = min(4 * num_cpus, nptcl)
print("Debugging mode. running on one thread with 8 particles")
nbatch = min(nptcl // 4, num_cpus)
infos = [[] for i in range(nbatch)]
for i, info in enumerate(pinfo):
infos[i % nbatch].append([i, info])
print("{} jobs, each with {:.1f} particles".format(
len(infos), np.mean([len(i) for i in infos])))
tids = []
for info in infos:
task = SpaAlignTask(info, options)
if options.debug:
task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output = [None] * nptcl
for i in tids:
ret = etc.get_results(i)[1]
for r in ret:
output[r[0]] = r[1]
del etc
fm = options.ptclout
save_lst_params(output, fm)
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] stack1.hdf stack2.mrcs ...
Program to erase gold fiducials and other high-density features from images, such as frames in DDD movies or images in tiltseries. Requires scipy.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
#parser.add_argument("--average", default=False, action="store_true", help="Erase gold from average of input stack(s).")
parser.add_argument("--apix",
default=None,
type=float,
help="Override Apix in image header.")
parser.add_argument(
"--lowpass",
default=1.11,
type=float,
help=
"Multiply lowpass filter frequency by this constant when filtering noise image. Default is 1.11."
)
parser.add_argument(
"--coords",
default="",
type=str,
required=True,
help=
"Specify box file with x and y gold coordinates. Must follow standard box file format (x<tab>y<tab>xsize<ysize>) although box sizes are ignored."
)
parser.add_argument(
"--keepdust",
default=False,
action="store_true",
help=
"Do not remove 'dust' from mask (include objects smaller than gold fiducials)."
)
parser.add_argument(
"--goldsize",
default=30,
type=float,
help="Diameter (in pixels) of gold fiducials to erase.")
parser.add_argument(
"--oversample",
default=4,
type=int,
help=
"Oversample noise image to smooth transitions from regions with different noise."
)
parser.add_argument("--boxsize",
default=128,
type=int,
help="Box size to use when computing local noise.")
parser.add_argument("--debug",
default=False,
action="store_true",
help="Save noise and mask/masked image(s).")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-2)
parser.add_argument(
"--parallel",
type=str,
default=None,
help=
"""Default=None (not used). Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel"""
)
parser.add_argument(
"--subset",
default=0,
type=int,
help=
"Default=0 (not used). Apply algorithm to only a subset of images in each stack file."
)
parser.add_argument(
"--nsigmas",
default=3.0,
type=float,
help=
"Default=3.0. Number of standard deviations above the mean to determine pixels to mask out (erase)."
)
(options, args) = parser.parse_args()
nfiles = len(args)
if options.parallel:
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel)
if options.coords:
try:
coords = np.loadtxt(options.coords)
except:
print(
"Failed to read coordinates. Check input box file path and contents."
)
sys.exit(1)
for argnum, arg in enumerate(args):
t0 = time.time()
newarg = ''
originalarg = arg
hdr = EMData(arg, 0,
True) #load header only to get parameters used below
if options.apix: apix = options.apix
else: apix = hdr['apix_x']
nx = hdr['nx']
ny = hdr['ny']
if '.ali' == arg[-4:] or '.mrc' == arg[-4:]:
#Unfortunately, e2proc2d.py appends to existing files instead of overwriting them. If you run this program two consecutive times and the first one failed for whatever reason,
#you'll find your stack growing.
#To prevent this, we create a 'dummy' file, but first remove any dummy files from previous failed runs. (If the program runs successfully to the end, the dummy file gets renamed).
try:
os.remove('dummy_stack.hdf')
except:
pass
#turn .ali or .mrc 3D images into a stack of 2D images that can be processed by this program.
cmd = 'e2proc2d.py ' + arg + ' dummy_stack.hdf --threed2twod'
if options.subset:
cmd += ' --first 0 --last ' + str(options.subset - 1)
runcmd(options, cmd)
#make the new stack of 2D images (dummy_stack.hdf) the new input (the name of the input file but with .hdf format); this intermediate file will be deleted in the end.
newarg = arg.replace(arg[-4:], '.hdf')
os.rename('dummy_stack.hdf', newarg)
arg = newarg
outf = "{}_efd.hdf".format(os.path.splitext(arg)[0])
if os.path.isfile(outf):
print((
"Results are already stored in {}. Please erase or move and try again."
.format(outf)))
sys.exit(1)
nfs = EMUtil.get_image_count(arg)
tasks = []
results = []
results = None
#parallelized tasks don't run "in order"; therefore, a dummy stack needs to be pre-created with as many images as the final stack will have
#(otherwise, writing output images to stack indexes randomly makes the program crash or produces garbage output)
dummy = EMData(nx, ny)
dummy.to_one()
dummy['apix_x'] = apix
dummy['apix_y'] = apix
for j in range(nfs):
dummy.write_image(outf, j)
#EMAN2 does not allow stacks of images with different size; this, and possibly some bug, prevent images written from the parallelization task from
#having the corret size if the pre-created dummy doesn't have the correct size to begin with. No point in writing big images for the dummy from the start.
#re-writing the index=0 image will change the size of all images in the stack to the correct size
dummy_correct_size = EMData(nx, ny)
dummy_correct_size.to_one()
dummy_correct_size['apix_x'] = apix
dummy_correct_size['apix_y'] = apix
dummy.write_image(outf, 0)
for i in range(nfs):
if options.verbose:
sys.stdout.write("\rstaging images ({}/{})".format(i + 1, nfs))
sys.stdout.flush()
if options.parallel:
#print "parallelism started"
task = EraseGold2DTask(options, arg, i, outf)
tasks.append(task)
else:
results = fiximage(options, arg, i, outf)
if options.parallel:
if tasks:
tids = etc.send_tasks(tasks)
if options.verbose:
print("\n(erase_gold) %d tasks queued" % (len(tids)))
results = get_results(etc, tids, options)
#if results:
# #pass
#
# if '.ali' == originalarg[-4:] or '.mrc' == originalarg[-4:]:
# #intermediate = arg.replace('.hdf','.mrcs')
# finaloutput = arg.replace('.hdf',originalarg[-4:])
# cmd = 'e2proc2d.py ' + arg + ' ' + finaloutput + ' --twod2threed --outmode int16'
# runcmd(options,cmd)
# os.remove(arg)
#
# if newarg: os.remove(newarg)
if results:
#pass
# not sure what this was supposed to do
# if options.parallel:
# outfstem = outf.replace('.hdf','')
# cmdbuildstack = 'e2buildstacks.py erasegold_tmp-*_proc.hdf --stackname ' + outfstem
# runcmd(options,cmdbuildstack)
# if options.debug:
# outfmasked = outf.replace('.hdf','_masked.hdf')
# cmdbuildstack = 'e2buildstacks.py erasegold_tmp-*_masked.hdf --stackname ' + outfmasked
# runcmd(options,cmdbuildstack)
# outfnoise= outf.replace('.hdf','_noise.hdf')
# cmdbuildstack = 'e2buildstacks.py erasegold_tmp-*_noise.hdf --stackname ' + outfnoise
# runcmd(options,cmdbuildstack)
if '.ali' == originalarg[-4:] or '.mrc' == originalarg[-4:]:
#intermediate = arg.replace('.hdf','.mrcs')
finaloutput = outf.replace('.hdf', originalarg[-4:])
cmd = 'e2proc2d.py ' + outf + ' ' + finaloutput + ' --twod2threed --outmode int16'
#print "\ncomand to generate finaloutput",cmd
runcmd(options, cmd)
os.remove(arg)
# if newarg:
# try:
# os.remove(newarg)
# except:
# try:
# #print "would have removed",newarg.replace('.hdf','_proc.hdf')
# os.remove(newarg.replace('.hdf','_proc.hdf'))
# except:
# pass
try:
filelist = [
tmpf for tmpf in os.listdir(".") if 'erasegold_tmp' in tmpf
]
for tf in filelist:
os.remove(tf)
except:
print("WARNING: cleanup failed.")
dt = time.time() - t0
if options.verbose:
print("\n")
sys.stdout.write("Erased fiducials from {} ({} minutes)\n".format(
arg, round(old_div(dt, 60.), 2)))
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog <output> [options]
Program to build an initial subtomogram average by averaging pairs from the largest subset
in --input that is a power of 2. For example, if you supply an input stack with 100 subtomograms,
this program will build an initial reference using 64, since 64 is the largest power of 2 contained in 100.
In the first iteration, particle 1 will be averaged with 2, 3 with 4, 5 with 6... etc.
32 new averages (each an average of 2 subtomograms) will be used for the second iteration.
Again, 1 will be averaged with 2, 3 with 4, etc... yielding 16 new averages.
The algorithm continues until the entire subset (64) has been merged into 1 average.
This program imports 'preprocfunc' from e2spt_preproc.py and 'alignment' from e2spt_classaverage.py
--mask=mask.sharp:outer_radius=<safe radius>
--preprocess=filter.lowpass.gauss:cutoff_freq=<1/resolution in A>
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_header(name="sptbtheader",
help="""Options below this label are specific to
sptbinarytree""",
title="### sptbinarytree options ###",
row=6,
col=0,
rowspan=1,
colspan=3,
mode="align")
parser.add_header(
name="caheader",
help="""Options below this label are specific to sptclassaverage""",
title="### sptclassaverage options ###",
row=3,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--path",
type=str,
default='spt_bt',
help=
"""Default=spt. Directory to store results in. The default is a numbered series of directories containing the prefix 'spt'; for example, spt_02 will be the directory by default if 'spt_01' already exists."""
)
parser.add_argument(
"--input",
type=str,
default='',
help=
"""Default=None. The name of the input volume stack. MUST be HDF since volume stack support is required.""",
guitype='filebox',
browser='EMSubTomosTable(withmodal=True,multiselect=False)',
row=0,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--npeakstorefine",
type=int,
help=
"""Default=1. The number of best coarse alignments to refine in search of the best final alignment. Default=1.""",
default=4,
guitype='intbox',
row=9,
col=0,
rowspan=1,
colspan=1,
nosharedb=True,
mode='alignment,breaksym[1]')
parser.add_argument(
"--parallel",
default="thread:1",
help=
"""default=thread:1. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel""",
guitype='strbox',
row=19,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--ppid",
type=int,
help=
"""Default=-1. Set the PID of the parent process, used for cross platform PPID""",
default=-1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"""Default=0. Verbose level [0-9], higner number means higher level of verboseness; 10-11 will trigger many messages that might make little sense since this level of verboseness corresponds to 'debugging mode'"""
)
#parser.add_argument("--resume",type=str,default='',help="""(Not working currently). tomo_fxorms.json file that contains alignment information for the particles in the set. If the information is incomplete (i.e., there are less elements in the file than particles in the stack), on the first iteration the program will complete the file by working ONLY on particle indexes that are missing. For subsequent iterations, all the particles will be used.""")
parser.add_argument(
"--plots",
action='store_true',
default=False,
help=
"""Default=False. Turn this option on to generatea plot of the ccc scores during each iteration. Running on a cluster or via ssh remotely might not support plotting."""
)
parser.add_argument(
"--subset",
type=int,
default=0,
help=
"""Default=0 (not used). Refine only this substet of particles from the stack provided through --input"""
)
parser.add_argument(
"--preavgproc1",
type=str,
default='',
help=
"""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)"""
)
parser.add_argument(
"--preavgproc2",
type=str,
default='',
help=
"""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)"""
)
parser.add_argument(
"--weighbytiltaxis",
type=str,
default='',
help=
"""Default=None. A,B, where A is an integer number and B a decimal. A represents the location of the tilt axis in the tomogram in pixels (eg.g, for a 4096x4096xZ tomogram, this value should be 2048), and B is the weight of the particles furthest from the tomogram. For example, --weighbytiltaxis=2048,0.5 means that praticles at the tilt axis (with an x coordinate of 2048) will have a weight of 1.0 during averaging, while the distance in the x coordinates of particles not-on the tilt axis will be used to weigh their contribution to the average, with particles at the edge(0+radius or 4096-radius) weighing 0.5, as specified by the value provided for B."""
)
parser.add_argument(
"--weighbyscore",
action='store_true',
default=False,
help=
"""Default=False. This option will weigh the contribution of each subtomogram to the average by score/bestscore."""
)
#parser.add_argument("--align",type=str,default="rotate_translate_3d:search=8:delta=12:dphi=12",help="""This is the aligner used to align particles to the previous class average. Default is rotate_translate_3d:search=8:delta=12:dphi=12, specify 'None' (with capital N) to disable.""", returnNone=True,guitype='comboparambox', choicelist='re_filter_list(dump_aligners_list(),\'3d\')', row=12, col=0, rowspan=1, colspan=3, nosharedb=True, mode="alignment,breaksym['rotate_symmetry_3d']")
parser.add_argument(
"--align",
type=str,
default="rotate_translate_3d_tree",
help=
"""Default is rotate_translate_3d_tree. See e2help.py aligners to see the list of parameters the aligner takes (for example, if there's symmetry, supply --align rotate_translate_3d_tree:sym=icos). This is the aligner used to align particles to the previous class average. Specify 'None' (with capital N) to disable."""
)
parser.add_argument(
"--aligncmp",
type=str,
default="ccc.tomo.thresh",
help=
"""Default=ccc.tomo.thresh. The comparator used for the --align aligner. Do not specify unless you need to use anotherspecific aligner.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_cmps_list(),\'tomo\')',
row=13,
col=0,
rowspan=1,
colspan=3,
mode="alignment,breaksym")
#parser.add_argument("--output", type=str, default='avg.hdf', help="""Default=avg.hdf. The name of the output class-average stack. MUST be HDF since volume stack support is required.""", guitype='strbox', row=2, col=0, rowspan=1, colspan=3, mode='alignment,breaksym')
#parser.add_argument("--classmx", type=str, default='', help="""Default=None. The name of the classification matrix specifying how particles in 'input' should be grouped. If omitted, all particles will be averaged.""")
#parser.add_argument("--ref", type=str, default='', help="""Default=None. Reference image(s). Used as an initial alignment reference and for final orientation adjustment if present. This is typically the projections that were used for classification.""", guitype='filebox', browser='EMBrowserWidget(withmodal=True,multiselect=True)', filecheck=False, row=1, col=0, rowspan=1, colspan=3, mode='alignment')
#parser.add_argument("--refpreprocess",action="store_true",default=False,help="""Default=False. This will preprocess the reference identically to the particles. It is off by default, but it is internally turned on when no reference is supplied.""")
#parser.add_argument("--resultmx",type=str,default=None,help="""Default=Npone. Specify an output image to store the result matrix. This is in the same format as the classification matrix. http://blake.bcm.edu/emanwiki/EMAN2/ClassmxFiles""")
#parser.add_argument("--refinemultireftag", type=str, default='', help="""Default=''. DO NOT USE THIS PARAMETER. It is passed on from e2spt_refinemulti.py if needed.""")
'''
ADVANCED parameters
'''
parser.add_argument(
"--averager",
type=str,
default="mean.tomo",
help=
"""Default=mean.tomo. The type of averager used to produce the class average. Default=mean.tomo."""
)
'''
PRE-FFT processing parameters
'''
#parser.add_argument("--nopreprocprefft",action="store_true",default=False,help="""Turns off all preprocessing that happens only once before alignment (--normproc, --mask, --maskfile, --clip, --threshold; i.e., all preprocessing excepting filters --highpass, --lowpass, --preprocess, and --shrink.""")
parser.add_argument(
"--shrink",
type=int,
default=1,
help=
"""Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount for coarse alignment.""",
guitype='shrinkbox',
row=5,
col=1,
rowspan=1,
colspan=1,
mode='alignment,breaksym')
parser.add_argument(
"--shrinkfine",
type=int,
default=1,
help=
"""Default=1 (no shrinking). Optionally shrink the input volumes by an integer amount for refine alignment.""",
guitype='intbox',
row=5,
col=2,
rowspan=1,
colspan=1,
mode='alignment')
parser.add_argument(
"--threshold",
type=str,
default='',
help=
"""Default=None. A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=10,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--mask",
type=str,
default='',
help=
"""Default=None. Masking processor applied to particles before alignment. IF using --clip, make sure to express outer mask radii as negative pixels from the edge.""",
returnNone=True,
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'mask\')',
row=11,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--maskfile",
type=str,
default='',
help=
"""Default=None. Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None."""
)
parser.add_argument(
"--normproc",
type=str,
default='',
help=
"""Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to particles before alignment. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'"""
)
parser.add_argument(
"--clip",
type=int,
default=0,
help=
"""Default=0 (which means it's not used). Boxsize to clip particles as part of preprocessing to speed up alignment. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary; still, there are some benefits from 'oversampling' the data during averaging; so you might still want an average of size 2x, but perhaps particles in a box of 1.5x are sufficiently good for alignment. In this case, you would supply --clip=75"""
)
'''
POST-FFT filtering parameters
'''
parser.add_argument(
"--preprocess",
type=str,
default='',
help=
"""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=10,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--preprocessfine",
type=str,
default='',
help=
"""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--lowpass",
type=str,
default='',
help=
"""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=17,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--lowpassfine",
type=str,
default='',
help=
"""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--highpass",
type=str,
default='',
help=
"""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=18,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--highpassfine",
type=str,
default='',
help=
"""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to FINE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--matchimgs",
action='store_true',
default=False,
help=
"""Default=False. Applies filter.matchto to one image so that it matches the other's spectral profile during preprocessing for pair-wise alignment purposes."""
)
parser.add_argument(
"--filterbyfsc",
action='store_true',
default=False,
help=
"""Default=False. If on, this parameter will use dynamic FSC filtering. --lowpass will be used to build initial references if no --ref supplied, then, the FSC between the even and odd initial references will be used to filter the data during preprocessing. If --ref is supplied, --lowpass will be used during the first iteration to align the particles against the reference. Thereafter, the FSC between the most current particle average and the original reference (--ref) will be used in the next iteration."""
)
'''
OTHER ADVANCED parameters
'''
parser.add_argument(
"--radius",
type=float,
default=0,
help=
"""Default=0 (which means it's not used by default). Hydrodynamic radius of the particle in Angstroms. This will be used to automatically calculate the angular steps to use in search of the best alignment. Make sure the apix is correct on the particles' headers, sine the radius will be converted from Angstroms to pixels. Then, the fine angular step is equal to 360/(2*pi*radius), and the coarse angular step 4 times that."""
)
parser.add_argument(
"--precision",
type=float,
default=1.0,
help=
"""Default=1.0. Precision in pixels to use when figuring out alignment parameters automatically using --radius. Precision would be the number of pixels that the the edge of the specimen is moved (rotationally) during the finest sampling, --falign. If precision is 1, then the precision of alignment will be that of the sampling (apix of your images) times the --shrinkfine factor specified."""
)
parser.add_argument(
"--search",
type=int,
default=8,
help=
""""Default=8. During COARSE alignment translational search in X, Y and Z, in pixels. Default=8. This WILL overwrite any search: provided through --align, EXCEPT if you provide --search=8, which is the default. In general, just avoid providing search twice (through here and through the aligner, --align). If you do, just be careful to make them consistent to minimize misinterpretation and error."""
)
parser.add_argument(
"--searchfine",
type=int,
default=2,
help=
""""Default=2. During FINE alignment translational search in X, Y and Z, in pixels. Default=2. This WILL overwrite any search: provided through --falign, EXCEPT if you provide --searchfine=2, which is the default. In general, just avoid providing search twice (through here and through the fine aligner --falign). If you do, just be careful to make them consistent to minimize misinterpretation and error."""
)
#parser.add_argument("--donotaverage",action="store_true", help="""If e2spt_refinemulti.py is calling e2spt_classaverage.py, the latter need not average any particles, but rather only yield the alignment results.""", default=False)
parser.add_argument(
"--iterstop",
type=int,
default=0,
help=
"""Default=0. (Not used). The program is called to convergence by default (all particles merge into one final average). To stop at an intermediate iteration, provide this parameter. For example, --iterstop=1, will only allow the algorithm to complete 1 iteration; --iterstop=2 will allow it to go through 2, etc."""
)
parser.add_argument(
"--savesteps",
action="store_true",
default=False,
help=
"""Default=False. If set, will save the average after each iteration to class_#.hdf. Each class in a separate file. Appends to existing files.""",
guitype='boolbox',
row=4,
col=0,
rowspan=1,
colspan=1,
mode='alignment,breaksym')
parser.add_argument(
"--saveali",
action="store_true",
default=False,
help=
"""Default=False. If set, will save the aligned particle volumes in class_ptcl.hdf. Overwrites existing file.""",
guitype='boolbox',
row=4,
col=1,
rowspan=1,
colspan=1,
mode='alignment,breaksym')
parser.add_argument(
"--saveallalign",
action="store_true",
default=False,
help=
"""Default=False. If set, will save the alignment parameters after each iteration""",
guitype='boolbox',
row=4,
col=2,
rowspan=1,
colspan=1,
mode='alignment,breaksym')
parser.add_argument(
"--sym",
dest="sym",
default='',
help=
"""Default=None (equivalent to c1). Symmetry to impose -choices are: c<n>, d<n>, h<n>, tet, oct, icos""",
guitype='symbox',
row=9,
col=1,
rowspan=1,
colspan=2,
mode='alignment,breaksym')
parser.add_argument(
"--postprocess",
type=str,
default='',
help=
"""A processor to be applied to the FINAL volume after averaging the raw volumes in their FINAL orientations, after all iterations are done.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=16,
col=0,
rowspan=1,
colspan=3,
mode='alignment,breaksym')
parser.add_argument(
"--procfinelikecoarse",
action='store_true',
default=False,
help=
"""If you supply this parameters, particles for fine alignment will be preprocessed identically to particles for coarse alignment by default. If you supply this, but want specific parameters for preprocessing particles for also supply: fine alignment, nd supply fine alignment parameters, such as --lowpassfine, --highpassfine, etc; to preprocess the particles for FINE alignment differently than for COARSE alignment."""
)
parser.add_argument(
"--falign",
type=str,
default=None,
help=
"""Default=None. This is the second stage aligner used to fine-tune the first alignment.""",
returnNone=True,
guitype='comboparambox',
choicelist='re_filter_list(dump_aligners_list(),\'refine.*3d\')',
row=14,
col=0,
rowspan=1,
colspan=3,
nosharedb=True,
mode='alignment,breaksym[None]')
parser.add_argument(
"--faligncmp",
type=str,
default="ccc.tomo.thresh",
help=
"""Default=ccc.tomo.thresh. The comparator used by the second stage aligner.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_cmps_list(),\'tomo\')',
row=15,
col=0,
rowspan=1,
colspan=3,
mode="alignment,breaksym")
#parser.add_argument("--nopreprocprefft",action="store_true",default=False,help="""Turns off all preprocessing that happens only once before alignment (--normproc, --mask, --maskfile, --clip, --threshold; i.e., all preprocessing excepting filters --highpass, --lowpass, --preprocess, and --shrink.""")
#parser.add_argument("--keep",type=float,default=1.0,help="""Default=1.0 (all particles kept). The fraction of particles to keep in each class.""", guitype='floatbox', row=6, col=0, rowspan=1, colspan=1, mode='alignment,breaksym')
#parser.add_argument("--keepsig", action="store_true", default=False,help="""Default=False. Causes the keep argument to be interpreted in standard deviations.""", guitype='boolbox', row=6, col=1, rowspan=1, colspan=1, mode='alignment,breaksym')
#parser.add_argument("--inixforms",type=str,default="",help="""Default=None. .json file containing a dict of transforms to apply to 'pre-align' the particles.""", guitype='dirbox', dirbasename='spt_|sptsym_', row=7, col=0,rowspan=1, colspan=2, nosharedb=True, mode='breaksym')
parser.add_argument(
"--breaksym",
action="store_true",
default=False,
help=
"""Default=False. Break symmetry. Do not apply symmetrization after averaging, even if searching the asymmetric unit provided through --sym only for alignment. Default=False""",
guitype='boolbox',
row=7,
col=2,
rowspan=1,
colspan=1,
nosharedb=True,
mode=',breaksym[True]')
#parser.add_argument("--groups",type=int,default=0,help="""Default=0 (not used; data not split). This parameter will split the data into a user defined number of groups. For purposes of gold-standard FSC computation later, select --group=2.""")
parser.add_argument(
"--randomizewedge",
action="store_true",
default=False,
help=
"""Default=False. This parameter is EXPERIMENTAL. It randomizes the position of the particles BEFORE alignment, to minimize missing wedge bias and artifacts during symmetric alignment where only a fraction of space is scanned"""
)
#parser.add_argument("--savepreproc",action="store_true", default=False,help="""Default=False. Will save stacks of preprocessed particles (one for coarse alignment and one for fine alignment if preprocessing options are different).""")
parser.add_argument(
"--autocenter",
type=str,
default='',
help=
"""Default=None. Autocenters each averaged pair during initial average generation with --btref and --hacref. Will also autocenter the average of all particles after each iteration of iterative refinement. Options are --autocenter=xform.centerofmass (self descriptive), or --autocenter=xform.centeracf, which applies auto-convolution on the average."""
)
parser.add_argument(
"--autocentermask",
type=str,
default='',
help=
"""Default=None. Masking processor to apply before autocentering. See 'e2help.py processors -v 10' at the command line."""
)
parser.add_argument(
"--autocenterpreprocess",
action='store_true',
default=False,
help=
"""Default=False. This will apply a highpass filter at a frequency of half the box size times the apix, shrink by 2, and apply a low pass filter at half nyquist frequency to any computed average for autocentering purposes if --autocenter is provided. Default=False."""
)
parser.add_argument(
"--tweak",
action='store_true',
default=False,
help=
"""WARNING: BUGGY. This will perform a final alignment with no downsampling [without using --shrink or --shrinkfine] if --shrinkfine > 1."""
)
'''
BT SPECIFIC PARAMETERS
'''
parser.add_argument("--nseedlimit",
type=int,
default=0,
help="""Maximum number of particles
to use. For example, if you supply a stack with 150 subtomograms, the program will
automatically select 128 as the limit to use because it's the largest power of 2 that is
smaller than 150. But if you provide, say --nseedlimit=100, then the number of particles
used will be 64, because it's the largest power of 2 that is still smaller than 100."""
)
(options, args) = parser.parse_args()
(optionsUnparsed, args) = parser.parse_args()
options.nopreprocprefft = False
if options.shrink < options.shrinkfine:
options.shrink = options.shrinkfine
print(
"\n(e2spt_binarytree)(main) it makes no sense for shrinkfine to be larger than shrink; therefore, shrink will be made to match shrinkfine"
)
options = checkinput(options)
from e2spt_classaverage import checksaneimagesize
checksaneimagesize(options, options.input)
'''
Make the directory where to create the database where the results will be stored
'''
#from e2spt_classaverage import sptmakepath
#options = sptmakepath(options,'spt_bt')
from EMAN2_utils import makepath
options = makepath(options, 'spt_bt')
optionsUnparsed.path = options.path
rootpath = os.getcwd()
if rootpath not in options.path:
options.path = rootpath + '/' + options.path
if not options.input:
parser.print_help()
exit(0)
elif options.subset:
subsetStack = options.path + '/subset' + str(options.subset).zfill(
len(str(options.subset))) + '.hdf'
print("\nSubset to be written to", subsetStack)
subsetcmd = 'e2proc3d.py ' + options.input + ' ' + subsetStack + ' --first=0 --last=' + str(
options.subset - 1)
print("Subset cmd is", subsetcmd)
p = subprocess.Popen(subsetcmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
text = p.communicate()
p.stdout.close()
options.input = subsetStack
from e2spt_classaverage import sptParseAligner
options = sptParseAligner(options)
'''
If --radius of the particle is provided, we calculate the optimal alignment steps for
coarse and fine alignment rounds using --shrink and --shrinkfine options and apix info
'''
if options.radius:
from e2spt_classaverage import calcAliStep
options = calcAliStep(options)
'''c:
c:Parse parameters such that "None" or "none" are adequately interpreted to turn off an option
c:'''
options = sptOptionsParser(options)
print("\nAFTER PARSING options type={}".format(type(options)))
print("\nAFTER PARSING optionsUnparsed type={}".format(
type(optionsUnparsed)))
writeParameters(options, 'e2spt_binarytree.py', 'spt_bt')
hdr = EMData(options.input, 0, True)
nx = hdr["nx"]
ny = hdr["ny"]
nz = hdr["nz"]
if nx != ny or ny != nz:
print("ERROR, input volumes are not cubes")
sys.exit(1)
logger = E2init(sys.argv, options.ppid)
'''
Initialize parallelism if being used; it will be turned on automatically by detectThreads unless --parallel-=None
'''
options = detectThreads(options)
if options.parallel:
print("\n\n(e2spt_classaverage.py) INITIALIZING PARALLELISM!")
print("\n\n")
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel)
pclist = [options.input]
etc.precache(pclist)
else:
etc = ''
options.raw = options.input
"""
if 'tree' in options.align:
options.falign = None
options.mask = None
options.lowpass = None
options.highpass = None
options.normproc = None
options.lowpassfine = None
options.highpassfine = None
options.preprocess = None
options.preprocessfine = None
else:
from e2spt_classaverage import cmdpreproc
cmdpreproc( options.input, options, False )
"""
nptcl = EMUtil.get_image_count(options.input)
if nptcl < 1:
print("ERROR : at least 2 particles required in input stack")
sys.exit(1)
ptclnums = list(range(nptcl))
nptclForRef = len(ptclnums)
nseed = 2**int(
floor(log(len(ptclnums), 2))
) # we stick with powers of 2 for this to make the tree easier to collapse
if options.nseedlimit:
nseed = 2**int(floor(log(options.nseedlimit, 2)))
#binaryTreeRef(options,nptclForRef,nseed,-1,etc)
binaryTreeRef(options, optionsUnparsed, nptclForRef, nseed, etc)
print("Will end logger")
E2end(logger)
print("logger ended")
sys.stdout.flush()
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
This program is part of the 'new' hierarchy of e2spt_ programs. It performs one iteration of a classical subtomogram refinement, ie - aligning particles with missing wedge to a reference in 3-D
The reference may be <volume> or <volume>,<n>
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--threads", default=4,type=int,help="Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.", guitype='intbox', row=24, col=2, rowspan=1, colspan=1, mode="refinement")
parser.add_argument("--iter",type=int,help="Iteration number within path. Default = start a new iteration",default=0)
parser.add_argument("--goldcontinue",action="store_true",help="Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",default=False)
parser.add_argument("--path",type=str,default=None,help="Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)")
parser.add_argument("--sym",type=str,default="c1",help="Symmetry of the input. Must be aligned in standard orientation to work properly.")
parser.add_argument("--maxres",type=float,help="Maximum resolution to consider in alignment (in A, not 1/A)",default=0)
parser.add_argument("--minres",type=float,help="Minimum resolution to consider in alignment (in A, not 1/A)",default=0)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
parser.add_argument("--maxshift", type=int, help="maximum shift for subtilt refine",default=8)
parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
parser.add_argument("--fromscratch",action="store_true",help=".",default=False)
parser.add_argument("--skipali3d",action="store_true",help="",default=False)
parser.add_argument("--skipali2d",action="store_true",help="",default=False)
parser.add_argument("--debug",action="store_true",help=".",default=False)
parser.add_argument("--plst",type=str,default=None,help="list of 2d particle with alignment parameters. will reconstruct before alignment.")
parser.add_argument("--smooth",type=float,help="smooth local motion by this factor. smoother local motion with larger numbers",default=-1)
parser.add_argument("--smoothN",type=float,help="number of neighboring particles used for smoothing. default 15",default=15)
(options, args) = parser.parse_args()
if options.path == None:
fls=[int(i[-2:]) for i in os.listdir(".") if i[:4]=="spt_" and len(i)==6 and str.isdigit(i[-2:])]
if len(fls)==0 : fls=[0]
options.path = "spt_{:02d}".format(max(fls)+1)
try: os.mkdir(options.path)
except: pass
if options.iter<=0 :
fls=[int(i[15:17]) for i in os.listdir(options.path) if i[:15]=="particle_parms_" and str.isdigit(i[15:17])]
if len(fls)==0 : options.iter=1
else: options.iter=max(fls)+1
if options.parallel==None:
options.parallel="thread:{}".format(options.threads)
# file may be "name" or "name,#"
reffile=args[1].split(",")[0]
try: refn=int(args[1].split(",")[1])
except: refn=0
NTHREADS=max(options.threads+1,2) # we have one thread just writing results
logid=E2init(sys.argv, options.ppid)
refnames=[reffile, reffile]
if options.goldcontinue:
ref=[]
try:
refnames=[reffile[:-4]+"_even.hdf", reffile[:-4]+"_odd.hdf"]
ref.append(EMData(refnames[0],0))
ref.append(EMData(refnames[1],0))
except:
print("Error: cannot find one of reference files, eg: ",EMData(reffile[:-4]+"_even.hdf",0))
n=-1
tasks=[]
readjson=False
if args[0].endswith(".lst") or args[0].endswith(".hdf"):
nptcl=EMUtil.get_image_count(args[0])
tasks.extend([(args[0],i,refnames, i%2) for i in range(nptcl)])
elif args[0].endswith(".json"):
js=js_open_dict(args[0])
readjson=True
jsinput=dict(js)
keys=sorted(js.keys())
nptcl=len(keys)
for k in keys:
src, ii=eval(k)
dic=js[k]
xf=dic["xform.align3d"]
tasks.append([src, ii, refnames, ii%2, xf])
if options.plst:
xinfo=[[] for i in range(nptcl)]
xfkey=["type","alt","az","phi","tx","ty","tz","alpha","scale"]
lst=LSXFile(options.plst)
for i in range(lst.n):
l=lst.read(i)
dc=eval(l[2])
pid=dc["pid"]
dxf=Transform({k:dc[k] for k in dc.keys() if k in xfkey})
xinfo[pid].append([dc["tid"],i, dxf])
lst=None
for i,xn in enumerate(xinfo):
sid=np.argsort([x[0] for x in xn])
xinfo[i]=[[xn[s][1],xn[s][2]] for s in sid]
for i in range(nptcl):
ii=tasks[i][1]
tasks[i].append(xinfo[ii])
from EMAN2PAR import EMTaskCustomer
etc=EMTaskCustomer(options.parallel, module="e2spt_align_subtlt.SptAlignTask")
num_cpus = etc.cpu_est()
options.nowtime=time.time()
if options.debug:
tasks=tasks[:num_cpus*4]
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob=num_cpus
tids=[]
for i in range(njob):
t=tasks[i::njob]
task=SptAlignTask(t, options)
if options.debug:
ret=task.execute(print)
return
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals)/100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output=[]
for i in tids:
rets=etc.get_results(i)[1]
for ret in rets:
fsp,n,dic=ret
output.append([fsp, n, dic])
angs={}
data=[]
for out in output:
fsp,n,dc=out
angs[(fsp, n)]={
"xform.align3d":dc["xform.align3d"],
"score":dc["score"]}
ts=dc.pop("transscr")
for i, ii in enumerate(dc["imgidx"]):
dxf=dc["imgxfs"][i].get_params("eman")
#dxf["score"]=float(dc["imgscore"][i])
dxf["pid"]=n; dxf["tid"]=i; dxf["class"]=n%2
tsmult=dc["tsmult"]
data.append({
"src":dc["imgsrc"], "srci":ii, "pid":n, "tid":i,
"xf":dxf, "coord":dc["coord"], "tsscr":ts[i]
})
m=options.maxshift
trans=np.indices((m*2+1, m*2+1)).reshape((2,-1)).T-m
alldtx=np.zeros((len(data), 3))
fnames=np.unique([d["src"] for d in data])
maxtid=np.max([d["tid"] for d in data])+1
print("Smoothing trajectories...")
for fname in fnames:
print(fname)
for tid in range(maxtid):
idx=[i for i,d in enumerate(data) if d["src"]==fname and d["tid"]==tid]
if len(idx)==0:
continue
coord=np.array([data[i]["coord"] for i in idx])
scrs=np.array([data[i]["tsscr"] for i in idx])
if options.smooth<=0:
s=np.argmin(scrs, 1)
sv=np.min(scrs, axis=1)
newdtx=np.hstack([trans[s], sv[:,None]])
if tid==0:print(" Skip smoothing")
else:
newdtx=[]
for i, crd in enumerate(coord):
dst=np.linalg.norm(coord-crd, axis=1)
srtid=np.argsort(dst)[:options.smoothN]
dst=dst[srtid]
wt=np.exp(-dst/options.smooth)
scr=scrs[srtid].copy()
scr=np.sum(scr*wt[:,None], axis=0)
pos=trans[np.argmin(scr)].copy()
pos=pos.astype(float)*tsmult
newdtx.append([pos[0], pos[1], np.min(scr)])
sys.stdout.write("\r {}: {:.4f}".format(tid, np.mean(scr)))
sys.stdout.flush()
newdtx=np.array(newdtx)
alldtx[idx]=newdtx
print()
for i,d in enumerate(data):
dxf=d["xf"]
dxf["tx"]+=alldtx[i][0]
dxf["ty"]+=alldtx[i][1]
dxf["score"]=alldtx[i][2]
d["xf"]=dxf
fm="{}/aliptcls_{:02d}.lst".format(options.path, options.iter)
if os.path.isfile(fm): os.remove(fm)
lout=LSXFile(fm, False)
for i,d in enumerate(data):
lout.write(-1, d["srci"], d["src"], d["xf"])
lout=None
f="{}/aliptcls_ts_{:02d}.hdf".format(options.path,options.iter)
if os.path.isfile(f): os.remove(f)
t=np.array([d["tsscr"] for d in data])
m=from_numpy(t).copy()
m.write_image(f)
#print(f, t.shape)
out="{}/particle_parms_{:02d}.json".format(options.path,options.iter)
if os.path.isfile(out):
os.remove(out)
js=js_open_dict(out)
js.update(angs)
js.close()
del etc
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options]
This program aligns a paricle to its symmetry axis. There are two algorithmic modes.
A coarse search followed by simplex minimization (not yet implimented) OR monte carlo course
search followed by simplex minimization. The Goal is to align the paricle to its
symmetry axis so symmetry can be applied for avergaing and for alignment speed up
(it is only necessary to search over one asymmetric unit!
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_header(
name="symsearch3dheader",
help="""Options below this label are specific to e2symsearch3d""",
title="### e2symsearch3d options ###",
row=3,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--input",
dest="input",
default='',
type=str,
help="""The name of input volume or hdf stack of volumes""",
guitype='filebox',
browser="EMBrowserWidget(withmodal=True,multiselect=False)",
row=0,
col=0,
rowspan=1,
colspan=2)
#parser.add_argument("--output", dest="output", default="""e2symsearch3d_OUTPUT.hdf""", type=str, help="The name of the output volume", guitype='strbox', filecheck=False, row=1, col=0, rowspan=1, colspan=2)
parser.add_argument(
"--ref",
type=str,
default='',
help=
"""Default=None. If provided and --average is also provided and --keep < 1.0 or --keepsig is specified, 'good particles' will be determined by correlation to --ref."""
)
parser.add_argument(
"--mirror",
type=str,
default='',
help=
"""Axis across of which to generate a mirrored copy of --ref. All particles will be compared to it in addition to the unmirrored image in --ref if --keepsig is provided or if --keep < 1.0."""
)
parser.add_argument("--path",
type=str,
default='',
help="""Name of path for output file""",
guitype='strbox',
row=2,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--plots",
action='store_true',
default=False,
help=
"""Default=False. Turn this option on to generate a plot of the ccc scores if --average is supplied. Running on a cluster or via ssh remotely might not support plotting."""
)
parser.add_argument(
"--sym",
dest="sym",
default="c1",
help=
"""Specify symmetry -choices are: c<n>, d<n>, h<n>, tet, oct, icos. For asymmetric reconstruction ommit this option or specify c1.""",
guitype='symbox',
row=4,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--shrink",
dest="shrink",
type=int,
default=0,
help=
"""Optionally shrink the input particles by an integer amount prior to computing similarity scores. For speed purposes. Default=0, no shrinking""",
guitype='shrinkbox',
row=5,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--mask",
type=str,
help=
"""Mask processor applied to particles before alignment. Default is mask.sharp:outer_radius=-2. IF using --clipali, make sure to express outer mask radii as negative pixels from the edge.""",
returnNone=True,
default="mask.sharp:outer_radius=-2",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'mask\')',
row=11,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--maskfile",
type=str,
default='',
help=
"""Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None."""
)
parser.add_argument(
"--normproc",
type=str,
default='',
help=
"""Normalization processor applied to particles before alignment. Default is to use normalize. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'"""
)
parser.add_argument(
"--threshold",
default='',
type=str,
help=
"""A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=10,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--preprocess",
default='',
type=str,
help=
"""Any processor (as in e2proc3d.py) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=10,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--lowpass",
type=str,
default='',
help=
"""A lowpass filtering processor (from e2proc3d.py; see e2help.py processors) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=17,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--highpass",
type=str,
default='',
help=
"""A highpass filtering processor (from e2proc3d.py, see e2help.py processors) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging.""",
guitype='comboparambox',
choicelist='re_filter_list(dump_processors_list(),\'filter\')',
row=18,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--clipali",
type=int,
default=0,
help=
"""Boxsize to clip particles as part of preprocessing to speed up alignment. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary; still, there are some benefits from 'oversampling' the data during averaging; so you might still want an average of size 2x, but perhaps particles in a box of 1.5x are sufficiently good for alignment. In this case, you would supply --clipali=75"""
)
parser.add_argument(
"--savepreprocessed",
action="store_true",
default=False,
help=
"""Default=False. Will save stacks of preprocessed particles (one for coarse alignment and one for fine alignment if preprocessing options are different)."""
)
parser.add_argument(
"--average",
action='store_true',
default=False,
help=
"""Default=False. If supplied and a stack is provided through --input, the average of the aligned and/or symmetrized stack will also be saved."""
)
parser.add_argument(
"--averager",
type=str,
default="mean.tomo",
help=
"""Default=mean.tomo. The type of averager used to produce the class average. Default=mean.tomo."""
)
parser.add_argument(
"--keep",
type=float,
default=1.0,
help=
"""Fraction of particles to include if --average is on, after correlating the particles with the average."""
)
parser.add_argument(
"--keepsig",
action="store_true",
default=False,
help=
"""Default=False. Causes theoptions.keep argument to be interpreted in standard deviations.""",
guitype='boolbox',
row=6,
col=1,
rowspan=1,
colspan=1,
mode='alignment,breaksym')
parser.add_argument(
"--avgiter",
type=int,
default=1,
help=
"""Default=1. If --keep is different from 1.0 and --average is on, the initial average will include all the particles, but then the percent specified byoptions.keep will be kept (the rest thrown away) and a new average will be computed. If --avgiter > 1, this new average will be compared again against all the particles. The procedure will be repeated for however many iterations --avgiter is given, or the process will stop automatically if in two consecutive rounds exactly the same particles are kept"""
)
parser.add_argument(
'--subset',
type=int,
default=0,
help=
"""Number of particles in a subset of particles from the --input stack of particles to run the alignments on."""
)
parser.add_argument("--steps",
dest="steps",
type=int,
default=10,
help="""Number of steps (for the MC). Default=10.""",
guitype='intbox',
row=5,
col=1,
rowspan=1,
colspan=1)
parser.add_argument("--symmetrize",
default=False,
action="store_true",
help="""Symmetrize volume after alignment.""",
guitype='boolbox',
row=6,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--cmp",
type=str,
help=
"""The name of a 'cmp' to be used in comparing the symmtrized object to unsymmetrized""",
default="ccc",
guitype='comboparambox',
choicelist='re_filter_list(dump_cmps_list(),\'tomo\', True)',
row=7,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--parallel",
"-P",
type=str,
help=
"""Run in parallel, specify type:<option>=<value>:<option>:<value>""",
default=None,
guitype='strbox',
row=8,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--ppid",
type=int,
help=
"""Set the PID of the parent process, used for cross platform PPID.""",
default=-1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"""verbose level [0-9], higner number means higher level ofoptions.verboseness."""
)
parser.add_argument(
"--nopath",
action='store_true',
default=False,
help=
"""If supplied, this option will save results in the directory where the command is run. A directory to store the results will not be made."""
)
parser.add_argument(
"--nolog",
action='store_true',
default=False,
help=
"""If supplied, this option will prevent logging the command run in .eman2log.txt."""
)
parser.add_argument(
"--saveali",
action='store_true',
default=False,
help="""Save the stack of aligned/symmetrized particles.""")
parser.add_argument(
"--savesteps",
action='store_true',
default=False,
help=
"""If --avgiter > 1, save all intermediate averages and intermediate aligned kept stacks."""
)
parser.add_argument(
"--notmatchimgs",
action='store_true',
default=False,
help=
"""Default=True. This option prevents applying filter.match.to to one image so that it matches the other's spectral profile during preprocessing for alignment purposes."""
)
parser.add_argument(
"--preavgproc1",
type=str,
default='',
help=
"""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)"""
)
parser.add_argument(
"--preavgproc2",
type=str,
default='',
help=
"""Default=None. A processor (see 'e2help.py processors -v 10' at the command line) to be applied to the raw particle after alignment but before averaging (for example, a threshold to exclude extreme values, or a highphass filter if you have phaseplate data.)"""
)
parser.add_argument(
"--weighbytiltaxis",
type=str,
default='',
help=
"""Default=None. A,B, where A is an integer number and B a decimal. A represents the location of the tilt axis in the tomogram in pixels (eg.g, for a 4096x4096xZ tomogram, this value should be 2048), and B is the weight of the particles furthest from the tomogram. For example, --weighbytiltaxis=2048,0.5 means that praticles at the tilt axis (with an x coordinate of 2048) will have a weight of 1.0 during averaging, while the distance in the x coordinates of particles not-on the tilt axis will be used to weigh their contribution to the average, with particles at the edge(0+radius or 4096-radius) weighing 0.5, as specified by the value provided for B."""
)
parser.add_argument(
"--weighbyscore",
action='store_true',
default=False,
help=
"""Default=False. This option will weigh the contribution of each subtomogram to the average by score/bestscore."""
)
parser.add_argument(
"--align",
type=str,
default='symalignquat',
help=
"""Default=symalignquat. WARNING: The aligner cannot be changed for this program currently. Option ignored."""
)
parser.add_argument(
"--tweak",
action='store_true',
default=False,
help=
"""WARNING: Not used for anything yet. This will perform a final alignment with no downsampling [without using --shrink or --shrinkfine] if --shrinkfine > 1."""
)
(options, args) = parser.parse_args()
if not options.input:
parser.print_help()
sys.exit(0)
#If no failures up until now, initialize logger
log = 0
if not options.nolog:
logid = E2init(sys.argv, options.ppid)
log = 1
#inimodeldir = os.path.join(".",options.path)
#if not os.access(inimodeldir, os.R_OK):
# os.mkdir(options.path)
#Make directory to save results
from e2spt_classaverage import sptmakepath
options = sptmakepath(options, 'symsearch')
if options.nopath:
options.path = '.'
rootpath = os.getcwd()
if rootpath not in options.path:
options.path = rootpath + '/' + options.path
from e2spt_classaverage import preprocessing
from EMAN2PAR import EMTaskCustomer
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser(options)
avgr = Averagers.get(options.averager[0], options.averager[1])
results = {}
scores = []
outputstack = options.path + '/all_ptcls_ali.hdf'
#Determine number of particles in the stack
n = EMUtil.get_image_count(options.input)
if options.subset and options.subset < n:
n = options.subset
for i in range(n):
print "\nI'll look for symmetry in particle number", i
#Load particle and make a copy to modify if preprocessing options are specified
volume = EMData(options.input, i)
preprocvol = volume.copy()
#Preprocess volume if any preprocessing options are specified
if (
options.shrink and options.shrink > 1
) or options.mask or options.maskfile or options.lowpass or options.highpass or options.normproc or options.preprocess or options.threshold or options.clipali:
print "\nHowever, I will first preprocess particle number", i
print "\nWill call preprocessing on ptcl", i
preprocvol = preprocessing(preprocvol, options, i)
#preprocessing(s2image,options, ptclindx, savetagp ,'no',round)
print "\nDone preprocessing on ptcl", i
if options.parallel:
etc = EMTaskCustomer(options.parallel)
else:
etc = EMTaskCustomer("thread:1")
symalgorithm = SymALignStrategy(preprocvol, options.sym, options.steps,
options.cmp, etc)
ret = symalgorithm.execute()
symxform = ret[0]
score = ret[1]
scores.append(score)
results.update({score: [symxform, i]})
print "\nWriting output for best alignment found for particle number", i
if options.shrink and options.shrink > 1:
trans = symxform.get_trans()
symxform.set_trans(trans[0] * options.shrink,
trans[1] * options.shrink,
trans[2] * options.shrink)
print "\nWrittng to output ptcl", i
#Rotate volume to the best orientation found, set the orientation in the header, apply symmetry if specified and write out the aligned (and symmetrized) particle to the output stack
output = volume.process('xform', {'transform': symxform})
output.set_attr('symxform', symxform)
print "\nApplying this transform to particle", symxform
if options.symmetrize:
output = output.process('xform.applysym', {'sym': options.sym})
output['spt_score'] = score
output.write_image(outputstack, -1)
#Averaging here only makes sense if all particles are going to be kept. Otherwise, different code is needed (below)
if options.average and options.keep == 1.0 and not options.keepsig:
avgr.add_image(output)
#Finalize average of all particles if non were set to be excluded. Otherwise, determine the discrimination threshold and then average the particles that pass it.
if options.average:
final_avg = avgr.finish()
final_avg['origin_x'] = 0
final_avg[
'origin_y'] = 0 #The origin needs to be reset to ZERO to avoid display issues in Chimera
final_avg['origin_z'] = 0
final_avg['xform.align3d'] = Transform()
if options.keep == 1.0 and not options.keepsig:
final_avg.write_image(options.path + '/final_avg.hdf', 0)
if options.avgiter > 1:
print """WARNING: --avgiter > 1 must be accompanied by --keepsig, or by --keep < 1.0"""
elif options.keep < 1.0 or options.keepsig:
if options.ref:
ref = EMData(options.ref, 0)
refComp(options, outputstack, ref, results, '')
if options.mirror:
ref.process_inplace('xform.mirror',
{'axis': options.mirror})
refComp(options, outputstack, ref, results, '_vs_mirror')
else:
ref2compare = final_avg
refComp(options, outputstack, final_avg, results, '')
if log:
E2end(logid)
return
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
This program is part of the 'new' hierarchy of e2spt_ programs. It performs one iteration of a classical subtomogram refinement, ie - aligning particles with missing wedge to a reference in 3-D
The reference may be <volume> or <volume>,<n>
If --goldstandard is specified, then even and odd particles will be aligned to different perturbed versions of the reference volume, phase-randomized past the specified resolution."""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--threads",
default=4,
type=int,
help=
"Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.",
guitype='intbox',
row=24,
col=2,
rowspan=1,
colspan=1,
mode="refinement")
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=0)
parser.add_argument(
"--goldstandard",
type=float,
help=
"If specified, will phase randomize the even and odd references past the specified resolution (in A, not 1/A)",
default=0)
parser.add_argument(
"--goldcontinue",
action="store_true",
help=
"Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
default=False)
#parser.add_argument("--saveali",action="store_true",help="Save a stack file (aliptcls.hdf) containing the aligned subtomograms.",default=False)
#parser.add_argument("--savealibin",type=int,help="shrink aligned particles before saving",default=1)
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
)
parser.add_argument(
"--sym",
type=str,
default="c1",
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly."
)
parser.add_argument(
"--maxres",
type=float,
help="Maximum resolution to consider in alignment (in A, not 1/A)",
default=0)
#parser.add_argument("--wtori",type=float,help="Weight for using the prior orientation in the particle header. default is -1, i.e. not used.",default=-1)
parser.add_argument(
"--nsoln",
type=int,
help="number of solutions to keep at low resolution for the aligner",
default=1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default=None)
parser.add_argument("--refine",
action="store_true",
help="local refinement from xform.align3d in header.",
default=False)
parser.add_argument("--refinentry",
type=int,
help="number of tests for refine mode. default is 8",
default=8)
parser.add_argument("--randphi",
action="store_true",
help="randomize phi during refine alignment",
default=False)
parser.add_argument("--breaksym",
action="store_true",
help="symmetry breaking.",
default=False)
parser.add_argument(
"--breaksymsym",
type=str,
help=
"the symmetry to use for breaksym. setting sym to c6 and this to c2 results in a c3 structure. default is the same as sym",
default=None)
parser.add_argument(
"--rand180",
action="store_true",
help="randomly add a 180 degree rotation during refine alignment",
default=False)
parser.add_argument(
"--skipali",
action="store_true",
help=
"skip alignment. the program will do nothing. mostly for testing...",
default=False)
parser.add_argument(
"--maxang",
type=float,
help="Maximum angular difference for the refine mode. default is 30",
default=30)
parser.add_argument(
"--maxshift",
type=float,
help="Maximum shift for the refine mode. default is 16",
default=-1)
(options, args) = parser.parse_args()
#task=SptAlignTask(0,1,2,options)
#from pickle import dumps,loads,dump,load
#f=open("task.tmp",'w')
#dump(task,f)
#f.close()
#print(task)
#return
if options.path == None:
fls = [
int(i[-2:]) for i in os.listdir(".")
if i[:4] == "spt_" and len(i) == 6 and str.isdigit(i[-2:])
]
if len(fls) == 0: fls = [0]
options.path = "spt_{:02d}".format(max(fls) + 1)
try:
os.mkdir(options.path)
except:
pass
if options.iter <= 0:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0: options.iter = 1
else: options.iter = max(fls) + 1
if options.parallel == None:
options.parallel = "thread:{}".format(options.threads)
if options.breaksym:
if options.breaksymsym == None:
if options.sym == "c1":
print("cannot break a c1 symmetry. breaksym disabled.")
options.breaksym = False
else:
options.breaksymsym = options.sym
# file may be "name" or "name,#"
reffile = args[1].split(",")[0]
try:
refn = int(args[1].split(",")[1])
except:
refn = 0
NTHREADS = max(options.threads + 1,
2) # we have one thread just writing results
logid = E2init(sys.argv, options.ppid)
refnames = []
if options.goldcontinue:
ref = []
try:
refnames = [reffile[:-4] + "_even.hdf", reffile[:-4] + "_odd.hdf"]
ref.append(EMData(refnames[0], 0))
ref.append(EMData(refnames[1], 0))
except:
print("Error: cannot find one of reference files, eg: ",
EMData(reffile[:-4] + "_even.hdf", 0))
else:
ref = []
ref.append(EMData(reffile, refn))
ref.append(EMData(reffile, refn))
if options.goldstandard > 0:
ref[0].process_inplace(
"filter.lowpass.randomphase",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[0].process_inplace(
"filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[1].process_inplace(
"filter.lowpass.randomphase",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[1].process_inplace(
"filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
refnames = [
"{}/align_ref_even.hdf".format(options.path),
"{}/align_ref_odd.hdf".format(options.path)
]
ref[0].write_image(refnames[0], 0)
ref[1].write_image(refnames[1], 0)
else:
refnames = [reffile, reffile]
#ref[0]=ref[0].do_fft()
#ref[0].process_inplace("xform.phaseorigin.tocorner")
#ref[1]=ref[1].do_fft()
#ref[1].process_inplace("xform.phaseorigin.tocorner")
#jsd=queue.Queue(0)
n = -1
tasks = []
if args[0].endswith(".lst") or args[0].endswith(".hdf"):
#### check if even/odd split exists
fsps = [args[0][:-4] + "__even.lst", args[0][:-4] + "__odd.lst"]
if os.path.isfile(fsps[0]) and os.path.isfile(fsps[1]):
print("Using particle list: \n\t {} \n\t {}".format(
fsps[0], fsps[1]))
for eo, f in enumerate(fsps):
N = EMUtil.get_image_count(f)
tasks.extend([(f, i, refnames, eo) for i in range(N)])
#### split by even/odd by default
else:
N = EMUtil.get_image_count(args[0])
tasks.extend([(args[0], i, refnames, i % 2) for i in range(N)])
#thrds=[threading.Thread(target=alifn,args=(jsd,args[0],i,ref[i%2],options)) for i in range(N)]
elif args[0].endswith(".json"):
print("Reading particles from json. This is experimental...")
js = js_open_dict(args[0])
keys = sorted(js.keys())
for k in keys:
src, ii = eval(k)
dic = js[k]
xf = dic["xform.align3d"]
tasks.append([src, ii, refnames, ii % 2, xf])
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel, module="e2spt_align.SptAlignTask")
num_cpus = etc.cpu_est()
options.nowtime = time.time()
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob = num_cpus #*4
tids = []
for i in range(njob):
t = tasks[i::njob]
task = SptAlignTask(t, options)
tid = etc.send_task(task)
tids.append(tid)
#print("starting...")
#for t in tasks:
#if len(t)>3:
#task = SptAlignTask(t[0], t[1], t[2], options, t[3])
#else:
#task = SptAlignTask(t[0], t[1], t[2], options)
#tid=etc.send_task(task)
#tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
#dics=[0]*nptcl
angs = {}
for i in tids:
rets = etc.get_results(i)[1]
for ret in rets:
fsp, n, dic = ret
if len(dic) == 1:
angs[(fsp, n)] = dic[0]
else:
angs[(fsp, n)] = dic
out = "{}/particle_parms_{:02d}.json".format(options.path, options.iter)
if os.path.isfile(out):
os.remove(out)
js = js_open_dict(out)
js.update(angs)
js.close()
del etc
## here we run the threads and save the results, no actual alignment done here
#print(len(thrds)," threads")
#thrtolaunch=0
#while thrtolaunch<len(thrds) or threading.active_count()>1:
## If we haven't launched all threads yet, then we wait for an empty slot, and launch another
## note that it's ok that we wait here forever, since there can't be new results if an existing
## thread hasn't finished.
#if thrtolaunch<len(thrds) :
#while (threading.active_count()==NTHREADS ) : time.sleep(.1)
#if options.verbose : print("Starting thread {}/{}".format(thrtolaunch,len(thrds)))
#thrds[thrtolaunch].start()
#thrtolaunch+=1
#else: time.sleep(1)
#while not jsd.empty():
#fsp,n,d=jsd.get()
#angs[(fsp,n)]=d
#if options.saveali:
#v=EMData(fsp,n)
#v.transform(d["xform.align3d"])
#if options.savealibin>1:
#v.process_inplace("math.meanshrink",{"n":options.savealibin})
#v.write_image("{}/aliptcls_{:02d}.hdf".format(options.path, options.iter),n)
#for t in thrds:
#t.join()
E2end(logid)
def main():
parser = EMArgumentParser(usage="")
parser.add_argument("--output", default="threed.hdf", help="Output reconstructed volume file name.")
parser.add_argument("--input", default=None, help="The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion")
parser.add_argument("--sym", dest="sym", default="c1", help="Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct.")
parser.add_argument("--pad", default=-1,type=int, help="Will zero-pad images to the specifed size. ")
parser.add_argument("--outsize", default=-1, type=int, help="Defines the dimensions of the final volume written to disk")
parser.add_argument("--keep", type=float, dest="keep", help="The fraction of slices to keep, based on quality scores (1.0 = use all slices). See keepsig.",default=.9)
parser.add_argument("--no_wt", action="store_true", dest="no_wt", default=False, help="This argument turns automatic weighting off causing all images to be weighted by 1.")
parser.add_argument("--mode", type=str, default="trilinear", help="Fourier reconstruction 'mode' to use. The default should not normally be changed.")
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--apix",metavar="A/pix",type=float,help="A/pix value for output, overrides automatic values",default=None)
parser.add_argument("--ref", type=str,help="ref", default=None)
parser.add_argument("--tidrange", type=str,help="range of tilt id to include", default="-1,-1")
parser.add_argument("--minres", type=float,help="", default=200)
parser.add_argument("--maxres", type=float,help="", default=5)
parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
parser.add_argument("--debug", action="store_true", default=False, help="")
parser.add_argument("--clsid", default=-1, type=int, help="only reconstruct a class of particles")
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
(options, args) = parser.parse_args()
logger=E2init(sys.argv,options.ppid)
# get basic image parameters
tmp=EMData(options.input,0,True)
boxsz=tmp["nx"]
if options.apix!=None : apix=options.apix
else : apix=tmp["apix_x"]
if options.pad<0:
options.pad=good_size(boxsz*1.5)
if options.outsize<0:
options.outsize=boxsz
options.tidrange=[int(i) for i in options.tidrange.split(',')]
if options.tidrange[0]>=0:
print("including tilt ids from {} to {}".format(options.tidrange[0], options.tidrange[1]))
data=initialize_data(options.input, options)
padvol=options.pad
from EMAN2PAR import EMTaskCustomer
if options.ref:
print("weighting by reference...")
ref=EMData(options.ref)
ref=ref.do_fft()
ref.process_inplace("xform.phaseorigin.tocenter")
ref.process_inplace("xform.fourierorigin.tocenter")
etc=EMTaskCustomer(options.parallel, module="e2spa_make3d.WeightptclTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
tasks=[data[i::num_cpus] for i in range(num_cpus)]
print("{} jobs".format(len(tasks)))
tids=[]
for t in tasks:
task = WeightptclTask(t, ref, options)
if options.debug:
task.execute(print)
return
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
E2progress(logger, .5*np.mean(st_vals)/100.)
if np.min(st_vals) == 100: break
time.sleep(5)
wts=[0]*len(data)
for i in tids:
wt=etc.get_results(i)[1]
for w in wt:
try:
wts[w[0]]=w[1]
except:
print(len(wts), w)
wts=np.array(wts)
del etc
r0=int(apix*boxsz/options.minres)
r1=int(apix*boxsz/options.maxres)
print(r0,r1)
scrs=np.mean(wts[:,r0:r1], axis=1)
if options.keep<1:
thr=np.sort(scrs)[int(len(scrs)*(1-options.keep))-1]
scrs[scrs<thr]=-1
for i,d in enumerate(data):
d["curve"]=wts[i]
d["weight"]=float(scrs[i])
etc=EMTaskCustomer(options.parallel, module="e2spa_make3d.Make3dTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
tasks=[data[i::num_cpus] for i in range(num_cpus)]
print("{} jobs".format(len(tasks)))
tids=[]
for t in tasks:
task = Make3dTask(t, options)
if options.debug:
task.execute(print)
return
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
E2progress(logger, .5+.5*np.mean(st_vals)/100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output=EMData(padvol, padvol, padvol)
normvol=EMData(padvol//2+1, padvol, padvol)
output.to_zero()
output.do_fft_inplace()
normvol.to_zero()
for i in tids:
threed, norm=etc.get_results(i)[1]
threed.process_inplace("math.multamplitude", {"amp":norm})
output.add(threed)
normvol.add(norm)
normvol.process_inplace("math.reciprocal")
output.process_inplace("math.multamplitude", {"amp":normvol})
output.do_ift_inplace()
output.depad()
output.process_inplace("xform.phaseorigin.tocenter")
del etc
if options.verbose>0 : print("Finished Reconstruction")
output["apix_x"]=output["apix_y"]=output["apix_z"]=apix
sz=options.outsize
output.clip_inplace(Region((padvol-sz)//2,(padvol-sz)//2,(padvol-sz)//2,sz,sz,sz))
if os.path.isfile(options.output):
os.remove(options.output)
output.write_image(options.output,0)
if options.verbose>0:
print("Output File: "+options.output)
E2end(logger)
print("Exiting")
def main():
usage = """e2tomopreproc.py <imgs> <options> .
This program takes a tiltseries ('.st' or '.ali' file from IMOD) and applies preprocessing operations to them, such as lowpass, highpass, masking, etc.
The options should be supplied in "--option=value" format, replacing "option" for a valid option name, and "value" for an acceptable value for that option.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--path",
type=str,
default='',
help="""Directory to store results in.
The default is a numbered series of directories containing the prefix 'tomopreproc';
for example, tomopreproc_02 will be the directory by default if 'tomopreproc_01'
already exists.""")
parser.add_pos_argument(name="stack_files",
default="",
help="Stacks or images to process.")
parser.add_argument(
"--input",
type=str,
default='',
help=
""""tiltseries to process. redundant with --tiltseries, or with providing images as arguments (separated by a space: e2tomopreproc.py stack1.hdf stack2.hdf), but --input takes precedence."""
)
parser.add_argument(
"--tiltseries",
type=str,
default='',
help=""""tiltseries to process. redundant with --input""")
parser.add_argument(
"--tltfile",
type=str,
default='',
help="""".tlt file containing the tilt angles for --tiltseries""")
parser.add_argument(
"--outmode",
type=str,
default='',
help=
"""All EMAN2 programs write images with 4-byte floating point values when possible by default. This allows specifying an alternate format when supported: float, int8, int16, int32, uint8, uint16, uint32. Values are rescaled to fill MIN-MAX range."""
)
parser.add_argument(
"--dontcleanup",
action='store_true',
default=False,
help="""If specified, intermediate files will be kept.""")
parser.add_argument(
"--clip",
type=str,
default='',
help=
"""Default=None. This resizes the 2-D images in the tilt series. If one number is provided, then x and y dimensions will be made the same. To specify both dimensions, supply two numbers, --clip=x,y. Clipping will be about the center of the image."""
)
#parser.add_argument("--apix",type=float,default=0.0,help="""True apix of images to be written on final stack.""")
parser.add_argument(
"--shrink",
type=float,
default=0.0,
help=
"""Default=0.0 (no shrinking). Can use decimal numbers, larger than 1.0. Optionally shrink the images by this factor. Uses processor math.fft.resample."""
)
parser.add_argument(
"--threshold",
type=str,
default='',
help="""Default=None. A threshold processor applied to each image.""")
parser.add_argument(
"--erasegold",
action='store_true',
default='',
help="""Default=False. Runs erase_gold.py on the stack.""")
parser.add_argument(
"--mask",
type=str,
default='',
help="""Default=None. Masking processor applied to each image.""")
parser.add_argument(
"--maskbyangle",
action='store_true',
default=False,
help=
"""Default=False. Requires --tltfile. This will mask out from tilted images the info that isn't present at the 0 tilt angle. It uses the tomo.tiltedgemask processor (type 'e2help.py processors' at the commandline to read a description of the processor and its parameters). Provide --maskbyanglefalloff and --maskbyanglesigma to modify the default parameters."""
)
parser.add_argument(
"--maskbyanglefalloff",
type=int,
default=4,
help=
"""Default=4. Number of pixels over which --maskbyangle will fall off to zero."""
)
parser.add_argument(
"--maskbyanglesigma",
type=float,
default=2.0,
help=
"""Default=2.0. Number of sigmas for the width of the gaussian fall off in --maskbyangle and --maskbyanglefalloff"""
)
parser.add_argument(
"--normproc",
type=str,
default='',
help=
"""Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to each image."""
)
parser.add_argument(
"--normalizeimod",
action='store_true',
default=False,
help=
"""Default=False. This will apply 'newstack -float 2' to the input stack. Requires IMOD."""
)
parser.add_argument(
"--preprocess",
type=str,
default='',
help=
"""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each image."""
)
parser.add_argument(
"--lowpassfrac",
type=float,
default=0.0,
help=
"""Default=0.0 (not used). Fraction of Nyquist to lowpass at. The processor used is filter.lowpass.tanh"""
)
parser.add_argument(
"--highpasspix",
type=int,
default=0,
help=
"""Default=0 (not used). Number of Fourier pixels to apply highpass filter at. The processor used is filter.highpass.gauss."""
)
parser.add_argument(
"--parallel",
type=str,
default="thread:1",
help=
"""default=thread:1. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel"""
)
parser.add_argument(
"--prenadminite",
type=int,
default=0,
help=
"""Default=0. Requires IMOD to be installed. Used to apply prenad filtering to a tiltseries. This is the --minite parameter in IMOD's preNAD program (minimum number of iterations)."""
)
parser.add_argument(
"--prenadmaxite",
type=int,
default=0,
help=
"""Default=0. Requires IMOD to be installed. Used to apply prenad filtering to a tiltseries. This is the --maxite parameter in IMOD's preNAD program (maximum number of iterations)."""
)
parser.add_argument(
"--prenadsigma",
type=int,
default=0,
help=
"""Default=0. Requires IMOD to be installed. Used to apply prenad filtering to a tiltseries. This is the --sigma parameter in IMOD's preNAD program (initial sigma for 'smoothing structure tensor')."""
)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness."
)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
logger = E2init(sys.argv, options.ppid)
print "\n(e2tomopreproc)(main) started log"
from e2spt_classaverage import sptmakepath
options = sptmakepath(options, 'tomopreproc')
#print "args are",args
infiles = []
if not options.input:
#try:
# infiles.append( sys.argv[1] )
#except:
if options.tiltseries:
infiles.append(options.tiltseries)
else:
if args:
print "copying args to infiles"
infiles = list(args)
print "infiles are", infiles
else:
print "\n(e2tomopreproc)(main) ERROR: must provide input files as arguments or via the --input or --tiltseries parameters."
elif options.input:
infiles.append(options.input)
if infiles:
print "\n(e2tomopreproc)(main) identified --input", options.input
#print " .ali in options.input[:-4]", '.ali' in options.input[-4:]
#print "options.input[-4] is", options.input[-4:]
for infile in infiles:
if '.ali' in infile[-4:] or '.st' in infile[
-3:] or '.mrc' in infile[-4:] or '.mrcs' in infile[
-5:] or '.hdf' in infile[-4:]:
pass
else:
print "\n(e2tomopreproc)(main) ERROR: invalid image extension %s for image %s. Extension must be .st, .ali, .hdf, .mrc or .mrcs" % (
options.input.split('.')[-1], infile)
sys.exit(1)
else:
print "\n(e2tomopreproc)(main) ERROR: no images found/provided"
sys.exit(1)
originalextension = infiles[0].split('.')[-1]
angles = {}
if options.maskbyangle or (options.prenadminite and options.prenadmaxite
and options.prenadsigma):
if not options.tltfile:
print "\n(e2tomopreproc)(main) ERROR: --maskbyangle and --prenad parameters require --tltfile"
sys.exit(1)
else:
f = open(options.tltfile, 'r')
lines = f.readlines()
print "\nnumber of lines read from --tltfile", len(lines)
f.close()
#print "lines in tlt file are", lines
k = 0
for line in lines:
line = line.replace('\t', '').replace('\n', '')
if line:
angle = float(line)
angles.update({k: angle})
if options.verbose:
print "appending angle", angle
k += 1
if len(angles) < 2:
print "\nERROR: something went terribly wrong with parsing the --tltlfile. This program does not work on single images"
sys.exit()
if len(angles) < 2:
print "\nERROR: (second angle check) something went terribly wrong with parsing the --tltlfile. This program does not work on single images"
sys.exit()
print "\n(e2spt_preproc)(main) - INITIALIZING PARALLELISM!\n"
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel)
pclist = [options.input]
etc.precache(pclist)
print "\n(e2spt_preproc)(main) - precaching --input"
tasks = []
results = []
mrcstacks = []
print "there are these many infiles to loop over", len(infiles)
if options.lowpassfrac:
hdr = EMData(infiles[0], 0, True)
apix = hdr['apix_x']
print "\n(e2spt_preproc)(main) apix is", apix
nyquist = 2.0 * apix
print "\n(e2spt_preproc)(main) therefore nyquist resolution is", nyquist
print
lowpassres = nyquist / options.lowpassfrac
options.lowpassfrac = 1.0 / (lowpassres)
if float(options.shrink) > 1.0:
options.lowpassfrac /= float(options.shrink)
print "there's shrinking", options.shrink
lowpassres = nyquist / options.lowpassfrac
print "\n(e2spt_preproc)(main) and final lowpass frequency is", options.lowpassfrac
print "corresponding to lowpassres of", lowpassres
for infile in infiles:
mrcstack = options.path + '/' + infile
print "infile is", infile
print "infile[-5:] is ", infile[-5:]
if '.hdf' in infile[-5:]:
print "replacing .hdf extension"
mrcstack = options.path + '/' + infile.replace('.hdf', '.mrc')
if '.mrcs' in infile[-5:]:
print "replacing .mrcs extension"
mrcstack = options.path + '/' + infile.replace('.mrcs', '.mrc')
if '.st' in infile[-5:]:
print "replacing .st extension"
mrcstack = options.path + '/' + infile.replace('.st', '.mrc')
if '.ali' in infile[-5:]:
print "replacing .ali extension"
mrcstack = options.path + '/' + infile.replace('.ali', '.mrc')
if '.tif' in infile[-5:]:
print "replacing .ali extension"
mrcstack = options.path + '/' + infile.replace('.tif', '.mrc')
#go = 0
#if go:
print "mrcstack is", mrcstack
#outname = outname.replace('.mrc','.mrcs')
mrcstacks.append(mrcstack)
go = 0
if options.maskbyangle:
outname = mrcstack.replace('.mrc', '_UNSTACKED.mrc')
print "therefore, outname is", outname
cmd = 'e2proc2d.py ' + infile + ' ' + outname + ' --unstacking --threed2twod'
#from shutil import copyfile
#copyfile(options.input, outname)
#print "copied input to", outname
if options.outmode:
cmd += ' --outmode=' + options.outmode
if options.verbose:
cmd += ' --verbose=' + str(options.verbose)
print "\ncommand to unstack original input tiltseries is", cmd
print "\n(e2tomopreproc)(main) unstacking command is", cmd
p = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
#p = subprocess.Popen( cmd , shell=True, stdout=subprocess.PIPE)
text = p.communicate()
#p.stdout.close()
p.wait()
if p.returncode == 0:
go = 1
else:
go = 1
if go:
imgs = []
if options.maskbyangle:
c = os.getcwd() + '/' + options.path
findir = os.listdir(os.getcwd() + '/' + options.path)
print "\n(e2tomopreproc)(main) directory to look for images is", c
for f in findir:
#if '.mrcs' in f:
if "_UNSTACKED" in f:
imgs.append(options.path + '/' + f)
kk = 0
imgs.sort()
print "\n(e2spt_preproc)(main) found these many images", len(
imgs)
for img in imgs:
#task=None
#if options.maskbyangle:
outimage = img.replace('.mrc', '_preproc.mrc')
task = TomoPreproc2DTask(img, options, angles[kk],
outimage)
tasks.append(task)
kk += 1
else:
outimage = options.path + '/' + infile.replace(
'.mrc', '_preproc.mrcs')
task = TomoPreproc2DTask(infile, options, 0, outimage)
tasks.append(task)
#else:
# newmrcs = mrcstack.replace('.mrc','.mrcs')
# print "copying file %s to %s" %(infile,newmrcs)
# copyfile( infile, newmrcs )
# imgs.append( newmrcs )
#print "and the final lowpass frequency will be", options.lowpassfrac
tids = etc.send_tasks(tasks)
if options.verbose:
print "\n(e2spt_preproc)(main) preprocessing %d tasks queued" % (
len(tids))
results = get_results(etc, tids, options)
print "\n(e2tomopreproc)(main) these many images have been processsed", len(
results)
imgspreproc = []
findir = os.listdir(os.getcwd() + '/' + options.path)
#for mrcstack in mrcstacks:
for f in findir:
if "_preproc.mrc" in f:
print "found preprocessed image", f
imgspreproc.append(options.path + '/' + f)
else:
print "this file is NOT a preprocessed image", f
imgspreproc.sort()
print "\n(e2tomopreproc)(main) these many preprocessed images loaded", len(
imgspreproc)
finalfiles = []
if options.maskbyangle:
outfile = mrcstack.replace('.mrc', '.mrcs')
print "for RESTACKING"
print "\n\n\noutfile is", outfile
for f in imgspreproc:
print "appending image %s to outfile %s" % (f, outfile)
cmd = 'e2proc2d.py ' + f + ' ' + outfile
if options.outmode:
cmd += ' --outmode=' + options.outmode
if options.verbose:
cmd += ' --verbose ' + str(options.verbose)
print "\ncmd is with .mrcs outputformat is", cmd
print "becauase outfile is", outfile
p = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
text = p.communicate()
p.stdout.close()
finaloutput = outfile.replace('.mrcs', '.' + originalextension)
os.rename(outfile, finaloutput)
finalfiles.append(finaloutput)
else:
finalfiles = list(imgspreproc)
for finalf in finalfiles:
if not options.tltfile:
break
if options.normalizeimod:
try:
cmd = 'newstack ' + finalf + ' ' + finalf + ' --float 2'
print "normalizeimod cmd is", cmd
p = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
text = p.communicate()
p.wait()
except:
print "\nERROR: --normalizeimod skipped. Doesn't seem like IMOD is installed on this machine"
if not options.dontcleanup and options.maskbyangle:
purge(options.path, '_preproc.mrc')
purge(options.path, '_UNSTACKED')
purge(options.path, '~')
if options.tltfile:
if options.prenadminite or options.prenadmaxite or options.prenadsigma:
if options.prenadminite and options.prenadmaxite and options.prenadsigma:
cmd = 'preNAD -input ' + finalf + ' -output ' + finalf.replace(
'.' + originalextension,
'_prenad.' + originalextension) + ' -minite ' + str(
options.prenadminite) + ' -maxite ' + str(
options.prenadmaxite) + ' -sigma ' + str(
options.prenadsigma
) + ' -angles ' + options.tltfile
if options.verbose:
print "\n(e2tomopreproc)(main) prenad cmd to run is", cmd
try:
p = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
text = p.communicate()
p.wait()
except:
print "\nERROR: check that a version of IMOD containing the preNAD program is correctly installed on this machine"
else:
if options.prenadminite:
if not options.prenadmaxite:
print "\nERROR: --prenadmaxite required with --prenadminite"
if not options.prenadsigma:
print "\nERROR: --prenadsigma required with --prenadminite"
if options.prenadmaxite:
if not options.prenadminite:
print "\nERROR: --prenadminite required with --prenadmaxite"
if not options.prenadsigma:
print "\nERROR: --prenadsigma required with --prenadmaxite"
if options.prenadsigma:
if not options.prenadminite:
print "\nERROR: --prenadminite required with --prenadsigma"
if not options.prenadmaxite:
print "\nERROR: --prenadmaxite required with --prenadsigma"
E2end(logger)
return ()
def main():
"""Program to validate a reconstruction by the Richard Henderson tilt validation method. A volume to validate, a small stack (~100 imgs) of untilted and ~10-15 degree
tilted particles must be presented. The untilted and tilted particle stack must have a one-to-one relationship. In the contour plot, the Tiltaxis is along positive 'Y'
The tiltaxis angle can be determined from e2RCTboxer.py uisng PairPicker mode. For example, if the tiltaxis is 45 degrees and the tilt angle is -15 degrees, there should
be a peak in the -X, -Y quadrant at 225 degrees at a magnitude of 15.
For more details see:
Optiomal Determination of Particle Orientation, Absolute Hand, and COntrast Loss in Single-particle Electron Cryomicroscopy. Rosenthal, P.B., and Henderson, R. JMB, 333 (2003) pg 721-745
"""
progname = os.path.basename(sys.argv[0])
usage = """prog [options]
Tiltvalidation using Richard Henderson's technique. To use a stack of untilted and tiltimages whose set relationship is one-to-one is required along with a
volume to validate. This can be generated using e2RCTboxer.py. After running this program two bits of data are products. A contour plot similar to Figure 5 in the Henderson paper(see below), and a list of
titlangles and tiltaxes between particle paris, which can be used to makes plot similar to Figure 6 in Hendersons paper. The contour plot is stored as contour.hdf and the tiltpairs data is
stored as bdb:perparticletilts.
For more information see:
Optimal determination of particle orientation, absolute hand, and contrast loss in
single-particle electron cryomicroscopy.
Rosenthal PB, Henderson R.
J Mol Biol. 2003 Oct 31;333(4):721-45
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
# options associated with e2tiltvalidate.py
parser.add_header(
name="tvheader",
help='Options below this label are specific to e2tiltvalidate',
title="### e2tiltvalidate options ###",
row=3,
col=0,
rowspan=1,
colspan=2,
mode="analysis,gui")
# "analysys" mode options
parser.add_argument(
"--untiltdata",
type=str,
help="Stack of untilted images",
default=None,
guitype='filebox',
browser='EMSetsTable(withmodal=True,multiselect=False)',
row=0,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_argument(
"--tiltdata",
type=str,
help="Stack of tilted images",
default=None,
guitype='filebox',
browser='EMSetsTable(withmodal=True,multiselect=False)',
row=1,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_argument(
"--volume",
type=str,
help="3D volume to validate",
default=None,
guitype='filebox',
browser='EMModelsTable(withmodal=True,multiselect=False)',
row=2,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_argument(
"--maxtiltangle",
type=float,
help="Maximum tiltangle permitted when finding tilt distances",
default=180.0,
guitype='floatbox',
row=4,
col=0,
rowspan=1,
colspan=1,
mode="analysis")
parser.add_argument("--quaternion",
action="store_true",
help="Use Quaterions for tilt distance computation",
default=False,
guitype='boolbox',
row=4,
col=1,
rowspan=1,
colspan=1,
mode='analysis')
parser.add_argument("--sym",
type=str,
help="The recon symmetry",
default="c1",
guitype='symbox',
row=5,
col=0,
rowspan=1,
colspan=1,
mode="analysis")
parser.add_argument("--docontourplot",
action="store_true",
help="Compute a contour plot",
default=False,
guitype='boolbox',
row=6,
col=0,
rowspan=1,
colspan=1,
expert=True,
mode="analysis")
parser.add_argument("--tiltrange",
type=int,
help="The angular tiltrange to search",
default=15,
guitype='intbox',
row=6,
col=1,
rowspan=1,
colspan=1,
expert=True,
mode="analysis")
parser.add_argument(
"--align",
type=str,
help="The name of a aligner to be used in comparing the aligned images",
default="translational",
guitype='comboparambox',
choicelist='re_filter_list(dump_aligners_list(),\'refine|3d\', 1)',
expert=True,
row=7,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_argument(
"--cmp",
type=str,
help="The name of a 'cmp' to be used in comparing the aligned images",
default="ccc",
guitype='comboparambox',
choicelist='re_filter_list(dump_cmps_list(),\'tomo\', True)',
expert=True,
row=8,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_header(
name="projheader",
help='Options below this label are specific to e2project',
title="### e2project options ###",
row=10,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_argument("--delta",
type=float,
help="The angular step size for alingment",
default=5.0,
guitype='floatbox',
row=11,
col=0,
rowspan=1,
colspan=1,
mode="analysis")
# options associated with e2simmx.py
parser.add_header(name="simmxheader",
help='Options below this label are specific to e2simmx',
title="### e2simmx options ###",
row=12,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_argument(
"--shrink",
dest="shrink",
type=int,
default=0,
help=
"Optionally shrink the input particles by an integer amount prior to computing similarity scores. For speed purposes. Defulat = 0, no shrinking",
guitype='shrinkbox',
row=13,
col=0,
rowspan=1,
colspan=1,
mode="analysis")
parser.add_argument(
"--simcmp",
type=str,
help=
"The name of a 'cmp' to be used in comparing the aligned images (default=ccc)",
default="ccc",
guitype='comboparambox',
choicelist='re_filter_list(dump_cmps_list(),\'tomo\', True)',
row=14,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
# options associated with e2projector3d.py
parser.add_argument(
"--simalign",
type=str,
help=
"The name of an 'aligner' to use prior to comparing the images (default=rotate_translate)",
default="rotate_translate",
guitype='comboparambox',
choicelist='re_filter_list(dump_aligners_list(),\'refine|3d\', 1)',
row=15,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_argument(
"--simaligncmp",
type=str,
help=
"Name of the aligner along with its construction arguments (default=ccc)",
default="ccc",
guitype='comboparambox',
choicelist='re_filter_list(dump_cmps_list(),\'tomo\', True)',
row=16,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_argument(
"--simralign",
type=str,
help=
"The name and parameters of the second stage aligner which refines the results of the first alignment",
default=None,
guitype='comboparambox',
choicelist='re_filter_list(dump_aligners_list(),\'refine\', 0)',
row=17,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_argument(
"--simraligncmp",
type=str,
help=
"The name and parameters of the comparitor used by the second stage aligner. (default=dot).",
default="dot",
guitype='comboparambox',
choicelist='re_filter_list(dump_cmps_list(),\'tomo\', True)',
row=18,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_argument("--parallel",
type=str,
help="Parallelism string",
default=None,
guitype='strbox',
row=9,
col=0,
rowspan=1,
colspan=2,
mode="analysis")
parser.add_argument(
"--verbose",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness",
guitype='intbox',
row=19,
col=0,
rowspan=1,
colspan=1,
mode="analysis")
# "gui" mode options
parser.add_argument("--path",
type=str,
help="The folder the results are placed",
default="",
guitype='dirbox',
dirbasename='TiltValidate',
row=0,
col=0,
rowspan=1,
colspan=2,
mode="gui")
parser.add_argument(
"--radcut",
type=float,
default=-1,
help=
"For use in the GUI, truncate the polar plot after R. -1 = no truncation",
guitype='floatbox',
row=4,
col=0,
rowspan=1,
colspan=1,
mode="gui")
parser.add_argument(
"--gui",
action="store_true",
help="Start the GUI for viewing the tiltvalidate plots",
default=False,
guitype='boolbox',
row=4,
col=1,
rowspan=1,
colspan=1,
mode="gui[True]")
parser.add_argument(
"--planethres",
type=float,
help=
"Maximum out of plane threshold for the tiltaxis. 0 = perfectly in plane, 1 = normal to plane",
default=360.0,
guitype='floatbox',
row=5,
col=0,
rowspan=1,
mode="gui")
parser.add_argument(
"--datalabelscolor",
type=str,
help=
"Set the color of the data labels. Any vaild matplotlib color is ok",
default='#00ff00',
guitype='strbox',
row=6,
col=0,
rowspan=1,
colspan=1,
mode="gui")
parser.add_argument("--datalabels",
action="store_true",
help="Add data labels to the plot",
default=False,
guitype='boolbox',
row=6,
col=1,
rowspan=1,
mode="gui")
parser.add_argument("--colorzaxis",
action="store_true",
help="Color scatter dots by Z axis",
default=False,
guitype='boolbox',
row=7,
col=0,
rowspan=1,
mode="gui")
#other options
parser.add_argument(
"--eulerfile",
type=str,
help=
"Euler angles file, to create tiltdistance from pre-aligned particles. Format is: imgnum, name, az, alt, phi",
default=None)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
# Run the GUI if in GUI mode
#print options
if options.gui:
display_validation_plots(options.path,
options.radcut,
options.planethres,
plotdatalabels=options.datalabels,
color=options.datalabelscolor,
plotzaxiscolor=options.colorzaxis)
exit(0)
if not (options.volume or options.eulerfile):
print("Error a volume to validate must be presented")
exit(1)
if not (options.tiltdata or options.eulerfile):
print("Error a stack of tilted images must be presented")
exit(1)
if not (options.untiltdata or options.eulerfile):
print("Error a stack of untiled images must be presented")
exit(1)
logid = E2init(sys.argv, options.ppid)
options.cmp = parsemodopt(options.cmp)
options.align = parsemodopt(options.align)
# Make a new dir for each run
if not options.path:
#options.path=numbered_path("TiltValidate",True)
# Create the run directory structure if it does not exist
i = 1
found = 1
while found == 1:
if i < 10:
run_dir = '0' + str(i)
else:
run_dir = str(i)
found = os.path.exists("TiltValidate_" + run_dir)
i = i + 1
os.mkdir("TiltValidate_" + run_dir)
options.path = "TiltValidate_" + run_dir
#Make tilt distance generator
tiltgenerator = ComputeTilts(options)
# Compute tilt distances from file if desired.
if options.eulerfile:
# Format is:
# untilt_imgnum name az alt phi
# tilt_imgnum name az alt phi
eulerfile = open(options.eulerfile, "r")
eulers = eulerfile.readlines()
eulerfile.close()
untilteulerlist = []
tilteulerlist = []
for i, euler in enumerate(eulers):
fields = euler.split()
if i % 2:
tilteulerlist.append({
'alt': float(fields[2]),
'az': float(fields[3]),
'phi': float(fields[4])
})
else:
untilteulerlist.append({
'alt': float(fields[2]),
'az': float(fields[3]),
'phi': float(fields[4])
})
tiltgenerator.findtilts_fromeulers(untilteulerlist, tilteulerlist)
exit(0)
# Initialize parallelism if being used
if options.parallel:
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
"e2tiltvalidate.CompareToTiltTask")
else:
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer("thread:1", "e2tiltvalidate.CompareToTiltTask")
#etc.precache(pclist)
# Otherwise compute tilt distances from data
#Read in the images
tiltimgs = EMData.read_images(options.tiltdata)
untiltimgs = EMData.read_images(options.untiltdata)
if len(tiltimgs) != len(untiltimgs):
print(
"The untilted image stack is not the same length as the tilted stack!!!"
)
exit(1)
# write projection command to DB. If we rerun this program no need to reproject if it was done using same pars before
cdb = js_open_dict('info/cmdcache.json')
projparmas = "%s%f%s" % (options.volume, options.delta, options.sym)
# try:
# if (cdb.has_key('projparmas') and cdb['projparmas'] == projparmas): raise IOError("Projection file does not exist")
# run("e2proc2d.py bdb:%s#projections_00 bdb:%s#projections_00"%(cdb['previouspath'], options.path))
# except:
# Do projections
e2projectcmd = "e2project3d.py %s --orientgen=eman:delta=%f:inc_mirror=1:perturb=0 --outfile=%s/projections_00.hdf --projector=standard --sym=%s" % (
options.volume, options.delta, options.path, options.sym
) # Seems to work better when I check all possibilites
if options.parallel: e2projectcmd += " --parallel=%s" % options.parallel
run(e2projectcmd)
cdb['projparmas'] = projparmas
cdb['previouspath'] = options.path
cdb.close()
# Make simmx
e2simmxcmd = "e2simmx.py %s/projections_00.hdf %s %s/simmx.hdf -f --saveali --cmp=%s --align=%s --aligncmp=%s --verbose=%d" % (
options.path, options.untiltdata, options.path, options.simcmp,
options.simalign, options.simaligncmp, options.verbose)
if options.simralign:
e2simmxcmd += " --ralign=%s --raligncmp=%s" % (options.simralign,
options.simraligncmp)
if options.parallel: e2simmxcmd += " --parallel=%s" % options.parallel
if options.shrink: e2simmxcmd += " --shrink=%d" % options.shrink
run(e2simmxcmd)
e2simmxcmd = "e2simmx.py %s/projections_00.hdf %s %s/simmx_tilt.hdf -f --saveali --cmp=%s --align=%s --aligncmp=%s --verbose=%d" % (
options.path, options.tiltdata, options.path, options.simcmp,
options.simalign, options.simaligncmp, options.verbose)
if options.simralign:
e2simmxcmd += " --ralign=%s --raligncmp=%s" % (options.simralign,
options.simraligncmp)
if options.parallel: e2simmxcmd += " --parallel=%s" % options.parallel
if options.shrink: e2simmxcmd += " --shrink=%d" % options.shrink
run(e2simmxcmd)
# Read in the data
simmx = EMData.read_images("%s/simmx.hdf" % options.path)
simmx_tilt = EMData.read_images("%s/simmx_tilt.hdf" % options.path)
projections = EMData.read_images("%s/projections_00.hdf" % options.path)
volume = EMData()
volume.read_image(
options.volume) # I don't know why I cant EMData.read_image.......
# Generate tilts from data
tiltgenerator.findtilts_fromdata(simmx, simmx_tilt, projections, volume,
untiltimgs, tiltimgs)
if options.docontourplot:
# Make contour plot to validate each particle
tasks = []
distplot = EMData(options.tiltrange * 2 + 1, options.tiltrange * 2 + 1)
distplot.to_zero()
for imgnum in range(simmx[0].get_ysize()):
bestscore = float('inf')
bestrefnum = 0
for refnum in range(simmx[0].get_xsize()):
if simmx[0].get_value_at(refnum, imgnum) < bestscore:
bestscore = simmx[0].get_value_at(refnum, imgnum)
bestrefnum = refnum
# Get the euler angle for this particle and call compare to tilt"bdb:%s#
euler_xform = projections[bestrefnum].get_attr('xform.projection')
tasks.append(
CompareToTiltTask(volume, tiltimgs[imgnum], imgnum,
euler_xform,
simmx[3].get_value_at(bestrefnum, imgnum),
distplot, options.tiltrange, 1, options))
# Farm out the work and hang till finished!
tids = etc.send_tasks(tasks)
while 1:
time.sleep(5)
proglist = etc.check_task(tids)
tids = [j for i, j in enumerate(tids) if proglist[i] != 100
] # remove any completed tasks from the list we ask about
if len(tids) == 0: break
# Make scoremx avg
scoremxs = EMData.read_images("%s/scorematrix.hdf" % options.path)
avgmxavger = Averagers.get('mean')
for mx in scoremxs:
avgmxavger.add_image(mx)
avgmx = avgmxavger.finish()
avgmx.write_image("%s/contour.hdf" % options.path)
distplot.write_image("%s/distplot.hdf" % options.path)
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog <output> [options]
The goal of this program is to reduce the heterogeneity of a reconstruction by splitting a single map
into two maps, each more homogeneous. You must run e2refine_easy to completion before using this program.
It will take the class-averaging results from the final iteration, and split the particles from each
class-average into 2 groups, producing 2 class-averages for each. The program then attempts to construct
a maximally self-consistent grouping of these pairs of class averages into 2 3-D maps.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--path",
default=None,
type=str,
help=
"The name of an existing refine_xx folder, where e2refine_easy ran to completion",
guitype='filebox',
filecheck=False,
browser="EMBrowserWidget(withmodal=True,multiselect=False)",
row=3,
col=0,
rowspan=1,
colspan=3)
parser.add_argument(
"--usebasis",
default=0,
type=int,
help=
"Select which Eigenimage to use for separation. With novarimax, n=0 is highest energy.",
guitype='intbox',
row=5,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--nbasis",
default=-1,
type=int,
help=
"Number of basis vectors to compute. Must be at least usebasis+1. Default 6 or usebasis+1.",
guitype='intbox',
row=4,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--novarimax",
action="store_true",
default=False,
help="Disable varimax rotation among computed basis vectors.",
guitype='boolbox',
row=7,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--mask",
default=None,
help="Optional 3D mask to focus the classification",
guitype='filebox',
browser='EMSetsTable(withmodal=True,multiselect=False)',
filecheck=False,
row=6,
col=0,
rowspan=1,
colspan=3,
mode="refinement")
parser.add_argument("--parallel",
default="thread:2",
help="Standard parallelism option. Default=thread:2",
guitype='strbox',
row=8,
col=0,
rowspan=1,
colspan=2)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
if options.nbasis <= 1:
options.nbasis = 6
if options.nbasis <= options.usebasis + 1:
options.nbasis = options.usebasis + 1
print "--nbasis adjusted to ", options.nbasis
if options.path == None:
paths = [i for i in os.listdir(".") if "refine_" in i and len(i) == 9]
paths.sort()
options.path = paths[-1]
pathnum = options.path[-2:]
# check the specified path for the files we need
try:
olddb = js_open_dict(options.path + "/0_refine_parms.json")
last_map = olddb["last_map"]
targetres = olddb["targetres"]
last_iter = int(last_map.split("_")[-1][:2])
try:
ptcls = olddb["inputavg"]
if ptcls == None: raise Exception
except:
ptcls = olddb["input"]
sym = olddb["sym"]
if options.verbose:
print "Found iteration {} in {}, using {}".format(
last_iter, options.path, " & ".join(ptcls))
except:
traceback.print_exc()
print "Error: Cannot find necessary files in ", options.path
sys.exit(1)
logger = E2init(sys.argv, options.ppid)
# classmx is a list with 2 elements. Each element is a list of EMData from the corresponding cls_result file
classmx = []
classmx.append(
EMData.read_images("{}/cls_result_{:02d}_even.hdf".format(
options.path, last_iter)))
classmx.append(
EMData.read_images("{}/cls_result_{:02d}_odd.hdf".format(
options.path, last_iter)))
ncls = max(
int(classmx[0][0]["maximum"]) + 1,
int(classmx[1][0]["maximum"]) + 1)
# Rearrange the info in classmx
classlists = [[] for i in xrange(ncls)] # empty list for each class
# This will produce a list of particles with Transforms for each class
for eo in (0, 1):
for y in xrange(classmx[eo][0]["ny"]):
ptcl = [
eo, y,
Transform({
"type": "2d",
"tx": classmx[eo][2][0, y],
"ty": classmx[eo][3][0, y],
"alpha": classmx[eo][4][0, y],
"mirror": int(classmx[eo][5][0, y])
})
]
#print ptcl,
#print int(classmx[eo][0][0,y])
classlists[int(classmx[eo][0][0, y])].append(ptcl)
#if len(classlists[0])>100 :
#print "Warning: this program is normally intended for use with downsampled data and fairly coarse angular sampling. If you try to use it with a large number of class-averages you may have a variety of problems, and should insure that your machine has sufficient RAM."
# Initialize parallelism
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel)
# Empty image to pad classes file
zero = EMData(str(ptcls[0]), 0)
zero.to_zero()
zero["ptcl_repr"] = 0
# Euler angles for averages
projin = "{}/projections_{:02d}_even.hdf".format(options.path, last_iter)
eulers = [
EMData(projin, i, True)["xform.projection"] for i in xrange(ncls)
]
# Prepare mask if specified
if options.mask != None:
mask = EMData(options.mask)
else:
mask = None
# prepare tasks
tasks = []
gc = 0
ns = [classmx[eo][0]["ny"] for eo in (0, 1)]
for c, cl in enumerate(classlists):
if len(
cl
) < 20: # we require at least 20 particles in a class to make the attempt
# zero.write_image(classout[0],c)
# zero.write_image(classout[1],c)
continue
if mask != None:
maskp = mask.project("standard", eulers[c])
else:
maskp = None
tasks.append(
ClassSplitTask(ptcls, ns, cl, c, eulers[c], maskp,
options.usebasis, options.nbasis, options.novarimax,
options.verbose - 1))
gc += 1
# for t in tasks: t.execute()
# execute task list
taskids = etc.send_tasks(tasks)
alltaskids = taskids[:]
classes = []
while len(taskids) > 0:
curstat = etc.check_task(taskids)
for i, j in enumerate(curstat):
if j == 100:
rslt = etc.get_results(taskids[i])
rsltd = rslt[1]
cls = rslt[0].options["classnum"]
if rsltd.has_key("failed"):
print "Bad average in ", cls
else:
#rsltd["avg1"].write_image(classout[0],cls)
#rsltd["avg2"].write_image(classout[1],cls)
ncls = rsltd["avg1"]["ptcl_repr"] + rsltd["avg2"][
"ptcl_repr"]
# note that the 2 results we get back are in arbitrary order!
# the next section of code with 3D reconstruction is designed to sort out
# which average should be paired with which
classes.append([
ncls, rsltd["avg1"]["xform.projection"], rsltd["avg1"],
rsltd["avg2"], rsltd["basis"], cls
]) # list of (ptcl_repr,xform,avg1,avg2)
taskids = [j for i, j in enumerate(taskids) if curstat[i] != 100]
if options.verbose and 100 in curstat:
print "%d/%d tasks remain" % (len(taskids), len(alltaskids))
if 100 in curstat:
E2progress(logger, 1.0 - (float(len(taskids)) / len(alltaskids)))
if options.verbose:
print "Completed all tasks\nGrouping consistent averages"
classes.sort(
reverse=True) # we want to start with the largest number of particles
apix = classes[0][2]["apix_x"]
boxsize = classes[0][2]["ny"]
pad = good_size(boxsize * 1.5)
if options.verbose:
print "Boxsize -> {}, padding to {}".format(boxsize, pad)
# a pair of reconstructors. we will then simultaneously reconstruct in the pair, and use each to decide on the best target for each particle
recon = [
Reconstructors.get("fourier", {
"size": [pad, pad, pad],
"sym": sym,
"mode": "gauss_5"
}) for i in (0, 1)
]
for r in recon:
r.setup()
# We insert the first class-average (with the most particles) randomly into reconstructor 1 or 2
p2 = classes[0][2].get_clip(
Region(-(pad - boxsize) / 2, -(pad - boxsize) / 2, pad, pad))
p3 = recon[0].preprocess_slice(p2, classes[0][1])
recon[0].insert_slice(p3, classes[0][1],
classes[0][2].get_attr_default("ptcl_repr", 1.0))
p2 = classes[0][3].get_clip(
Region(-(pad - boxsize) / 2, -(pad - boxsize) / 2, pad, pad))
p3 = recon[1].preprocess_slice(p2, classes[0][1])
recon[1].insert_slice(p3, classes[0][1],
classes[0][3].get_attr_default("ptcl_repr", 1.0))
classes[0].append(0)
if options.verbose: print "Reconstruction: pass 1"
for i, c in enumerate(classes[1:]):
proj = EMData(projin,
c[5]) # the projection corresponding to this average
# while this does cost us a final interpolation, high resolution isn't the primary aim anyway, and getting the alignment consistent is important
# also gives us a chance to normalize
c[2]["xform.align2d"] = Transform()
ali2 = c[2].align("refine", proj)
ali2.process_inplace("normalize.toimage", {
"to": proj,
"ignore_zero": 1
})
c[3]["xform.align2d"] = Transform()
ali3 = c[3].align("refine", proj)
ali3.process_inplace("normalize.toimage", {
"to": proj,
"ignore_zero": 1
})
# print "ROT:\t",ali2["xform.align2d"].get_params("2d"),"\t",ali3["xform.align2d"].get_params("2d")
# note that ali2 and c[2] are the same except for a final alignment
a2 = ali2.get_clip(
Region(-(pad - boxsize) / 2, -(pad - boxsize) / 2, pad,
pad)) # first class-average
a3 = recon[0].preprocess_slice(a2, classes[0][1])
a3n = c[2].get_attr_default("ptcl_repr", 1.0)
# similarly ali3 and c[3] are the same
b2 = ali3.get_clip(
Region(-(pad - boxsize) / 2, -(pad - boxsize) / 2, pad, pad))
b3 = recon[1].preprocess_slice(
b2, classes[0][1]
) # I don't believe it matters if we use recon[0] or 1 here, but haven't checked
b3n = c[3].get_attr_default("ptcl_repr", 1.0)
recon[0].determine_slice_agreement(a3, c[1], a3n, False)
# print a3.get_attr_dict()
q0a = a3[
"reconstruct_absqual_lowres"] # quality for average a in reconstruction0
# n0a=a3["reconstruct_norm"] # normalization for same
recon[1].determine_slice_agreement(a3, c[1], a3n, False)
q1a = a3[
"reconstruct_absqual_lowres"] # quality for average a in reconstruction0
# n1a=a3["reconstruct_norm"] # normalization for same
recon[0].determine_slice_agreement(b3, c[1], b3n, False)
q0b = b3[
"reconstruct_absqual_lowres"] # quality for average a in reconstruction0
# n0b=b3["reconstruct_norm"] # normalization for same
recon[1].determine_slice_agreement(b3, c[1], b3n, False)
q1b = b3[
"reconstruct_absqual_lowres"] # quality for average a in reconstruction0
# n1b=b3["reconstruct_norm"] # normalization for same
if options.verbose > 1:
print i, q0a, q1a, q0b, q1b, q0a + q1b, q1a + q0b
if options.verbose > 2: print "\t\t", n0a, n1a, n0b, n1b
if q0a + q1b > q1a + q0b: # if true, a -> recon0 and b -> recon1
c.append(
0
) # we put a 0 at the end of the classes element if we use a->0,b->1 ordering, 1 if swapped
# a3.mult(n0a)
recon[0].insert_slice(a3, c[1], a3n)
# b3.mult(n1b)
recon[1].insert_slice(b3, c[1], b3n)
else:
c.append(1)
# a3.mult(n1a)
recon[1].insert_slice(a3, c[1], a3n)
# b3.mult(n0b)
recon[0].insert_slice(b3, c[1], b3n)
if options.verbose: print "Reconstruction: pass 2"
# another pass with the filled reconstruction to make sure our initial assignments were ok
# for i,c in enumerate(classes[1:]):
# a2=c[2].get_clip(Region(-(pad-boxsize)/2,-(pad-boxsize)/2,pad,pad)) # first class-average
# a3=recon[0].preprocess_slice(a2,classes[0][1])
# a3n=c[2].get_attr_default("ptcl_repr",1.0)
#
# b2=c[3].get_clip(Region(-(pad-boxsize)/2,-(pad-boxsize)/2,pad,pad))
# b3=recon[1].preprocess_slice(b2,classes[0][1]) # I don't believe it matters if we use recon[0] or 1 here, but haven't checked
# b3n=c[3].get_attr_default("ptcl_repr",1.0)
#
# recon[0].determine_slice_agreement(a3,c[1],a3n,0) # c[-1]==0
# q0a=a3["reconstruct_absqual"] # quality for average a in reconstruction0
# n0a=a3["reconstruct_norm"] # normalization for same
#
# recon[1].determine_slice_agreement(a3,c[1],a3n,0) # c[-1]==1
# q1a=a3["reconstruct_absqual"] # quality for average a in reconstruction0
# n1a=a3["reconstruct_norm"] # normalization for same
#
# recon[0].determine_slice_agreement(b3,c[1],b3n,0) # c[-1]==1
# q0b=b3["reconstruct_absqual"] # quality for average a in reconstruction0
# n0b=b3["reconstruct_norm"] # normalization for same
#
# recon[1].determine_slice_agreement(b3,c[1],b3n,0) # c[-1]==0
# q1b=b3["reconstruct_absqual"] # quality for average a in reconstruction0
# n1b=b3["reconstruct_norm"] # normalization for same
#
# if options.verbose>1 : print i,q0a,q1a,q0b,q1b,q0a+q1b,q1a+q0b
#
# if q0a+q1b>q1a+q0b : # if true, a -> recon0 and b -> recon1
# if c[-1]==1 :
# c[-1]=0
# print i," 1->0"
#
# c.append(0) # we put a 0 at the end of the classes element if we use a->0,b->1 ordering, 1 if swapped
# a3.mult(n0a)
# recon[0].insert_slice(a3,c[1],a3n)
# b3.mult(n1b)
# recon[1].insert_slice(b3,c[1],b3n)
# else:
# if c[-1]==0 :
# c[-1]=1
# print i," 0->1"
#
# c.append(1)
# a3.mult(n1a)
# recon[1].insert_slice(a3,c[1],a3n)
# b3.mult(n0b)
#
#
if options.verbose: print "All done, writing output"
if mask != None: msk = "_msk"
else: msk = ""
classout = [
"{}/classes_{:02d}_bas{}{}_split0.hdf".format(options.path, last_iter,
options.usebasis, msk),
"{}/classes_{:02d}_bas{}{}_split1.hdf".format(options.path, last_iter,
options.usebasis, msk)
]
basisout = "{}/classes_{:02d}{}_basis".format(options.path, last_iter, msk)
threedout = "{}/threed_{:02d}{}_split.hdf".format(options.path, last_iter,
msk)
threedout2 = "{}/threed_{:02d}{}_split_filt_bas{}.hdf".format(
options.path, last_iter, msk, options.usebasis)
setout = [
"sets/split_{}{}_bas{}_0.lst".format(pathnum, msk, options.usebasis),
"sets/split_{}{}_bas{}_1.lst".format(pathnum, msk, options.usebasis)
]
split = [
r.finish(True).get_clip(
Region((pad - boxsize) / 2, (pad - boxsize) / 2,
(pad - boxsize) / 2, boxsize, boxsize, boxsize))
for r in recon
]
split[0]["apix_x"] = apix
split[0]["apix_y"] = apix
split[0]["apix_z"] = apix
split[1]["apix_x"] = apix
split[1]["apix_y"] = apix
split[1]["apix_z"] = apix
split[0].process_inplace("mask.soft", {"outer_radius": -8, "width": 4})
split[1].process_inplace("mask.soft", {"outer_radius": -8, "width": 4})
split[0].write_image(threedout, 0)
split[1].write_image(threedout, 1)
# now we write the class-averages and the new (split) particle files
lstin = [LSXFile(ptcls[0], True), LSXFile(ptcls[1], True)]
try:
os.unlink("sets/split0.lst")
os.unlink("sets/split1.lst")
except:
pass
lstout = [LSXFile("sets/split0.lst"), LSXFile("sets/split1.lst")]
for i, c in enumerate(classes):
c[2].write_image(classout[c[-1]], i) # class-average
ptcln = c[2]["class_eoidxs"] # eofile/ptcl# pairs
for p in xrange(0, len(ptcln), 2):
lstout[0][-1] = lstin[ptcln[p]][ptcln[
p + 1]] # wierd syntax, but the -1 here appends
c[3].write_image(classout[c[-1] ^ 1], i)
ptcln = c[3]["class_eoidxs"] # eofile/ptcl# pairs
for p in xrange(0, len(ptcln), 2):
lstout[1][-1] = lstin[ptcln[p]][ptcln[
p + 1]] # wierd syntax, but the -1 here appends
if options.verbose > 2:
c[4][0].write_image(basisout + "1.hdf", i)
c[4][1].write_image(basisout + "2.hdf", i)
c[4][2].write_image(basisout + "3.hdf", i)
launch_childprocess("e2proclst.py sets/split0.lst --mergesort {}".format(
setout[0]))
launch_childprocess("e2proclst.py sets/split1.lst --mergesort {}".format(
setout[1]))
try:
os.unlink("sets/split0.lst")
os.unlink("sets/split1.lst")
except:
pass
if os.path.exists("strucfac.txt"):
launch_childprocess(
"e2proc3d.py {} {} --setsf strucfac.txt --process filter.wiener.byfsc:fscfile={}/fsc_masked_{:02d}.txt:snrmult=2:sscale=1.1:maxfreq={} --process mask.soft:outer_radius=-9:width=4"
.format(threedout, threedout2, options.path, last_iter,
1.0 / targetres))
else:
print "Missing structure factor, cannot filter properly"
launch_childprocess(
"e2proc3d.py {} {} --process filter.wiener.byfsc:fscfile={}/fsc_masked_{:02d}.txt:snrmult=2:sscale=1.1:maxfreq={} --process mask.soft:outer_radius=-9:width=4"
.format(threedout, threedout2, options.path, last_iter,
1.0 / targetres))
E2end(logger)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog <output> [options]
This program produces iterative class-averages, one of the secrets to EMAN's rapid convergence.
Normal usage is to provide a stack of particle images and a classification matrix file defining
class membership. Members of each class are then iteratively aligned to each other and averaged
together with (optional) CTF correction. It is also possible to use this program on all of the
images in a single stack.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--input",
type=str,
help="The name of the input particle stack",
default=None)
parser.add_argument("--output",
type=str,
help="The name of the output class-average stack",
default=None)
parser.add_argument(
"--oneclass",
type=int,
help="Create only a single class-average. Specify the number.",
default=None)
parser.add_argument(
"--classmx",
type=str,
help=
"The name of the classification matrix specifying how particles in 'input' should be grouped. If omitted, all particles will be averaged.",
default=None)
parser.add_argument(
"--ref",
type=str,
help=
"Reference image(s). Used as an initial alignment reference and for final orientation adjustment if present. Also used to assign euler angles to the generated classes. This is typically the projections that were used for classification.",
default=None)
parser.add_argument(
"--storebad",
action="store_true",
help=
"Even if a class-average fails, write to the output. Forces 1->1 numbering in output",
default=False)
parser.add_argument(
"--decayedge",
action="store_true",
help=
"Applies an edge decay to zero on the output class-averages. A very good idea if you plan on 3-D reconstruction.",
default=False)
parser.add_argument(
"--resultmx",
type=str,
help=
"Specify an output image to store the result matrix. This contains 5 images where row is particle number. Rows in the first image contain the class numbers and in the second image consist of 1s or 0s indicating whether or not the particle was included in the class. The corresponding rows in the third, fourth and fifth images are the refined x, y and angle (respectively) used in the final alignment, these are updated and accurate, even if the particle was excluded from the class.",
default=None)
parser.add_argument(
"--iter",
type=int,
help="The number of iterations to perform. Default is 1.",
default=1)
parser.add_argument(
"--prefilt",
action="store_true",
help=
"Filter each reference (c) to match the power spectrum of each particle (r) before alignment and comparison",
default=False)
parser.add_argument(
"--align",
type=str,
help=
"This is the aligner used to align particles to the previous class average. Default is None.",
default=None)
parser.add_argument(
"--aligncmp",
type=str,
help="The comparitor used for the --align aligner. Default is ccc.",
default="ccc")
parser.add_argument(
"--ralign",
type=str,
help=
"This is the second stage aligner used to refine the first alignment. This is usually the \'refine\' aligner.",
default=None)
parser.add_argument(
"--raligncmp",
type=str,
help="The comparitor used by the second stage aligner.",
default="ccc")
parser.add_argument(
"--averager",
type=str,
help="The type of averager used to produce the class average.",
default="mean")
parser.add_argument(
"--setsfref",
action="store_true",
help=
"This will impose the 1-D structure factor of the reference on the class-average (recommended when a reference is available)",
default=False)
parser.add_argument(
"--cmp",
type=str,
help=
"The comparitor used to generate quality scores for the purpose of particle exclusion in classes, strongly linked to the keep argument.",
default="ccc")
parser.add_argument(
"--keep",
type=float,
help="The fraction of particles to keep in each class.",
default=1.0)
parser.add_argument(
"--keepsig",
action="store_true",
help=
"Causes the keep argument to be interpreted in standard deviations.",
default=False)
parser.add_argument(
"--automask",
action="store_true",
help=
"Applies a 2-D automask before centering. Can help with negative stain data, and other cases where centering is poor."
)
parser.add_argument(
"--center",
type=str,
default="xform.center",
help=
"If the default centering algorithm (xform.center) doesn't work well, you can specify one of the others here (e2help.py processor center)"
)
parser.add_argument("--bootstrap",
action="store_true",
help="Ignored. Present for historical reasons only.")
parser.add_argument(
"--normproc",
type=str,
help=
"Normalization processor applied to particles before alignment. Default is normalize.edgemean. If you want to turn this option off specify \'None\'",
default="normalize.edgemean")
parser.add_argument(
"--usefilt",
dest="usefilt",
default=None,
help=
"Specify a particle data file that has been low pass or Wiener filtered. Has a one to one correspondence with your particle data. If specified will be used to align particles to the running class average, however the original particle will be used to generate the actual final class average"
)
parser.add_argument("--idxcache",
default=False,
action="store_true",
help="Ignored. Present for historical reasons.")
parser.add_argument("--dbpath",
help="Ignored. Present for historical reasons.",
default=".")
parser.add_argument(
"--resample",
action="store_true",
help=
"If set, will perform bootstrap resampling on the particle data for use in making variance maps.",
default=False)
parser.add_argument(
"--odd",
default=False,
help=
"Used by EMAN2 when running eotests. Includes only odd numbered particles in class averages.",
action="store_true")
parser.add_argument(
"--even",
default=False,
help=
"Used by EMAN2 when running eotests. Includes only even numbered particles in class averages.",
action="store_true")
parser.add_argument("--parallel",
default=None,
help="parallelism argument")
parser.add_argument("--force",
"-f",
dest="force",
default=False,
action="store_true",
help="Force overwrite the output file if it exists.")
parser.add_argument(
"--saveali",
action="store_true",
help=
"Writes aligned particle images to aligned.hdf. Normally resultmx produces more useful informtation. This can be used for debugging.",
default=False)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness")
parser.add_argument(
"--debug",
"-d",
action="store_true",
help=
"Print debugging infromation while the program is running. Default is off.",
default=False)
parser.add_argument(
"--nofilecheck",
action="store_true",
help=
"Turns file checking off in the check functionality - used by e2refine.py.",
default=False)
parser.add_argument("--check",
"-c",
action="store_true",
help="Performs a command line argument check only.",
default=False)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
if (options.check):
options.verbose = 9 # turn verbose on if the user is only checking...
error = check(options, True)
if options.align: options.align = parsemodopt(options.align)
if options.ralign: options.ralign = parsemodopt(options.ralign)
if options.aligncmp: options.aligncmp = parsemodopt(options.aligncmp)
if options.raligncmp: options.raligncmp = parsemodopt(options.raligncmp)
if options.averager: options.averager = parsemodopt(options.averager)
if options.cmp: options.cmp = parsemodopt(options.cmp)
if options.normproc: options.normproc = parsemodopt(options.normproc)
if options.resultmx != None: options.storebad = True
if (options.verbose > 0):
if (error):
print "e2classaverage.py command line arguments test.... FAILED"
else:
print "e2classaverage.py command line arguments test.... PASSED"
# returning a different error code is currently important to e2refine.py - returning 0 tells e2refine.py that it has enough
# information to execute this script
if error: exit(1)
if options.check: exit(0)
logger = E2init(sys.argv, options.ppid)
print "Class averaging beginning"
try:
classmx = EMData.read_images(
options.classmx
) # we keep the entire classification matrix in memory, since we need to update it in most cases
ncls = int(classmx[0]["maximum"]) + 1
except:
ncls = 1
if options.resultmx != None:
print "resultmx can only be specified in conjunction with a valid classmx input."
sys.exit(1)
nptcl = EMUtil.get_image_count(options.input)
try:
apix = EMData(options.input, 0, True)["apix_x"]
except:
apix = 1.0
print "WARNING: could not get apix from first image. Setting to 1.0. May impact results !"
# Initialize parallelism
if options.parallel:
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel)
pclist = [options.input]
if options.ref: pclist.append(options.ref)
if options.usefilt: pclist.append(options.usefilt)
etc.precache(pclist)
# prepare tasks
tasks = []
if ncls > 1:
if options.oneclass == None: clslst = range(ncls)
else: clslst = [options.oneclass]
for cl in clslst:
ptcls = classmx_ptcls(classmx[0], cl)
if options.resample:
ptcls = [
random.choice(ptcls) for i in ptcls
] # this implements bootstrap resampling of the class-average
if options.odd: ptcls = [i for i in ptcls if i % 2 == 1]
if options.even: ptcls = [i for i in ptcls if i % 2 == 0]
tasks.append(
ClassAvTask(options.input, ptcls, options.usefilt, options.ref,
options.iter, options.normproc, options.prefilt,
options.align, options.aligncmp, options.ralign,
options.raligncmp, options.averager, options.cmp,
options.keep, options.keepsig, options.automask,
options.saveali, options.setsfref, options.verbose,
cl, options.center))
else:
ptcls = range(nptcl)
if options.resample: ptcls = [random.choice(ptcls) for i in ptcls]
if options.odd: ptcls = [i for i in ptcls if i % 2 == 1]
if options.even: ptcls = [i for i in ptcls if i % 2 == 0]
tasks.append(
ClassAvTask(options.input, range(nptcl), options.usefilt,
options.ref, options.iter, options.normproc,
options.prefilt, options.align, options.aligncmp,
options.ralign, options.raligncmp, options.averager,
options.cmp, options.keep, options.keepsig,
options.automask, options.saveali, options.setsfref,
options.verbose, 0, options.center))
# execute task list
if options.parallel: # run in parallel
taskids = etc.send_tasks(tasks)
alltaskids = taskids[:]
while len(taskids) > 0:
curstat = etc.check_task(taskids)
for i, j in enumerate(curstat):
if j == 100:
rslt = etc.get_results(taskids[i])
if rslt[1]["average"] != None:
rslt[1]["average"]["class_ptcl_src"] = options.input
if options.decayedge:
nx = rslt[1]["average"]["nx"]
rslt[1]["average"].process_inplace(
"normalize.circlemean",
{"radius": nx / 2 - nx / 15})
rslt[1]["average"].process_inplace(
"mask.gaussian", {
"inner_radius": nx / 2 - nx / 15,
"outer_radius": nx / 20
})
#rslt[1]["average"].process_inplace("mask.decayedge2d",{"width":nx/15})
if options.ref != None:
rslt[1]["average"][
"projection_image"] = options.ref
if options.storebad:
rslt[1]["average"].write_image(
options.output, rslt[1]["n"])
else:
rslt[1]["average"].write_image(options.output, -1)
# Update the resultsmx if requested
if options.resultmx != None:
allinfo = rslt[1][
"info"] # the info result array list of (qual,xform,used) tuples
pnums = rslt[0].data["images"][
2] # list of image numbers corresponding to information
for n, info in enumerate(allinfo):
y = pnums[n] # actual particle number
# find the matching class in the existing classification matrix
for x in range(classmx[0]["nx"]):
if classmx[0][x, y] == rslt[1][
"n"]: # if the class number in the classmx matches the current class-average number
break
else:
print "Resultmx error: no match found ! (%d %d %d)" % (
x, y, rslt[1]["n"])
continue
xform = info[1].get_params("2d")
classmx[1][x, y] = info[2] # used
classmx[2][x, y] = xform["tx"] # dx
classmx[3][x, y] = xform["ty"] # dy
classmx[4][x, y] = xform["alpha"] # da
classmx[5][x, y] = xform["mirror"] # flip
try:
classmx[6][x, y] = xform["scale"]
except:
pass
# failed average
elif options.storebad:
blk = EMData(options.ref, 0)
apix = blk["apix_x"]
blk = EMData(blk["nx"], blk["ny"], 1)
blk["apix_x"] = apix
blk.to_zero()
blk.set_attr("ptcl_repr", 0)
blk.set_attr("apix_x", apix)
blk.write_image(options.output, rslt[1]["n"])
taskids = [j for i, j in enumerate(taskids) if curstat[i] != 100]
if options.verbose and 100 in curstat:
print "%d/%d tasks remain" % (len(taskids), len(alltaskids))
if 100 in curstat:
E2progress(logger,
1.0 - (float(len(taskids)) / len(alltaskids)))
time.sleep(3)
if options.verbose: print "Completed all tasks"
# single thread
else:
for t in tasks:
rslt = t.execute()
if rslt == None: sys.exit(1)
if rslt["average"] != None:
rslt["average"]["class_ptcl_src"] = options.input
if options.decayedge:
nx = rslt["average"]["nx"]
rslt["average"].process_inplace(
"normalize.circlemean", {"radius": nx / 2 - nx / 15})
rslt["average"].process_inplace(
"mask.gaussian", {
"inner_radius": nx / 2 - nx / 15,
"outer_radius": nx / 20
})
#rslt["average"].process_inplace("mask.decayedge2d",{"width":nx/15})
if options.ref != None:
rslt["average"]["projection_image"] = options.ref
try:
if options.storebad:
rslt["average"].write_image(options.output,
t.options["n"])
else:
rslt["average"].write_image(options.output, -1)
except:
traceback.print_exc()
print "Error writing class average {} to {}".format(
t.options["n"], options.output)
print "Image attr: ", rslt["average"].get_attr_dict()
display(rslt["average"])
sys.exit(1)
# Update the resultsmx if requested
if options.resultmx != None:
allinfo = rslt[
"info"] # the info result array list of (qual,xform,used) tuples
pnums = t.data["images"][
2] # list of image numbers corresponding to information
for n, info in enumerate(allinfo):
y = pnums[n] # actual particle number
# find the matching class in the existing classification matrix
for x in range(classmx[0]["nx"]):
if classmx[0][x, y] == rslt[
"n"]: # if the class number in the classmx matches the current class-average number
break
else:
print "Resultmx error: no match found ! (%d %d %d)" % (
x, y, rslt[1]["n"])
continue
xform = info[1].get_params("2d")
classmx[1][x, y] = info[2] # used
classmx[2][x, y] = xform["tx"] # dx
classmx[3][x, y] = xform["ty"] # dy
classmx[4][x, y] = xform["alpha"] # da
classmx[5][x, y] = xform["mirror"] # flip
try:
classmx[6][x, y] = xform["scale"]
except:
pass
# Failed average
elif options.storebad:
blk = EMData(options.ref, 0)
apix = blk["apix_x"]
blk = EMData(blk["nx"], blk["ny"], 1)
blk["apix_x"] = apix
blk.to_zero()
blk.set_attr("ptcl_repr", 0)
blk.set_attr("apix_x", apix)
blk.write_image(options.output, t.options["n"])
if options.resultmx != None:
if options.verbose: print "Writing results matrix"
for i, j in enumerate(classmx):
j.write_image(options.resultmx, i)
print "Class averaging complete"
E2end(logger)
def main():
usage=" "
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--path", type=str,help="path", default=None, guitype='strbox',row=0, col=0,rowspan=1, colspan=1)
parser.add_argument("--iter", type=int,help="start from iteration X", default=-1, guitype='intbox',row=0, col=1,rowspan=1, colspan=1)
parser.add_argument("--niters", type=int,help="run this many iterations. Default is 4.", default=4, guitype='intbox',row=0, col=2,rowspan=1, colspan=1)
parser.add_argument("--sym", type=str,help="symmetry. will use symmetry from spt refinement by default", default="", guitype='strbox',row=2, col=0,rowspan=1, colspan=1)
parser.add_argument("--padby", type=float,help="pad by factor. default is 2", default=2., guitype='floatbox',row=1, col=1,rowspan=1, colspan=1)
parser.add_argument("--keep", type=float,help="propotion of tilts to keep. default is 0.5", default=0.5, guitype='floatbox',row=1, col=2,rowspan=1, colspan=1)
parser.add_argument("--maxalt", type=float,help="max altitude to insert to volume", default=90.0, guitype='floatbox',row=1, col=0,rowspan=1, colspan=1)
parser.add_argument("--nogs", action="store_true", default=False ,help="skip gold standard...", guitype='boolbox',row=2, col=1,rowspan=1, colspan=1)
parser.add_argument("--localfilter", type=int, default=-1 ,help="use tophat local. specify 0 or 1 to overwrite the setting in the spt refinement")
parser.add_argument("--mask", type=str, default="None" ,help="Refinement masking. default is the same as the spt refinement. Leave this empty for automasking",guitype='strbox',row=3, col=0,rowspan=1, colspan=2)
parser.add_argument("--threads", type=int,help="Number of CPU threads to use. Default is 12.", default=12, guitype='intbox',row=2, col=2,rowspan=1, colspan=1)
parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use. Default is thread:12", default="thread:12", guitype='strbox',row=4, col=0,rowspan=1, colspan=3)
parser.add_argument("--refineastep", type=float,help="angular variation for refine alignment (gauss std)", default=8.)
parser.add_argument("--refinentry", type=int,help="number of starting points for refine alignment", default=32)
parser.add_argument("--maxshift", type=int,help="maximum shift allowed", default=10)
parser.add_argument("--buildsetonly", action="store_true", default=False ,help="build sets only")
parser.add_argument("--output", type=str,help="Write results to this directory. We do not recommend changing this.", default="subtlt")#, guitype='intbox',row=2, col=1,rowspan=1, colspan=1)
parser.add_argument("--debug", action="store_true", default=False ,help="Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)")
parser.add_argument("--ppid", type=int,help="ppid...", default=-1)
(options, args) = parser.parse_args()
logid=E2init(sys.argv)
itr = options.iter
oldpath = options.path
if not oldpath:
print("No input path. Exit.")
return
if options.iter != -1:
itr = options.iter
elif "spt" in oldpath:
for f in sorted(os.listdir(oldpath)):
if "particle_parms" in f:
itrstr = f[15:].split(".")[0]
if os.path.isfile("{}/threed_{}.hdf".format(oldpath,itrstr)):
itr = int(itrstr)
else:
for f in sorted(os.listdir(oldpath)):
if re.match("threed_[0-9][0-9].hdf",f):
itr = int(f[7:].split(".")[0])
# print(oldpath)
fromspt=True
if "0_subtlt_params.json" in os.listdir(oldpath):
print("Continuing from a subtilt refinement...")
fromspt=False
path = make_path(options.output)
if not os.path.isfile("{}/threed_{:02d}.hdf".format(oldpath,itr)):
print("Could not locate {}/threed_{:02d}.hdf".format(oldpath,itr))
print("Please specify the iteration number (--iter) of a completed subtomogram refinement.")
sys.exit(1)
#elif not os.path.isfile("{}/particle_parms_{:02d}.json".format(oldpath,itr)):
#print("Could not locate {}/particle_parms_{:02d}.json".format(oldpath,itr))
#print("Please specify the iteration number (--iter) of a completed subtomogram refinement.")
#sys.exit(1)
else:
#copy2("{}/0_spt_params.json".format(oldpath),"{}/0_subtlt_params.json".format(path))
oldmap = os.path.join(oldpath,"threed_{:02d}.hdf".format(itr))
oem = os.path.join(oldpath,"threed_{:02d}_even.hdf".format(itr))
oom = os.path.join(oldpath,"threed_{:02d}_odd.hdf".format(itr))
oldfsc = os.path.join(oldpath, "fsc_masked_{:02d}.txt".format(itr))
copy2(oldmap,os.path.join(path,"threed_00.hdf"))
copy2(oldfsc, os.path.join(path, "fsc_masked_00.txt"))
copy2(oem,os.path.join(path,"threed_00_even.hdf"))
copy2(oom,os.path.join(path,"threed_00_odd.hdf"))
if fromspt:
oldparm = os.path.join(oldpath,"particle_parms_{:02d}.json".format(itr))
copy2(oldparm,os.path.join(path,"particle_parms_00.json"))
else:
for eo in ["even", "odd"]:
oali = os.path.join(oldpath,"ali_ptcls_{:02d}_{}.lst".format(itr, eo))
copy2(oali,os.path.join(path,"ali_ptcls_00_{}.lst".format(eo)))
e=EMData(os.path.join(path,"threed_00.hdf"))
bxsz=e["nx"]
apix=e["apix_x"]
jd = js_open_dict("{}/0_subtlt_params.json".format(path))
jd.update(vars(options))
jd["cmd"] = " ".join(sys.argv)
jd["path"] = oldpath
jd["iter"] = itr
jd["output"] = path
if fromspt:
sptparms = os.path.join(oldpath,"0_spt_params.json")
else:
sptparms = os.path.join(oldpath,"0_subtlt_params.json")
if os.path.isfile(sptparms):
oldjd = js_open_dict(sptparms)
#print(oldjd.keys())
jd["mass"] = oldjd["mass"]
jd["setsf"] = oldjd["setsf"]
jd["sym"] = oldjd["sym"]
jd["localfilter"]=oldjd["localfilter"]
jd["mask"]=oldjd["mask"]
oldjd.close()
else:
print("Cannot find {}. exit.".format(sptparms))
if options.mask.lower()!="none":
print("Overwritting masking")
jd["mask"]=options.mask
if options.localfilter==0:
jd["localfilter"]=False
elif options.localfilter==1:
jd["localfilter"]=True
if len(options.sym)>0:
jd["sym"]=options.sym
jsparams=jd.data
jd.close()
jd = jsparams
if fromspt:
js=js_open_dict(os.path.join(path,"particle_parms_00.json"))
k=list(js.keys())[0]
src=eval(k)[0]
print("loading 3D particles from {}".format(src))
print("box size {}, apix {:.2f}".format(bxsz, apix))
lname=[os.path.join(path, "ali_ptcls_00_{}.lst".format(eo)) for eo in ["even", "odd"]]
for l in lname:
try: os.remove(l)
except:pass
lst=[LSXFile(m, False) for m in lname]
n3d=len(list(js.keys()))
for ii in range(n3d):
e=EMData(src, ii, True)
fname=e["class_ptcl_src"]
ids=e["class_ptcl_idxs"]
ky="('{}', {})".format(src, ii)
dic=js[ky]
xali=dic["xform.align3d"]
for i in ids:
try:
m=EMData(fname, i, True)
except:
continue
xf=m["xform.projection"]
dc=xf.get_params("xyz")
if abs(dc["ytilt"])>options.maxalt:
continue
rot=xf*xali.inverse()
lst[ii%2].write(-1, i, fname, str(rot.get_params("eman")))
for l in lst:
l.close()
js=None
if options.buildsetonly: return
for itr in range(0,options.niters):
from EMAN2PAR import EMTaskCustomer
for eo in ["even", "odd"]:
if options.nogs:
threedname="{}/threed_{:02d}.hdf".format(path, itr)
else:
threedname="{}/threed_{:02d}_{}.hdf".format(path, itr, eo)
lstname="{}/ali_ptcls_{:02d}_{}.lst".format(path, itr, eo)
lst=LSXFile(lstname, True)
m=EMData(threedname)
m.process_inplace('normalize.edgemean')
pinfo=[]
if options.debug: nptcl=options.threads*8
else: nptcl=lst.n
for i in range(nptcl):
pinfo.append(lst.read(i))
lst=None
etc=EMTaskCustomer(options.parallel)
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
print("{} jobs".format(nptcl))
infos=[[] for i in range(num_cpus)]
for i,info in enumerate(pinfo):
infos[i%num_cpus].append([i, info])
tids=[]
for info in infos:
task = SptTltRefineTask(info, m, options)
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
#print(tids)
if np.min(st_vals) == 100: break
time.sleep(5)
dics=[0]*nptcl
for i in tids:
ret=etc.get_results(i)[1]
for r in ret:
#print(r)
ii=r.pop("idx")
dics[ii]=r
del etc
allscr=np.array([d["score"] for d in dics])
print(np.min(allscr), np.mean(allscr), np.max(allscr), np.std(allscr))
allscr*=-1
s=allscr.copy()
s-=np.mean(s)
s/=np.std(s)
clp=2
ol=abs(s)>clp
print("Removing {} outliers from {} particles..".format(np.sum(ol), len(s)))
s=old_div(old_div((s+clp),clp),2)
s[ol]=0
allscr=s
#allscr-=np.min(allscr)-1e-5
#allscr/=np.max(allscr)
lname="{}/ali_ptcls_{:02d}_{}.lst".format(path, itr+1, eo)
try: os.remove(lname)
except: pass
lout=LSXFile(lname, False)
for i, dc in enumerate(dics):
d=dc["xform.align3d"].get_params("eman")
d["score"]=float(allscr[i])
l=pinfo[i]
lout.write(-1, l[0], l[1], str(d))
lout=None
pb=options.padby
threedout="{}/threed_{:02d}_{}.hdf".format(path, itr+1, eo)
cmd="e2make3dpar.py --input {inp} --output {out} --pad {pd} --padvol {pdv} --threads {trd} --outsize {bx} --apix {apx} --mode gauss_var --keep {kp} --sym {sm}".format(
inp=lname,
out=threedout,
bx=bxsz, pd=int(bxsz*pb), pdv=int(bxsz*pb), apx=apix, kp=options.keep, sm=jd["sym"], trd=options.threads)
run(cmd)
run("e2proc3d.py {} {}".format(threedout, "{}/threed_raw_{}.hdf".format(path, eo)))
s = ""
if jd.has_key("goldstandard"):
if jd["goldstandard"] > 0:
s += " --align"
if jd.has_key("setsf"):
s += " --setsf {}".format(jd['setsf']) #options.setsf)
if jd.has_key("localfilter"):
s += " --tophat local"
msk = jd["mask"] #{}/mask_tight.hdf".format(path)
if len(msk)>0:
if os.path.isfile(msk):
msk=" --automask3d mask.fromfile:filename={}".format(msk)
else:
msk=" --automask3d {}".format(msk)
# get target resolution from last iteration map
ref=os.path.join(path, "threed_{:02d}.hdf".format(itr))
fsc=np.loadtxt(os.path.join(path, "fsc_masked_{:02d}.txt".format(itr)))
rs=1./fsc[fsc[:,1]<0.3, 0][0]
curres=rs*.5
#os.system("rm {}/mask*.hdf {}/*unmasked.hdf".format(path, path))
ppcmd="e2refine_postprocess.py --even {} --odd {} --output {} --iter {:d} --restarget {} --threads {} --sym {} --mass {} {}".format(
os.path.join(path, "threed_{:02d}_even.hdf".format(itr+1)),
os.path.join(path, "threed_{:02d}_odd.hdf".format(itr+1)),
os.path.join(path, "threed_{:02d}.hdf".format(itr+1)), itr+1, curres, options.threads, jd["sym"], jd["mass"], s)
run(ppcmd)
fsc=np.loadtxt(os.path.join(path, "fsc_masked_{:02d}.txt".format(itr+1)))
rs=1./fsc[fsc[:,1]<0.3, 0][0]
print("Resolution (FSC<0.3) is ~{:.1f} A".format(rs))
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog <output> [options]
This program is used to preprocess subtomograms before aligning them. The same can be accomplished with
e2proc3d, except that this program is parallelized and thus should be substantially faster for large subtomograms.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--input",
type=str,
default='',
help=
"""Default=None. The name of the input volume stack. MUST be HDF since volume stack support is required."""
)
parser.add_argument(
"--output",
type=str,
default='',
help=
"""Default=None. Specific name of HDF file to write processed particles to."""
)
parser.add_argument(
"--parallel",
type=str,
default='',
help=
"""default=None. Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel"""
)
parser.add_argument(
"--ppid",
type=int,
help=
"""Default=-1. Set the PID of the parent process, used for cross platform PPID""",
default=-1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"""Default=0. Verbose level [0-9], higner number means higher level of verboseness"""
)
parser.add_argument(
"--subset",
type=int,
default=0,
help=
"""Default=0 (not used). Refine only this substet of particles from the stack provided through --input"""
)
parser.add_argument(
"--apix",
type=float,
default=0.0,
help=
"""Default=0.0 (not used). Use this apix value where relevant instead of whatever is in the header of the reference and the particles. Will overwrite particle header as well."""
)
parser.add_argument(
"--shrink",
type=int,
default=0,
help=
"""Default=0 (no shrinking). Optionally shrink the input volumes by an integer amount for coarse alignment."""
)
parser.add_argument(
"--threshold",
type=str,
default='',
help=
"""Default=None. A threshold applied to the subvolumes after normalization. For example, --threshold=threshold.belowtozero:minval=0 makes all negative pixels equal 0, so that they do not contribute to the correlation score."""
)
parser.add_argument(
"--mask",
type=str,
default='',
help=
"""Default=None. Masking processor applied to particles before alignment. IF using --clip, make sure to express outer mask radii as negative pixels from the edge."""
)
parser.add_argument(
"--maskfile",
type=str,
default='',
help=
"""Default=None. Mask file (3D IMAGE) applied to particles before alignment. Must be in HDF format. Default is None."""
)
parser.add_argument(
"--normproc",
type=str,
default='',
help=
"""Default=None (see 'e2help.py processors -v 10' at the command line). Normalization processor applied to particles before alignment. If normalize.mask is used, results of the mask option will be passed in automatically. If you want to turn this option off specify \'None\'"""
)
parser.add_argument(
"--preprocess",
type=str,
default='',
help=
"""Any processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--lowpass",
type=str,
default='',
help=
"""Default=None. A lowpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--highpass",
type=str,
default='',
help=
"""Default=None. A highpass filtering processor (see 'e2help.py processors -v 10' at the command line) to be applied to each volume prior to COARSE alignment. Not applied to aligned particles before averaging."""
)
parser.add_argument(
"--clip",
type=int,
default=0,
help=
"""Default=0 (which means it's not used). Boxsize to clip particles. For example, the boxsize of the particles might be 100 pixels, but the particles are only 50 pixels in diameter. Aliasing effects are not always as deleterious for all specimens, and sometimes 2x padding isn't necessary."""
)
parser.add_argument(
"--nopath",
action='store_true',
default=False,
help=
"""If supplied, this option will save results in the directory where the command is run. A directory to store the results will not be made."""
)
parser.add_argument(
"--path",
type=str,
default='sptpreproc',
help=
"""Default=spt. Directory to store results in. The default is a numbered series of directories containing the prefix 'sptpreproc'; for example, sptpreproc_02 will be the directory by default if 'sptpreproc_01' already exists."""
)
(options, args) = parser.parse_args()
logger = E2init(sys.argv, options.ppid)
print "\n(e2spt_preproc)(main) started log"
from e2spt_classaverage import sptmakepath
if options.path and not options.nopath:
options = sptmakepath(options, 'sptpreproc')
if options.parallel == 'None' or options.parallel == 'none':
options.parallel = None
if not options.input:
try:
options.input = sys.argv[1]
except:
print "\n(e2spt_preproc)(main) ERROR: invalid input file"
if options.mask or options.maskfile or options.threshold or options.clip or options.threshold or options.normproc or options.preprocess or options.lowpass or options.highpass or int(
options.shrink) > 1:
preprocstack = str(
os.path.basename(options.input).replace('.hdf', '_preproc.hdf'))
if options.path and not options.nopath:
preprocstack = options.path + '/' + preprocstack
if options.output:
if '.hdf' in options.output[-4:]:
preprocstack = options.output
else:
print "\n(e2spt_preproc)(main) ERROR: '.hdf' must be the last four characters of the output filename."
print "\n(e2spt_preproc)(main) output stack will be %s" % (
preprocstack)
n = 0
try:
n = EMUtil.get_image_count(options.input)
except:
print "\n(e2spt_preproc)(main) ERROR: --input stack seems to be invalid"
sys.exit()
print "\n(e2spt_preproc)(main) number of particles is %d" % (n)
c = os.getcwd()
findir = os.listdir(c)
if preprocstack not in findir:
dimg = EMData(8, 8, 8)
dimg.to_one()
for i in range(n):
dimg.write_image(preprocstack, i)
else:
print "\n(e2spt_preproc)(main) WARNING: a file with the name of the output stack %s is already in the current directory and will be DELETED" % (
preprocstack)
os.remove(preprocstack)
dimg = EMData(8, 8, 8)
dimg.to_one()
for i in range(n):
dimg.write_image(preprocstack, i)
finalbox = EMData(options.input, 0, True)['nx']
if options.clip:
finalbox = options.clip
#dimglarge=EMData(finalbox,finalbox,finalbox)
#dimglarge.to_one()
#dimglarge.write_image(preprocstack,0)
#n=EMUtil.get_image_count(options.input)
#if options.subset:
# n=options.subset
#dimglarge.write_image(preprocstack,n-1)
if options.verbose:
print "\n(e2spt_preproc)(main) wrote dummy ptcls to %s" % (
preprocstack)
print "\n(e2spt_preproc)(main) - INITIALIZING PARALLELISM!\n"
if options.parallel:
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel)
pclist = [options.input]
etc.precache(pclist)
print "\n(e2spt_preproc)(main) - precaching --input"
tasks = []
results = []
from e2spt_classaverage import sptOptionsParser
options = sptOptionsParser(options)
for j in range(n):
#print "processing particle", j
img = EMData(options.input, j)
if options.parallel:
#task = Preproc3DTask( ["cache",options.input,j], options, j, preprocstack )
task = Preproc3DTask(img, options, j, preprocstack)
tasks.append(task)
else:
img = EMData(options.input, j)
pimg = preprocfunc(img, options, j, preprocstack)
if options.parallel and tasks:
tids = etc.send_tasks(tasks)
if options.verbose:
print "\n(e2spt_preproc)(main) preprocessing %d tasks queued" % (
len(tids))
results = get_results(etc, tids, options)
#print "\n(e2spt_preproc)(main) preprocessing results are", results
#print "\n(e2spt_preproc)(main) input changing to preprocstack"
#options.input = preprocstack
#cache needs to be reloaded with the new options.input
else:
print "\n(e2spt_preproc)(main) Nothing to do. No preprocessing parameters specified."
E2end(logger)
return
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--ptclin",
type=str,
help="particle input",
default=None)
parser.add_argument("--output",
type=str,
help="score output",
default=None)
parser.add_argument(
"--parallel",
type=str,
help="Thread/mpi parallelism to use. Default is thread:12",
default="thread:12")
parser.add_argument(
"--debug",
action="store_true",
default=False,
help=
"Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)"
)
parser.add_argument("--maxres",
type=float,
default=15,
help="max resolution for cmp")
parser.add_argument("--minres",
type=float,
default=300,
help="min resolution for cmp")
parser.add_argument("--sym", type=str, help="symmetry. ", default="c1")
parser.add_argument("--mask", type=str, help="mask. ", default=None)
parser.add_argument("--maxshift", type=int, help="max shift.", default=0)
parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
threedname = args
lst = LSXFile(options.ptclin, True)
pinfo = []
nptcl = lst.n
for i in range(nptcl):
pinfo.append(lst.read(i))
lst = None
e = EMData(options.ptclin, 0, True)
nx = e["nx"]
apix = e["apix_x"]
b = good_size(nx * apix / options.maxres * 2)
options.shrink = max(1, nx / b)
print("Initializing parallelism...")
etc = EMTaskCustomer(options.parallel,
module="e2spa_classify.SpaClassifyTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
print("{} jobs".format(nptcl))
infos = [[] for i in range(num_cpus)]
for i, info in enumerate(pinfo):
infos[i % num_cpus].append([i, info])
tids = []
for info in infos:
task = SpaClassifyTask(info, threedname, options)
if options.debug:
task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
dics = np.zeros((nptcl, len(threedname)))
for i in tids:
ret = etc.get_results(i)[1]
for r in ret:
#print(r)
ii = r.pop("idx")
dics[ii] = r["score"]
del etc
np.savetxt(options.output, dics)
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] stack1.hdf stack2.mrcs ...
Program to erase gold fiducials and other high-density features from images, such as frames in DDD movies or images in tiltseries. Requires scipy.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
#parser.add_argument("--average", default=False, action="store_true", help="Erase gold from average of input stack(s).")
parser.add_argument(
"--lowpass",
default=False,
action="store_true",
help=
"Also lowpass filter noise based on local properties. Useful for processing tomographic tilt series."
)
parser.add_argument(
"--keepdust",
default=False,
action="store_true",
help=
"Do not remove 'dust' from mask (include objects smaller than gold fiducials)."
)
parser.add_argument(
"--goldsize",
default=30,
type=float,
help="Diameter (in pixels) of gold fiducials to erase.")
#parser.add_argument("--downsample", default=1.0, type=float, help="Downsample the input stack(s). Default is 1, i.e. no downsampling.")
parser.add_argument(
"--oversample",
default=4,
type=int,
help=
"Oversample noise image to smooth transitions from regions with different noise."
)
parser.add_argument("--boxsize",
default=128,
type=int,
help="Box size to use when computing local noise.")
parser.add_argument("--debug",
default=False,
action="store_true",
help="Save noise and mask/masked image(s).")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-2)
parser.add_argument(
"--parallel",
type=str,
default=None,
help=
"""Default=None (not used). Parallelism. See http://blake.bcm.edu/emanwiki/EMAN2/Parallel"""
)
(options, args) = parser.parse_args()
nfiles = len(args)
if options.parallel:
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel)
for argnum, arg in enumerate(args):
t0 = time.time()
newarg = ''
originalarg = arg
hdr = EMData(arg, 0,
True) #load header only to get parameters used below
apix = hdr['apix_x']
nx = hdr['nx']
ny = hdr['ny']
if '.ali' == arg[-4:] or '.mrc' == arg[-4:]:
#Unfortunately, e2proc2d.py appends to existing files instead of overwriting them. If you run this program two consecutive times and the first one failed for whatever reason,
#you'll find your stack growing.
#To prevent this, we create a 'dummy' file, but first remove any dummy files from previous failed runs. (If the program runs successfully to the end, the dummy file gets renamed).
try:
os.remove('dummy_stack.hdf')
except:
pass
#turn .ali or .mrc 3D images into a stack of 2D images that can be processed by this program.
cmd = 'e2proc2d.py ' + arg + ' dummy_stack.hdf --threed2twod'
runcmd(options, cmd)
#make the new stack of 2D images (dummy_stack.hdf) the new input (the name of the input file but with .hdf format); this intermediate file will be deleted in the end.
newarg = arg.replace(arg[-4:], '.hdf')
os.rename('dummy_stack.hdf', newarg)
arg = newarg
outf = "{}_proc.hdf".format(os.path.splitext(arg)[0])
if os.path.isfile(outf):
print(
"Results are already stored in {}. Please erase or move and try again."
.format(outf))
sys.exit(1)
nfs = EMUtil.get_image_count(arg)
tasks = []
results = []
results = None
#parallelized tasks don't run "in order"; therefore, a dummy stack needs to be pre-created with as many images as the final stack will have
#(otherwise, writing output images to stack indexes randomly makes the program crash or produces garbage output)
dummy = EMData(8, 8)
dummy.to_one()
dummy['apix_x'] = apix
dummy['apix_y'] = apix
for j in range(nfs):
dummy.write_image(outf, j)
#EMAN2 does not allow stacks of images with different size; this, and possibly some bug, prevent images written from the parallelization task from
#having the corret size if the pre-created dummy doesn't have the correct size to begin with. No point in writing big images for the dummy from the start.
#re-writing the index=0 image will change the size of all images in the stack to the correct size
dummy_correct_size = EMData(nx, ny)
dummy_correct_size.to_one()
dummy_correct_size['apix_x'] = apix
dummy_correct_size['apix_y'] = apix
dummy.write_image(outf, 0)
for i in range(nfs):
if options.verbose:
sys.stdout.write("\rstaging images ({}/{})".format(i + 1, nfs))
sys.stdout.flush()
if options.parallel:
#print "parallelism started"
task = EraseGold2DTask(options, arg, i, outf)
tasks.append(task)
else:
results = fiximage(options, arg, i, outf)
if options.parallel:
if tasks:
tids = etc.send_tasks(tasks)
if options.verbose:
print "\n(erase_gold) %d tasks queued" % (len(tids))
results = get_results(etc, tids, options)
if results:
#pass
if '.ali' == originalarg[-4:] or '.mrc' == originalarg[-4:]:
#intermediate = arg.replace('.hdf','.mrcs')
finaloutput = arg.replace('.hdf', originalarg[-4:])
cmd = 'e2proc2d.py ' + arg + ' ' + finaloutput + ' --twod2threed --outmode int16'
runcmd(options, cmd)
os.remove(arg)
if newarg: os.remove(newarg)
dt = time.time() - t0
if options.verbose:
print("\n")
sys.stdout.write("Erased fiducials from {} ({} minutes)\n".format(
arg, round(dt / 60., 2)))
