def execute(self,alignment_jobs,files,caller):
'''
The main function
@param alignment_jobs a list of alignment pair indices like this [[0,1],[2,1],[2,3],[0,5],...] etc the indices pair represent images to be aligned and correspond to the order of the files argument
@param files a list of filenames - used to read image based on the indices present in alignment_jobs
@param caller - the calling object - it needs to have a function called process_output that takes a dictionary as the argument
'''
options = self.options
align_data = EMAN2.parsemodopt(options.align)
align_cmp_data = EMAN2.parsemodopt(options.aligncmp)
cmp_data = EMAN2.parsemodopt(options.cmp)
ralign_data = None
if options.ralign != None:
ralign_data = EMAN2.parsemodopt(options.ralign)
ralign_cmp_data = EMAN2.parsemodopt(options.raligncmp)
data = {}
data["align"] = align_data
data["aligncmp"] = align_cmp_data
data["cmp"] = cmp_data
if ralign_data:
data["ralign"] = ralign_data
data["raligncmp"] = ralign_cmp_data
data["using_cuda"] = self.using_cuda
data["nsoln"] = self.nsoln
if self.options.parallel :
task_customers = []
tids = []
if options.shrink:
scratch_name_1 = numbered_bdb("bdb:tomo_scratch#scratch_shrink")
scratch_name_2 = numbered_bdb("bdb:tomo_scratch##scratch_shrink")
else: print "no shrink"
for i,j in alignment_jobs:
if options.shrink or options.filter:
a = EMData(files[i],0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",{"n":options.shrink})
a.set_attr("src_image",files[i])
a.write_image(scratch_name_1,0)
a = EMData(files[j],0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",{"n":options.shrink})
a.set_attr("src_image",files[j])
a.write_image(scratch_name_2,0)
data["probe"] = ("cache",scratch_name_1,0)
data["target"] = ("cache",scratch_name_2,0)
else:
data["probe"] = ("cache",files[i],0)
data["target"] = ("cache",files[j],0)
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTaskDC(data=data)
from EMAN2PAR import EMTaskCustomer
etc=EMTaskCustomer(self.options.parallel)
#print "Est %d CPUs"%etc.cpu_est()
tid=etc.send_task(task)
#print "Task submitted tid=",tid
task_customers.append(etc)
tids.append(tid)
self.dc_monitor(task_customers,tids,caller)
else:
n = len(alignment_jobs)
p = 0.0
for i,j in alignment_jobs:
probe = EMData(files[i],0)
target = EMData(files[j],0)
if options.filter:
print "filtered"
filter_params = EMAN2.parsemodopt(options.filter)
probe.process_inplace(filter_params[0],filter_params[1])
target.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
probe.process_inplace("math.meanshrink",{"n":options.shrink})
target.process_inplace("math.meanshrink",{"n":options.shrink})
else:
print "no shrink"
data["target"] = target
data["probe"] = probe
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTask(data=data)
rslts = task.execute(self.progress_callback)
if options.shrink:
self.correction_translation(rslts,options.shrink)
caller.process_output(rslts)
p += 1.0
class EMParallelProject3D:
def __init__(self, options, fsp, sym, start, modeln=0, logger=None):
"""
@param options the options produced by (options, args) = parser.parse_args()
@param args the options produced by (options, args) = parser.parse_args()
@param logger and EMAN2 logger, i.e. logger=E2init(sys.argv)
assumes you have already called the check function.
"""
self.options = options
self.args = fsp
self.sym = sym
self.logger = logger
self.start = start
self.modeln = modeln
from EMAN2PAR import EMTaskCustomer
self.etc = EMTaskCustomer(options.parallel)
print "Precache ", fsp
self.etc.precache([fsp])
self.num_cpus = self.etc.cpu_est()
print self.num_cpus, " total CPUs available"
if self.num_cpus > 64: # upper limit
self.num_cpus = 64
self.__task_options = None
def __init_memory(self, options):
"""
"""
sym_object = parsesym(self.sym)
[og_name, og_args] = parsemodopt(options.orientgen)
self.eulers = sym_object.gen_orientations(og_name, og_args)
def __get_task_options(self, options):
if self.__task_options == None:
d = {}
d["projector"] = parsemodopt(options.projector)
d["prethreshold"] = options.prethreshold
self.__task_options = d
return self.__task_options
def execute(self):
# from EMAN2PAR import EMTaskCustomer
if len(self.options.parallel) > 1:
self.__init_memory(self.options)
num_tasks = self.num_cpus
# In the worst case we can only spawn as many tasks as there are eulers
if self.num_cpus > len(self.eulers):
num_tasks = len(self.eulers)
eulers_per_task = len(self.eulers) / num_tasks
resid_eulers = len(self.eulers) - eulers_per_task * num_tasks # we can distribute the residual evenly
first = 0
task_customers = []
tids = []
# self.etc=EMTaskCustomer(self.options.parallel)
for i in xrange(0, num_tasks):
last = first + eulers_per_task
if resid_eulers > 0:
last += 1
resid_eulers -= 1
tmp_eulers = self.eulers[first:last]
indices = range(first, last)
data = {}
data["input"] = ("cache", self.args, 0)
data["eulers"] = tmp_eulers
data["indices"] = indices
task = EMProject3DTaskDC(data=data, options=self.__get_task_options(self.options))
# print "Est %d CPUs"%etc.cpu_est()
tid = self.etc.send_task(task)
# print "Task submitted tid=",tid
tids.append(tid)
first = last
print "Task ids are", tids
while 1:
print len(tids), "projection tasks left in main loop"
st_vals = self.etc.check_task(tids)
for i in xrange(len(tids) - 1, -1, -1):
st = st_vals[i]
if st == 100:
tid = tids[i]
rslts = self.etc.get_results(tid)
if not self.__write_output_data(rslts[1]):
print "There was a problem with the task of id", tid
if self.logger != None:
E2progress(self.logger, 1.0 - len(tids) / float(num_tasks))
if self.options.verbose > 0:
print "%d/%d\r" % (num_tasks - len(tids), num_tasks)
sys.stdout.flush()
print "Task", tids.pop(i), "completed"
print "These tasks are remaining:", tids
if len(tids) == 0:
break
time.sleep(5)
return len(self.eulers)
else:
raise NotImplementedError(
"The parallelism option you specified (%s) is not suppored" % self.options.parallel
)
def __write_output_data(self, rslts):
for idx, image in rslts.items():
if not isinstance(image, EMData):
continue # this is here because we get the dimensions of the database as a key (e.g. '40x40x1').
image["model_id"] = self.modeln
if self.options.append:
image.write_image(self.options.outfile, -1)
else:
image.write_image(self.options.outfile, idx + self.start)
return True
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
This program is part of the 'new' hierarchy of e2spt_ programs. It performs one iteration of a classical subtomogram refinement, ie - aligning particles with missing wedge to a reference in 3-D
The reference may be <volume> or <volume>,<n>
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--threads",
default=4,
type=int,
help=
"Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.",
guitype='intbox',
row=24,
col=2,
rowspan=1,
colspan=1,
mode="refinement")
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=0)
parser.add_argument(
"--goldcontinue",
action="store_true",
help=
"Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
default=False)
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
)
parser.add_argument(
"--sym",
type=str,
default="c1",
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly."
)
parser.add_argument(
"--maxres",
type=float,
help="Maximum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument(
"--minres",
type=float,
help="Minimum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument("--maxshift",
type=int,
help="maximum shift for subtilt refine",
default=8)
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default=None)
parser.add_argument("--fromscratch",
action="store_true",
help=".",
default=False)
parser.add_argument("--skipali3d",
action="store_true",
help="",
default=False)
parser.add_argument("--skipali2d",
action="store_true",
help="",
default=False)
parser.add_argument("--debug",
action="store_true",
help=".",
default=False)
parser.add_argument(
"--plst",
type=str,
default=None,
help=
"list of 2d particle with alignment parameters. will reconstruct before alignment."
)
parser.add_argument(
"--smooth",
type=float,
help=
"smooth local motion by this factor. smoother local motion with larger numbers",
default=-1)
parser.add_argument(
"--smoothN",
type=float,
help="number of neighboring particles used for smoothing. default 15",
default=15)
(options, args) = parser.parse_args()
if options.path == None:
fls = [
int(i[-2:]) for i in os.listdir(".")
if i[:4] == "spt_" and len(i) == 6 and str.isdigit(i[-2:])
]
if len(fls) == 0: fls = [0]
options.path = "spt_{:02d}".format(max(fls) + 1)
try:
os.mkdir(options.path)
except:
pass
if options.iter <= 0:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0: options.iter = 1
else: options.iter = max(fls) + 1
if options.parallel == None:
options.parallel = "thread:{}".format(options.threads)
# file may be "name" or "name,#"
reffile = args[1].split(",")[0]
try:
refn = int(args[1].split(",")[1])
except:
refn = 0
NTHREADS = max(options.threads + 1,
2) # we have one thread just writing results
logid = E2init(sys.argv, options.ppid)
refnames = [reffile, reffile]
if options.goldcontinue:
ref = []
try:
refnames = [reffile[:-4] + "_even.hdf", reffile[:-4] + "_odd.hdf"]
ref.append(EMData(refnames[0], 0))
ref.append(EMData(refnames[1], 0))
except:
print("Error: cannot find one of reference files, eg: ",
EMData(reffile[:-4] + "_even.hdf", 0))
n = -1
tasks = []
readjson = False
if args[0].endswith(".lst") or args[0].endswith(".hdf"):
nptcl = EMUtil.get_image_count(args[0])
tasks.extend([(args[0], i, refnames, i % 2) for i in range(nptcl)])
elif args[0].endswith(".json"):
js = js_open_dict(args[0])
readjson = True
jsinput = dict(js)
keys = sorted(js.keys())
nptcl = len(keys)
for k in keys:
src, ii = eval(k)
dic = js[k]
xf = dic["xform.align3d"]
tasks.append([src, ii, refnames, ii % 2, xf])
if options.plst:
xinfo = [[] for i in range(nptcl)]
xfkey = [
"type", "alt", "az", "phi", "tx", "ty", "tz", "alpha", "scale"
]
lst = LSXFile(options.plst)
for i in range(lst.n):
l = lst.read(i)
dc = eval(l[2])
pid = dc["pid"]
dxf = Transform({k: dc[k] for k in dc.keys() if k in xfkey})
xinfo[pid].append([dc["tid"], i, dxf])
lst = None
for i, xn in enumerate(xinfo):
sid = np.argsort([x[0] for x in xn])
xinfo[i] = [[xn[s][1], xn[s][2]] for s in sid]
for i in range(nptcl):
ii = tasks[i][1]
tasks[i].append(xinfo[ii])
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_align_subtlt.SptAlignTask")
num_cpus = etc.cpu_est()
options.nowtime = time.time()
if options.debug:
tasks = tasks[:num_cpus * 4]
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob = num_cpus
tids = []
for i in range(njob):
t = tasks[i::njob]
task = SptAlignTask(t, options)
if options.debug:
ret = task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output = []
for i in tids:
rets = etc.get_results(i)[1]
for ret in rets:
fsp, n, dic = ret
output.append([fsp, n, dic])
angs = {}
data = []
for out in output:
fsp, n, dc = out
angs[(fsp, n)] = {
"xform.align3d": dc["xform.align3d"],
"score": dc["score"]
}
ts = dc.pop("transscr")
for i, ii in enumerate(dc["imgidx"]):
dxf = dc["imgxfs"][i].get_params("eman")
#dxf["score"]=float(dc["imgscore"][i])
dxf["pid"] = n
dxf["tid"] = i
dxf["class"] = n % 2
tsmult = dc["tsmult"]
data.append({
"src": dc["imgsrc"],
"srci": ii,
"pid": n,
"tid": i,
"xf": dxf,
"coord": dc["coord"],
"tsscr": ts[i]
})
m = options.maxshift
trans = np.indices((m * 2 + 1, m * 2 + 1)).reshape((2, -1)).T - m
alldtx = np.zeros((len(data), 3))
fnames = np.unique([d["src"] for d in data])
maxtid = np.max([d["tid"] for d in data]) + 1
print("Smoothing trajectories...")
for fname in fnames:
print(fname)
for tid in range(maxtid):
idx = [
i for i, d in enumerate(data)
if d["src"] == fname and d["tid"] == tid
]
if len(idx) == 0:
continue
coord = np.array([data[i]["coord"] for i in idx])
scrs = np.array([data[i]["tsscr"] for i in idx])
if options.smooth <= 0:
s = np.argmin(scrs, 1)
sv = np.min(scrs, axis=1)
newdtx = np.hstack([trans[s], sv[:, None]])
if tid == 0: print(" Skip smoothing")
else:
newdtx = []
for i, crd in enumerate(coord):
dst = np.linalg.norm(coord - crd, axis=1)
srtid = np.argsort(dst)[:options.smoothN]
dst = dst[srtid]
wt = np.exp(-dst / options.smooth)
scr = scrs[srtid].copy()
scr = np.sum(scr * wt[:, None], axis=0)
pos = trans[np.argmin(scr)].copy()
pos = pos.astype(float) * tsmult
newdtx.append([pos[0], pos[1], np.min(scr)])
sys.stdout.write("\r {}: {:.4f}".format(tid, np.mean(scr)))
sys.stdout.flush()
newdtx = np.array(newdtx)
alldtx[idx] = newdtx
print()
for i, d in enumerate(data):
dxf = d["xf"]
dxf["tx"] += alldtx[i][0]
dxf["ty"] += alldtx[i][1]
dxf["score"] = alldtx[i][2]
d["xf"] = dxf
fm = "{}/aliptcls_{:02d}.lst".format(options.path, options.iter)
if os.path.isfile(fm): os.remove(fm)
lout = LSXFile(fm, False)
for i, d in enumerate(data):
lout.write(-1, d["srci"], d["src"], d["xf"])
lout = None
f = "{}/aliptcls_ts_{:02d}.hdf".format(options.path, options.iter)
if os.path.isfile(f): os.remove(f)
t = np.array([d["tsscr"] for d in data])
m = from_numpy(t).copy()
m.write_image(f)
#print(f, t.shape)
out = "{}/particle_parms_{:02d}.json".format(options.path, options.iter)
if os.path.isfile(out):
os.remove(out)
js = js_open_dict(out)
js.update(angs)
js.close()
del etc
E2end(logid)
def main():
parser = EMArgumentParser(usage="")
parser.add_argument("--output",
default="threed.hdf",
help="Output reconstructed volume file name.")
parser.add_argument(
"--input",
default=None,
help=
"The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion"
)
parser.add_argument(
"--sym",
dest="sym",
default="c1",
help=
"Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct."
)
parser.add_argument("--pad",
default=-1,
type=int,
help="Will zero-pad images to the specifed size. ")
parser.add_argument(
"--outsize",
default=-1,
type=int,
help="Defines the dimensions of the final volume written to disk")
parser.add_argument(
"--keep",
type=float,
dest="keep",
help=
"The fraction of slices to keep, based on quality scores (1.0 = use all slices). See keepsig.",
default=.9)
parser.add_argument(
"--no_wt",
action="store_true",
dest="no_wt",
default=False,
help=
"This argument turns automatic weighting off causing all images to be weighted by 1."
)
parser.add_argument(
"--mode",
type=str,
default="trilinear",
help=
"Fourier reconstruction 'mode' to use. The default should not normally be changed."
)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--apix",
metavar="A/pix",
type=float,
help="A/pix value for output, overrides automatic values",
default=None)
parser.add_argument("--ref", type=str, help="ref", default=None)
parser.add_argument("--tidrange",
type=str,
help="range of tilt id to include",
default="-1,-1")
parser.add_argument("--minres", type=float, help="", default=200)
parser.add_argument("--maxres", type=float, help="", default=5)
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default="thread:1")
parser.add_argument("--debug", action="store_true", default=False, help="")
parser.add_argument("--clsid",
default=-1,
type=int,
help="only reconstruct a class of particles")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
logger = E2init(sys.argv, options.ppid)
# get basic image parameters
tmp = EMData(options.input, 0, True)
boxsz = tmp["nx"]
if options.apix != None: apix = options.apix
else: apix = tmp["apix_x"]
if options.pad < 0:
options.pad = good_size(boxsz * 1.5)
if options.outsize < 0:
options.outsize = boxsz
options.tidrange = [int(i) for i in options.tidrange.split(',')]
if options.tidrange[0] >= 0:
print("including tilt ids from {} to {}".format(
options.tidrange[0], options.tidrange[1]))
data = initialize_data(options.input, options)
padvol = options.pad
from EMAN2PAR import EMTaskCustomer
if options.ref:
print("weighting by reference...")
ref = EMData(options.ref)
ref = ref.do_fft()
ref.process_inplace("xform.phaseorigin.tocenter")
ref.process_inplace("xform.fourierorigin.tocenter")
etc = EMTaskCustomer(options.parallel,
module="e2spa_make3d.WeightptclTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
tasks = [data[i::num_cpus] for i in range(num_cpus)]
print("{} jobs".format(len(tasks)))
tids = []
for t in tasks:
task = WeightptclTask(t, ref, options)
if options.debug:
task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
E2progress(logger, .5 * np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
wts = [0] * len(data)
for i in tids:
wt = etc.get_results(i)[1]
for w in wt:
try:
wts[w[0]] = w[1]
except:
print(len(wts), w)
wts = np.array(wts)
del etc
r0 = int(apix * boxsz / options.minres)
r1 = int(apix * boxsz / options.maxres)
print(r0, r1)
scrs = np.mean(wts[:, r0:r1], axis=1)
if options.keep < 1:
thr = np.sort(scrs)[int(len(scrs) * (1 - options.keep)) - 1]
scrs[scrs < thr] = -1
for i, d in enumerate(data):
d["curve"] = wts[i]
d["weight"] = float(scrs[i])
etc = EMTaskCustomer(options.parallel, module="e2spa_make3d.Make3dTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
tasks = [data[i::num_cpus] for i in range(num_cpus)]
print("{} jobs".format(len(tasks)))
tids = []
for t in tasks:
task = Make3dTask(t, options)
if options.debug:
task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
E2progress(logger, .5 + .5 * np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output = EMData(padvol, padvol, padvol)
normvol = EMData(padvol // 2 + 1, padvol, padvol)
output.to_zero()
output.do_fft_inplace()
normvol.to_zero()
for i in tids:
threed, norm = etc.get_results(i)[1]
threed.process_inplace("math.multamplitude", {"amp": norm})
output.add(threed)
normvol.add(norm)
normvol.process_inplace("math.reciprocal")
output.process_inplace("math.multamplitude", {"amp": normvol})
output.do_ift_inplace()
output.depad()
output.process_inplace("xform.phaseorigin.tocenter")
del etc
if options.verbose > 0: print("Finished Reconstruction")
output["apix_x"] = output["apix_y"] = output["apix_z"] = apix
sz = options.outsize
output.clip_inplace(
Region((padvol - sz) // 2, (padvol - sz) // 2, (padvol - sz) // 2, sz,
sz, sz))
if os.path.isfile(options.output):
os.remove(options.output)
output.write_image(options.output, 0)
if options.verbose > 0:
print("Output File: " + options.output)
E2end(logger)
print("Exiting")
def main():
parser = EMArgumentParser(usage=get_usage())
parser.add_argument("--output",
default="threed.hdf",
help="Output reconstructed volume file name.")
parser.add_argument(
"--input",
default=None,
help=
"The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion"
)
parser.add_argument(
"--input_model",
default=None,
help=
"If the class-averages have the model_id parameter (produced by e2refinemulti.py), this will use only class-averages with the specified model_id for the reconstruction."
)
parser.add_argument(
"--tlt",
help=
"An imod tlt file containing alignment angles. If specified slices will be inserted using these angles in the IMOD convention",
type=str,
default=None)
parser.add_argument(
"--sym",
dest="sym",
default="c1",
help=
"Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct."
)
parser.add_argument(
"--fillangle",
type=float,
dest="fillangle",
help=
"An angular range used for both alt & az over which the projection should be averaged. Generally the angular step used when making projections.",
default=0)
parser.add_argument(
"--pad",
metavar="x or x,y",
default=None,
type=str,
help=
"Will zero-pad images to the specifed size (x,y) or (x,x) prior to reconstruction. If not specified or 0 no padding occurs. If a negative value is specified automatic padding is performed. "
)
parser.add_argument(
"--padvol",
metavar="x or x,y,z",
default=None,
type=str,
help=
"Defines the dimensions (x,y,z) or (x,x,x) of the reconstructed volume. If ommitted, implied value based on padded 2D images is used."
)
parser.add_argument(
"--outsize",
metavar="x or x,y,z",
default=None,
type=str,
help=
"Defines the dimensions (x,y,z) or (x,x,x) of the final volume written to disk, if ommitted, size will be based on unpadded input size"
)
parser.add_argument(
"--compressbits",
type=int,
help=
"Bits to keep when writing volumes with compression. 0->lossless floating point. Default 10 (3 significant figures)",
default=10)
#parser.add_argument("--clipz", default=None, type=int, help="Extract a specified number of central z-slices. Option disabled by default.")
parser.add_argument(
"--savenorm",
default=None,
type=str,
help=
"If set, will save the normalization volume showing Fourier space filling to the specified file"
)
parser.add_argument(
"--keep",
type=float,
dest="keep",
help=
"The fraction of slices to keep, based on quality scores (1.0 = use all slices). See keepsig.",
default=1.0)
parser.add_argument(
"--keepsig",
action="store_true",
default=False,
dest="keepsig",
help=
"If set, keep will be interpreted as a standard deviation coefficient instead of as a percentage."
)
parser.add_argument(
"--keepabs",
action="store_true",
default=False,
dest="keepabs",
help=
"If set, keep will refer to the absolute quality of the class-average, not a local quality relative to other similar sized classes."
)
parser.add_argument(
"--altedgemask",
action="store_true",
default=False,
dest="altedgemask",
help=
"If set, assumes tomographic data with a thin specimen, and masks out the +-X edges based on the alt Euler angle."
)
parser.add_argument(
"--no_wt",
action="store_true",
dest="no_wt",
default=False,
help=
"This argument turns automatic weighting off causing all images to be weighted by 1. If this argument is not specified images inserted into the reconstructed volume are weighted by the number of particles that contributed to them (i.e. as in class averages), which is extracted from the image header (as the ptcl_repr attribute)."
)
parser.add_argument(
"--sqrt_wt",
action="store_true",
default=False,
help=
"Normally class-averages are weighted into the reconstruction based on the number of particles in the average. This option causes the sqrt of the number of particles to be used instead."
)
parser.add_argument(
"--iterative",
action="store_true",
default=False,
help=
"Uses iterative interpolation in Fourier space instead of single step gridding or interpolation. --mode and --usessnr are ignored with this option."
)
parser.add_argument(
"--itermask",
type=str,
default=None,
help=
"Used in iterative mode to define a mask to apply between iterations. The resulting reconstruction won't be masked per-se, but will be biased towards lower densities outside the mask."
)
parser.add_argument(
"--usessnr",
action="store_true",
default=False,
help=
"Makes use of the class_ssnr header data to weight each slice during insertion, instead of the default behavior of just using the number of particles in the average as a global weight."
)
parser.add_argument(
"--mode",
type=str,
default="gauss_var",
help=
"Fourier reconstruction 'mode' to use. The default should not normally be changed. default='gauss_var'"
)
parser.add_argument(
"--noradcor",
action="store_true",
default=False,
help=
"Normally a radial correction will be applied based on the --mode used. This option disables that correction."
)
parser.add_argument(
"--seedmap",
type=str,
default=None,
help=
"If specified this volume will be used as a starting point for the reconstruction, filling any missing values in Fourier space. experimental."
)
parser.add_argument(
"--seedweight",
type=float,
default=1.0,
help=
"If seedmap specified, this is how strongly the seedmap will bias existing values. 1 is default, and is equivalent to a one particle weight."
)
parser.add_argument(
"--seedweightmap",
type=str,
default=None,
help=
"Specify a full map of weights for the seed. This must be in the same format as the --savenorm output map."
)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--threads",
default=4,
type=int,
help=
"Number of threads to run in parallel on a single computer. This is the only parallelism supported by e2make3dpar",
guitype='intbox',
row=24,
col=2,
rowspan=1,
colspan=1,
mode="refinement")
parser.add_argument(
"--preprocess",
metavar="processor_name(param1=value1:param2=value2)",
type=str,
action="append",
help=
"preprocessor to be applied to the projections prior to 3D insertion. There can be more than one preprocessor and they are applied in the order in which they are specifed. Applied before padding occurs. See e2help.py processors for a complete list of available processors."
)
parser.add_argument(
"--setsf",
type=str,
help=
"Force the structure factor to match a 'known' curve prior to postprocessing (<filename>, auto or none). default=none",
default="none")
parser.add_argument(
"--postprocess",
metavar="processor_name(param1=value1:param2=value2)",
type=str,
action="append",
help=
"postprocessor to be applied to the 3D volume once the reconstruction is completed. There can be more than one postprocessor, and they are applied in the order in which they are specified. See e2help.py processors for a complete list of available processors."
)
parser.add_argument(
"--apix",
metavar="A/pix",
type=float,
help="A/pix value for output, overrides automatic values",
default=None)
parser.add_argument(
"--useseedasref",
action="store_true",
default=False,
help="use seed map as reference to weight the particles.")
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default=None)
# Database Metadata storage
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
# options.fillangle=options.fillangle*pi/180.0
# make sure that the user has atleast specified an input file
# if len(args) <1:
# parser.error("No input file specified")
# options.input = args[0]
if (not options.keep and not options.keepsig):
print(
"Warning, neither the keep nor the keepsig argument was specified. Setting keep=1 (keeping 100% of inserted slices)"
)
options.keep = 1
if options.input_model != None:
options.input_model = int(options.input_model)
print("e2make3dpar.py")
logger = E2init(sys.argv, options.ppid)
# update the no_wt parameter
if options.sqrt_wt: options.no_wt = 2
elif options.no_wt: options.no_wt = 1
else: options.no_wt = 0
# get basic image parameters
tmp = EMData()
tmp.read_image(options.input, 0, True)
if options.no_wt != 1:
try:
n = 1
while tmp["ptcl_repr"] == 0:
tmp.read_image(options.input, n, True)
n += 1
except:
pass
nx = tmp["nx"]
ny = tmp["ny"]
nslice = tmp["nz"]
if options.apix != None: apix = options.apix
else: apix = tmp["apix_x"]
if options.verbose > 0: print("Image dimensions %d x %d" % (nx, ny))
# parse the padding options, to make sure we have a 2 or 3 tuple for each
try:
if options.pad == None or options.pad == 0:
options.pad = (max(nx, ny), max(nx, ny))
elif options.pad[0] == '-':
sz = max(nx, ny)
sz = good_size(sz * 1.25)
options.pad = (sz, sz)
print("padding to {}x{}".format(sz, sz))
elif "," in options.pad:
s = options.pad.split(",")
options.pad = (int(s[0]), int(s[1]))
else:
options.pad = (int(options.pad), int(options.pad))
if options.verbose > 0:
print("Pad to %d x %d" % (options.pad[0], options.pad[1]))
except:
print("Couldn't parse pad option :", options.pad)
exit(1)
try:
if options.padvol == None:
padvol = (max(options.pad), max(options.pad), max(options.pad))
elif "," in options.padvol:
s = options.padvol.split(",")
padvol = (int(s[0]), int(s[1]), int(s[2]))
else:
padvol = (int(options.padvol), int(options.padvol),
int(options.padvol))
if options.verbose > 0:
print("Padded volume to reconstruct %d x %d x %d" %
(padvol[0], padvol[1], padvol[2]))
except:
print("Couldn't parse padvol option :", options.padvol)
exit(1)
try:
if options.outsize == None:
outsize = (max(nx, ny), max(nx, ny), max(nx, ny))
elif "," in options.outsize:
s = options.outsize.split(",")
outsize = (int(s[0]), int(s[1]), int(s[2]))
else:
outsize = (int(options.outsize), int(options.outsize),
int(options.outsize))
if options.verbose > 0:
print("Final output volume %d x %d x %d" %
(outsize[0], outsize[1], outsize[2]))
except:
print("Couldn't parse outsize option :", options.outsize)
exit(1)
data = initialize_data(options.input, options.input_model, options.tlt,
options.pad, options.no_wt, options.preprocess)
# Filter out averages/images which aren't good enough
if options.verbose:
print("Filtering data, %d images ->" % len(data), end=' ')
# we have an absolute threshold
if options.keepabs: thr = options.keep
else:
quals = np.array(sorted([i["quality"] for i in data]))
if np.std(quals) == 0: thr = max(quals) + 1.0
else:
if options.keepsig:
thr = np.mean(quals) + options.keep * np.std(quals)
else:
try:
thr = quals[int(floor(len(quals) * options.keep))]
except:
thr = max(quals) + 1.0
excluded = [
max(i["fileslice"], i["filenum"]) for i in data if i["quality"] > thr
]
included = [
max(i["fileslice"], i["filenum"]) for i in data if i["quality"] <= thr
]
data = [i for i in data if i["quality"] <= thr]
ptclcount = sum([i["weight"] for i in data])
if options.verbose: print("After filter, %d images" % len(data))
if options.itermask:
itmask = EMData(options.itermask)
itmask = itmask.get_clip(
Region(old_div((itmask["nx"] - options.pad[1]), 2),
old_div((itmask["ny"] - options.pad[1]), 2),
old_div((itmask["nz"] - options.pad[1]), 2), options.pad[1],
options.pad[1], options.pad[1]))
# Get the reconstructor and initialize it correctly
if options.iterative:
a = {
"size": padvol,
"sym": options.sym,
"verbose": options.verbose - 1
}
if options.savenorm != None: a["savenorm"] = options.savenorm
recon = Reconstructors.get("fourier_iter", a)
niter = 4
else:
a = {
"size": padvol,
"sym": options.sym,
"mode": options.mode,
"usessnr": options.usessnr,
"verbose": options.verbose - 1
}
if options.savenorm != None: a["savenorm"] = options.savenorm
recon = Reconstructors.get("fourier", a)
niter = 1
#########################################################
# The actual reconstruction
options.padvol3 = padvol
if options.parallel != None:
par = options.parallel.split(':')
if par[0].startswith("thread"):
#options.parallel=None
options.threads = int(par[1])
elif par[0] == "mpi":
nthr = int(par[1])
ppt = len(data) / nthr
if ppt > 16:
print("Inserting {:.1f} images per thread...".format(ppt))
### prepare some options
else:
print(
"Too few images in input ({:.1f} images per thread). Switching back to threading"
.format(ppt))
options.parallel = None
options.threads = nthr
else:
options.parallel = None
refmap = None
if options.seedmap != None:
seed = EMData(options.seedmap)
#seed.clip_inplace(Region(old_div((nx-padvol[0]),2),old_div((ny-padvol[1]),2),old_div((nslice-padvol[2]),2),padvol[0],padvol[1],padvol[2]))
seed = seed.get_clip(
Region((seed["nx"] - padvol[0]) // 2,
(seed["ny"] - padvol[1]) // 2,
(seed["nz"] - padvol[2]) // 2, padvol[0], padvol[1],
padvol[2]))
if options.useseedasref:
refmap = seed.copy()
refmap = refmap.do_fft()
refmap.process_inplace("xform.phaseorigin.tocenter")
refmap.process_inplace("xform.fourierorigin.tocenter")
#seed.process_inplace("threshold.belowtozero",{"minval":0})
seed.process_inplace("xform.phaseorigin.tocenter")
seed.do_fft_inplace()
else:
seed = None
for it in range(niter):
if options.parallel:
print(
"running in mpi mode. This is experimental, so please switch back to threading if anything goes wrong..."
)
if it > 0:
seed = output
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2make3dpar.Make3dTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
tasks = [data[i::num_cpus] for i in range(num_cpus)]
print("{} jobs".format(len(tasks)))
tids = []
for t in tasks:
task = Make3dTask(t, seed, refmap, options)
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
E2progress(logger, np.mean(st_vals) / 100.)
#print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
#print(tids)
if np.min(st_vals) == 100: break
time.sleep(5)
#dics=[0]*nptcl
output = EMData(padvol[0], padvol[1], padvol[2])
normvol = EMData(padvol[0] // 2 + 1, padvol[1], padvol[2])
output.to_zero()
output.do_fft_inplace()
normvol.to_zero()
for i in tids:
threed, norm = etc.get_results(i)[1]
threed.process_inplace("math.multamplitude", {"amp": norm})
output.add(threed)
normvol.add(norm)
normvol.process_inplace("math.reciprocal")
output.process_inplace("math.multamplitude", {"amp": normvol})
output.do_ift_inplace()
output.depad()
output.process_inplace("xform.phaseorigin.tocenter")
del etc
else:
threads = [
threading.Thread(target=reconstruct,
args=(data[i::options.threads], recon,
options.preprocess, options.pad,
options.fillangle, options.altedgemask,
max(options.verbose - 1, 0),
options.input.endswith(".lst"), refmap))
for i in range(options.threads)
]
if it == 0:
if options.seedmap != None:
if options.seedweightmap == None:
recon.setup_seed(seed, options.seedweight)
else:
seedweightmap = EMData(seedweightmap, 0)
recon.setup_seedandweights(seed, seedweightmap)
else:
recon.setup()
else:
recon.setup_seed(output, 1.0)
for i, t in enumerate(threads):
if options.verbose > 1: print("started thread ", i)
t.start()
for t in threads:
t.join()
# output = recon.finish(it==niter-1) # only return real-space on the final pass
output = recon.finish(True)
if options.verbose:
print("Iteration ", it)
if options.verbose > 2: output.write_image("it{}.hdf".format(it))
#if options.verbose>2 and it!=niter-1:
#output.process("xform.phaseorigin.tocenter").do_ift().write_image("it{}.hdf".format(it))
if it < niter - 1:
if options.itermask:
output.mult(itmask)
else:
output.process_inplace(
"threshold.compress", {
"range": output["sigma"],
"value": output["mean"],
"clamponly": True
})
if options.verbose:
print("Iteration ", it)
output.write_image("it{}.hdf".format(it))
#if options.verbose>2 and it!=niter-1:
#output.process("xform.phaseorigin.tocenter").do_ift().write_image("it{}.hdf".format(it))
if options.verbose > 0: print("Finished Reconstruction")
try:
output.set_attr("ptcl_repr", ptclcount)
if len(included) > 0: output.set_attr("threed_ptcl_idxs", included)
if len(excluded) > 0:
output.set_attr("threed_excl_ptcl_idxs", excluded)
output.set_attr("threed_ptcl_src", data[0]["filename"])
except:
print("Warning, error setting reconstruction attributes")
# progress += 10
# E2progress(logid,float(progress)/total_progress)
#
########################################################3
# Correct for Gaussian falloff
if not options.noradcor:
cor = {
"gauss_2": 4.0,
"gauss_3": 6.4,
"gauss_4": 8.8,
"gauss_5": 10.4,
"gauss_5_slow": 10.4
} # Gaussian widths for different reconstruction modes
if options.mode in cor:
output.process_inplace("math.gausskernelfix",
{"gauss_width": cor[options.mode]})
elif options.mode in ["gridding_5", "gridding_7"]:
output.process_inplace("math.gridkernelfix",
{"mode": options.mode})
else:
print("Warning: no radial correction applied for this mode")
# clip to the requested final dimensions
if output["nx"] != outsize[0] or output["ny"] != outsize[1] or output[
"nz"] != outsize[2]:
output.clip_inplace(
Region(old_div((output["nx"] - outsize[0]), 2),
old_div((output["ny"] - outsize[1]), 2),
old_div((output["nz"] - outsize[2]), 2), outsize[0],
outsize[1], outsize[2]))
#if options.clipz != None:
# output.clip_inplace(Region(0,0,(output["nz"]-options.clipz)//2, output["nx"], output["ny"], options.clipz))
if options.apix != None: apix = options.apix
output["apix_x"] = apix
output["apix_y"] = apix
output["apix_z"] = apix
# Structure factor setting
if options.setsf.lower() != "none":
if options.setsf.lower() != "auto":
try:
sfcurve = XYData()
sfcurve.read_file(options.setsf)
for i in range(sfcurve.get_size()):
v = sfcurve.get_y(i)
if v <= 0:
print(
"Warning values <=0 found in structure factor file. Please remove."
)
sfcurve.update()
except:
print(
"ERROR: Specified structure factor ({}) not found.".format(
options.setsf))
sys.exit(1)
else:
try:
sfcurve = XYData()
sfcurve.read_file("strucfac.txt")
for i in range(sfcurve.get_size()):
v = sfcurve.get_y(i)
if v <= 0:
print(
"Warning values <=0 found in structure factor file. Please remove."
)
sfcurve.update()
except:
sfcurve = None
if sfcurve == None:
print(
"ERROR : Structure factor read failed. Not applying structure factor"
)
else:
# need to be really careful about the corners
# for i in range(sfcurve.get_size()):
# if sfcurve.get_x(i)>1.0/(2.0*apix) : sfcurve.set_y(i,0.0)
output.process_inplace("filter.setstrucfac", {
"apix": apix,
"strucfac": sfcurve
})
# output.process_inplace("filter.lowpass.tophat",{"apix":apix,"cutoff_abs":0.49})
if options.postprocess != None:
for p in options.postprocess:
try:
(processorname, param_dict) = parsemodopt(p)
if not param_dict: param_dict = {}
output.process_inplace(str(processorname), param_dict)
except:
print("warning - application of the post processor", p,
" failed. Continuing anyway")
# output.process_inplace("normalize.circlemean")
# just remove the output file
if file_exists(options.output):
remove_file(options.output)
# write the reconstruction to disk
# output.write_image(options.output,0)
output.write_compressed(options.output, 0, options.compressbits)
if options.verbose > 0:
print("Output File: " + options.output)
E2end(logger)
print("Exiting")
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_pos_argument(
name="tomograms",
help=
"Specify tomograms from which you wish to extract boxed particles.",
default="",
guitype='filebox',
browser="EMTomoBoxesTable(withmodal=True,multiselect=True)",
row=0,
col=0,
rowspan=1,
colspan=2,
mode="extract")
parser.add_argument("--boxsz_unbin",
type=int,
help="box size in unbinned tomogram",
default=-1,
guitype='intbox',
row=2,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--label",
type=str,
help=
"Only extract particle with this name. Leave blank to extract all particles.",
default=None,
guitype='strbox',
row=2,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--newlabel",
type=str,
help=
"Label of output particles. Same as original particle label by default.",
default="",
guitype='strbox',
row=6,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock1",
help='Just a visual separation',
title="Options",
row=3,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--threads",
type=int,
help="threads",
default=12,
guitype='intbox',
row=4,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--maxtilt",
type=int,
help="max tilt",
default=100,
guitype='intbox',
row=4,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--padtwod",
type=float,
help="padding factor",
default=2.0,
guitype='floatbox',
row=5,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--noctf",
action="store_true",
default=False,
help="skip ctf correction.")
parser.add_argument(
"--wiener",
action="store_true",
default=False,
help="wiener filter the particles using ctf information..",
guitype='boolbox',
row=6,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--alltomograms",
action="store_true",
default=False,
help="use all tomograms.",
guitype='boolbox',
row=1,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--dotest",
action="store_true",
default=False,
help="only make 1 batch of subtomograms for testing")
parser.add_argument(
"--shrink",
type=float,
help=
"Shrinking factor for output particles. 1.5 or integers allowed. Default is 1 (no shrink).",
default=1,
guitype='floatbox',
row=8,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--tltkeep",
type=float,
help=
"keep a fraction of tilt images with good score determined from tomogram reconstruction",
default=1.0,
guitype='floatbox',
row=8,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--rmbeadthr",
type=float,
help=
"remove 2d particles with high contrast object beyond N sigma at 100A. Note that this may result in generating fewer particles than selected. Default is -1 (include all particles). 0.5 might be a good choice for removing gold beads but may need some testing...",
default=-1,
guitype='floatbox',
row=9,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock2",
help='Just a visual separation',
title="Extract from curves",
row=10,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--curves",
type=int,
default=-1,
help="specify curve id to extract particles from saved curves. ",
guitype='intbox',
row=11,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--curves_overlap",
type=float,
help=
"fraction of overlap when generating particle along curves. default is 0.5",
default=0.5,
guitype='floatbox',
row=12,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock3",
help='Just a visual separation',
title="Re-extraction from spt",
row=13,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--jsonali",
type=str,
help=
"re-extract particles using a particle_param_xx json file from a spt alignment",
default="",
guitype='filebox',
browser="EMBrowserWidget(withmodal=True,multiselect=False)",
row=14,
col=0,
rowspan=1,
colspan=2,
mode="extract")
parser.add_argument(
"--mindist",
type=float,
help="minimum distance between particles in A. for reextraction only",
default=10)
parser.add_argument(
"--keep",
type=float,
help=
"fraction of particles to keep fron previous alignment. for reextraction only.",
default=.9)
parser.add_argument("--postproc",
type=str,
help="processor after 3d particle reconstruction",
default="")
parser.add_argument(
"--postmask",
type=str,
help=
"masking after 3d particle reconstruction. The mask is transformed if json ",
default="")
parser.add_argument(
"--textin",
type=str,
help="text file for particle coordinates. do not use..",
default=None)
parser.add_argument(
"--compressbits",
type=int,
help=
"Bits to keep for compression. default is -1 meaning uncompressed floating point. 8 bit seems fine...",
default=-1,
guitype='intbox',
row=5,
col=1,
rowspan=1,
colspan=1,
mode="extract[8]")
# parser.add_argument("--saveint", action="store_true", default=False ,help="save particles in uint8 format to save space. still under testing.")
parser.add_argument("--norewrite",
action="store_true",
default=False,
help="skip existing files. do not rewrite.")
parser.add_argument("--parallel", type=str, help="parallel", default="")
#parser.add_argument("--alioffset", type=str,help="coordinate offset when re-extract particles. (x,y,z)", default="0,0,0", guitype='strbox', row=12, col=0,rowspan=1, colspan=1, mode="extract")
parser.add_argument(
"--postxf",
type=str,
help=
"a file listing post transforms (see http://eman2.org/e2tomo_more), or for simple symmetry, <sym>,<cx>,<cy>,<cz> where the coordinates specify the center of a single subunit",
default=None)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-2)
parser.add_argument(
"--skip3d",
action="store_true",
default=False,
help=
"do not make 3d particles. only generate 2d particles and 3d header. ")
#parser.add_argument("--shrink3d", type=int, help="Only shrink 3d particles by x factor",default=-1)
parser.add_argument("--verbose", type=int, help="verbose", default=1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
allxfs = {}
allinfo = {}
if options.textin:
allxfs, allinfo = parse_text(options)
if len(options.jsonali) > 0:
allxfs, allinfo = parse_json(options)
if options.alltomograms:
fld = "tomograms/"
args = [fld + f for f in os.listdir(fld) if f.endswith(".hdf")]
### do not count a tomogram twice when multiple versions exist
uq, uid = np.unique([base_name(a) for a in args], return_index=True)
args = [args[i] for i in uid]
if len(args) == 0 and len(allxfs) == 0:
print("No input. Exit.")
return
if options.parallel != "":
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_extract.SptExtractTask")
num_cpus = etc.cpu_est()
options.nowtime = time.time()
print(
"Running in parallel mode with {} workers, each running {} threads"
.format(num_cpus, options.threads))
njob = num_cpus
options.verbose = 0
if len(allxfs) > 0:
fnames = list(allxfs.keys())
elif len(args) > 0:
fnames = args
tids = []
for i in range(njob):
task = SptExtractTask(fnames[i::njob], options, allxfs, allinfo)
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if np.min(st_vals) == 100: break
#print(st_vals)
time.sleep(1)
else:
if len(allxfs) > 0:
for fname in allxfs.keys():
#### it seems options is changed inplace somewhere...
(options, args) = parser.parse_args()
do_extraction(fname, options, allxfs[fname], allinfo[fname])
elif len(args) > 0:
for a in args:
do_extraction(a, options)
return
print("Particle extraction finished.")
E2end(logid)
return
def main():
usage = """
Alternative to e2spt_align.py, but uses 2D subtilt images instead of 3D particle volume for alignment. Also a modified scipy minimizer is used in place of EMAN2 aligners.
e2spt_align_subtlt.py sets/ptcls.lst reference.hdf --path spt_xx --iter x
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
)
parser.add_argument("--iter",
type=int,
help="Iteration number within path. Default = 0",
default=0)
parser.add_argument(
"--goldcontinue",
action="store_true",
help=
"Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
default=False)
parser.add_argument(
"--sym",
type=str,
default="c1",
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly."
)
parser.add_argument(
"--maxres",
type=float,
help="Maximum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument(
"--minres",
type=float,
help="Minimum resolution to consider in alignment (in A, not 1/A)",
default=0)
#parser.add_argument("--maxtilt",type=float,help="Excluding tilt images beyond the angle",default=-1)
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default="thread:4")
parser.add_argument(
"--fromscratch",
action="store_true",
help=
"Start from exhaustive coarse alignment. Otherwise will use alignment from the previous rounds and do local search only.",
default=False)
parser.add_argument(
"--use3d",
action="store_true",
help="use projection of 3d particles instead of 2d sub tilt series",
default=False)
parser.add_argument(
"--preprocess",
metavar="processor_name:param1=value1:param2=value2",
type=str,
default=None,
help="Preprocess each 2-D subtilt while loading (alignment only)")
parser.add_argument(
"--debug",
action="store_true",
help=
"Debug mode. Will run a small number of particles directly without parallelism with lots of print out. ",
default=False)
parser.add_argument(
"--plst",
type=str,
default=None,
help=
"list of 2d particle with alignment parameters. The program will reconstruct before alignment so it can be slower."
)
parser.add_argument(
"--maxshift",
type=int,
help=
"Maximum shift from the center of the box or the previous alignment. default box size//6",
default=-1)
parser.add_argument(
"--maxang",
type=int,
help=
"Maximum angle difference from starting point. Ignored when --fromscratch is on.",
default=30)
parser.add_argument(
"--curve",
action="store_true",
help=
"Mode for filament structure refinement. Still under testing. Ignored when --fromscratch is on.",
default=False)
parser.add_argument(
"--skipali",
action="store_true",
help=
"Skip alignment and only calculate the score. Incompatible with --fromscratch, but --breaksym will still be considered.",
default=False)
parser.add_argument(
"--breaksym",
type=str,
default=None,
help="Specify symmetry to break. Ignored when --fromscratch is on.")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv, options.ppid)
options.ref = args[1]
#### we assume the particle_info files are in the right place so the bookkeeping is easier
## they should be created by e2spt_refine_new or e2spt_refine_multi_new
options.info3dname = "{}/particle_info_3d.lst".format(options.path)
options.info2dname = "{}/particle_info_2d.lst".format(options.path)
if options.preprocess != None:
options.preprocess = parsemodopt(options.preprocess)
#### this should generate a list of dictionaries (one dictionary per 3d particle)
tasks = load_lst_params(args[0])
#### keep a copy of the particle index here, so we can make sure the output list has the same order as the input list
for i, t in enumerate(tasks):
t["ii"] = i
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_align_subtlt.SptAlignTask")
num_cpus = etc.cpu_est()
options.nowtime = time.time()
#### only run a few particles for debugging
if options.debug:
tasks = tasks[:num_cpus * 4]
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob = num_cpus
#### send out jobs
tids = []
for i in range(njob):
t = tasks[i::njob]
task = SptAlignTask(t, options)
if options.debug:
ret = task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
#### each job returns a 3-object tuple:
## (index of 3d particle, 3d particle align info dict, list of dict for 2d particles align info)
output3d = [None] * len(tasks)
output2d = [None] * len(tasks)
for i in tids:
rets = etc.get_results(i)[1]
for r in rets:
output3d[r[0]] = r[1]
output2d[r[0]] = r[2]
del etc
#### merge the 2d particle alignment info into one list
output2d = sum(output2d, [])
#### just dump the list of dictionaries to list files
fm3d = f"{options.path}/aliptcls3d_{options.iter:02d}.lst"
save_lst_params(output3d, fm3d)
fm2d = f"{options.path}/aliptcls2d_{options.iter:02d}.lst"
save_lst_params(output2d, fm2d)
E2end(logid)
class EMParallelProject3D(object):
def __init__(self, options, fsp, sym, start, modeln=0, logger=None):
'''
@param options the options produced by (options, args) = parser.parse_args()
@param args the options produced by (options, args) = parser.parse_args()
@param logger and EMAN2 logger, i.e. logger=E2init(sys.argv)
assumes you have already called the check function.
'''
self.options = options
self.args = fsp
self.sym = sym
self.logger = logger
self.start = start
self.modeln = modeln
from EMAN2PAR import EMTaskCustomer
self.etc = EMTaskCustomer(options.parallel,
module="e2project3d.EMProject3DTaskDC")
print("Precache ", fsp)
self.etc.precache([fsp])
self.num_cpus = self.etc.cpu_est()
print(self.num_cpus, " total CPUs available")
if self.num_cpus > 64: # upper limit
self.num_cpus = 64
self.__task_options = None
def __init_memory(self, options):
'''
'''
sym_object = parsesym(self.sym)
[og_name, og_args] = parsemodopt(options.orientgen)
self.eulers = sym_object.gen_orientations(og_name, og_args)
def __get_task_options(self, options):
if self.__task_options == None:
d = {}
d["projector"] = parsemodopt(options.projector)
d["prethreshold"] = options.prethreshold
self.__task_options = d
return self.__task_options
def execute(self):
# from EMAN2PAR import EMTaskCustomer
if len(self.options.parallel) > 1:
self.__init_memory(self.options)
num_tasks = self.num_cpus
# In the worst case we can only spawn as many tasks as there are eulers
if self.num_cpus > len(self.eulers): num_tasks = len(self.eulers)
eulers_per_task = old_div(len(self.eulers), num_tasks)
resid_eulers = len(
self.eulers
) - eulers_per_task * num_tasks # we can distribute the residual evenly
first = 0
task_customers = []
tids = []
# self.etc=EMTaskCustomer(self.options.parallel)
for i in range(0, num_tasks):
last = first + eulers_per_task
if resid_eulers > 0:
last += 1
resid_eulers -= 1
tmp_eulers = self.eulers[first:last]
indices = list(range(first, last))
data = {}
data["input"] = ("cache", self.args, 0)
data["eulers"] = tmp_eulers
data["indices"] = indices
task = EMProject3DTaskDC(data=data,
options=self.__get_task_options(
self.options))
#print "Est %d CPUs"%etc.cpu_est()
tid = self.etc.send_task(task)
#print "Task submitted tid=",tid
tids.append(tid)
first = last
print("Task ids are", tids)
while 1:
print(len(tids), "projection tasks left in main loop")
st_vals = self.etc.check_task(tids)
for i in range(len(tids) - 1, -1, -1):
st = st_vals[i]
if st == 100:
tid = tids[i]
rslts = self.etc.get_results(tid)
if not self.__write_output_data(rslts[1]):
print("There was a problem with the task of id",
tid)
if self.logger != None:
E2progress(
self.logger,
1.0 - old_div(len(tids), float(num_tasks)))
if self.options.verbose > 0:
print("%d/%d\r" %
(num_tasks - len(tids), num_tasks))
sys.stdout.flush()
print("Task", tids.pop(i), "completed")
print("These tasks are remaining:", tids)
if len(tids) == 0: break
time.sleep(5)
return len(self.eulers)
else:
raise NotImplementedError(
"The parallelism option you specified (%s) is not suppored" %
self.options.parallel)
def __write_output_data(self, rslts):
for idx, image in list(rslts.items()):
if not isinstance(image, EMData):
continue # this is here because we get the dimensions of the database as a key (e.g. '40x40x1').
image["model_id"] = self.modeln
#if self.options.append : image.write_image(self.options.outfile,-1)
#else : image.write_image(self.options.outfile,idx+self.start)
if self.options.append:
image.write_compressed(self.options.outfile, -1,
self.options.compressbits)
else:
image.write_compressed(self.options.outfile, idx + self.start,
self.options.compressbits)
return True
def execute(self,alignment_jobs,files,caller):
'''
The main function
@param alignment_jobs a list of alignment pair indices like this [[0,1],[2,1],[2,3],[0,5],...] etc the indices pair represent images to be aligned and correspond to the order of the files argument
@param files a list of filenames - used to read image based on the indices present in alignment_jobs
@param caller - the calling object - it needs to have a function called process_output that takes a dictionary as the argument
'''
options = self.options
align_data = EMAN2.parsemodopt(options.align)
align_cmp_data = EMAN2.parsemodopt(options.aligncmp)
cmp_data = EMAN2.parsemodopt(options.cmp)
ralign_data = None
if options.ralign != None:
ralign_data = EMAN2.parsemodopt(options.ralign)
ralign_cmp_data = EMAN2.parsemodopt(options.raligncmp)
data = {}
data["align"] = align_data
data["aligncmp"] = align_cmp_data
data["cmp"] = cmp_data
if ralign_data:
data["ralign"] = ralign_data
data["raligncmp"] = ralign_cmp_data
data["using_cuda"] = self.using_cuda
data["nsoln"] = self.nsoln
if self.options.parallel :
task_customers = []
tids = []
if options.shrink:
scratch_name_1 = numbered_bdb("bdb:tomo_scratch#scratch_shrink")
scratch_name_2 = numbered_bdb("bdb:tomo_scratch##scratch_shrink")
else: print("no shrink")
for i,j in alignment_jobs:
if options.shrink or options.filter:
a = EMData(files[i],0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",{"n":options.shrink})
a.set_attr("src_image",files[i])
a.write_image(scratch_name_1,0)
a = EMData(files[j],0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",{"n":options.shrink})
a.set_attr("src_image",files[j])
a.write_image(scratch_name_2,0)
data["probe"] = ("cache",scratch_name_1,0)
data["target"] = ("cache",scratch_name_2,0)
else:
data["probe"] = ("cache",files[i],0)
data["target"] = ("cache",files[j],0)
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTaskDC(data=data)
from EMAN2PAR import EMTaskCustomer
etc=EMTaskCustomer(self.options.parallel)
#print "Est %d CPUs"%etc.cpu_est()
tid=etc.send_task(task)
#print "Task submitted tid=",tid
task_customers.append(etc)
tids.append(tid)
self.dc_monitor(task_customers,tids,caller)
else:
n = len(alignment_jobs)
p = 0.0
for i,j in alignment_jobs:
probe = EMData(files[i],0)
target = EMData(files[j],0)
if options.filter:
print("filtered")
filter_params = EMAN2.parsemodopt(options.filter)
probe.process_inplace(filter_params[0],filter_params[1])
target.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
probe.process_inplace("math.meanshrink",{"n":options.shrink})
target.process_inplace("math.meanshrink",{"n":options.shrink})
else:
print("no shrink")
data["target"] = target
data["probe"] = probe
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTask(data=data)
rslts = task.execute(self.progress_callback)
if options.shrink:
self.correction_translation(rslts,options.shrink)
caller.process_output(rslts)
p += 1.0
def main():
progname = os.path.basename(sys.argv[0])
usage = """
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--path",
type=str,
help=
"Path to a folder where results should be stored, following standard naming conventions",
default="subtlt_00")
parser.add_argument("--ref", type=str, help="reference map", default=None)
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=0)
parser.add_argument(
"--goldcontinue",
action="store_true",
help=
"Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
default=False)
parser.add_argument(
"--maxres",
type=float,
help="Maximum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument(
"--minres",
type=float,
help="Minimum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument(
"--smooth",
type=float,
help=
"smooth local motion by this factor. smoother local motion with larger numbers. default 100",
default=100)
parser.add_argument(
"--smoothN",
type=int,
help="number of neighboring particles used for smoothing. default 15",
default=15)
parser.add_argument("--maxshift",
type=float,
help="max shift in pixel. default default box size/6",
default=-1)
parser.add_argument("--refine_trans",
action="store_true",
help="do translational alignment.",
default=False)
parser.add_argument(
"--refine_rot",
action="store_true",
help=
"do translational-rotational alignment. better to start from an existing translational alignment.",
default=False)
parser.add_argument(
"--refine_defocus",
action="store_true",
help=
"do defocus refinement. need aliptcls input. doesn't work with refine_trans or rot yet..",
default=False)
parser.add_argument(
"--use3d",
action="store_true",
help="use projection of 3d particles instead of 2d ones..",
default=False)
parser.add_argument(
"--aliptcls2d",
type=str,
help=
"optional aliptcls input. the program can start search from the position from last run.",
default="")
parser.add_argument("--aliptcls3d",
type=str,
help="optional aliptcls input.",
default="")
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default="thread:4")
parser.add_argument("--debug",
action="store_true",
help="for testing.",
default=False)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv, options.ppid)
options.info2dname = "{}/particle_info_2d.lst".format(options.path)
options.info3dname = "{}/particle_info_3d.lst".format(options.path)
n = EMUtil.get_image_count(options.info2dname)
tasks = list(range(n))
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_subtlt_local.SptAlignTask")
num_cpus = etc.cpu_est()
if options.debug:
tasks = tasks[:num_cpus * 4]
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob = num_cpus
tids = []
for i in range(njob):
t = tasks[i::njob]
task = SptAlignTask(t, options)
if options.debug:
ret = task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output = [None] * len(tasks)
for i in tids:
rets = etc.get_results(i)[1]
for r in rets:
output[r[0]] = r[1]
del etc
fm = "{}/aliptcls2d_{:02d}.lst".format(options.path, options.iter)
save_lst_params(output, fm)
E2end(logid)
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--ptclin",
type=str,
help="particle input",
default=None)
parser.add_argument("--ptclout",
type=str,
help="particle output",
default=None)
parser.add_argument("--ref",
type=str,
help="reference input",
default=None)
parser.add_argument("--keep",
type=float,
help="propotion of tilts to keep. default is 0.8",
default=0.8)
parser.add_argument(
"--parallel",
type=str,
help="Thread/mpi parallelism to use. Default is thread:12",
default="thread:12")
parser.add_argument(
"--debug",
action="store_true",
default=False,
help=
"Turn on debug mode. This will only process a small subset of the data"
)
parser.add_argument("--maxshift",
type=int,
help="maximum shift allowed",
default=-1)
parser.add_argument("--localrefine",
action="store_true",
default=False,
help="local refinement")
#parser.add_argument("--ctfweight", action="store_true", default=False ,help="weight by ctf. not used yet...")
parser.add_argument("--slow",
action="store_true",
default=False,
help="slow but finer search")
parser.add_argument("--maxres",
type=float,
default=-1,
help="max resolution for cmp")
parser.add_argument("--minrespx",
type=int,
default=4,
help="skip the first x pixel in fourier space")
parser.add_argument("--sym", type=str, help="symmetry. ", default="c1")
parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
#parser.add_argument("--nkeep", type=int,help="", default=1)
parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
lstname = options.ptclin
threedname = options.ref
lname = options.ptclout
lst = LSXFile(lstname, True)
m = EMData(threedname)
bxsz = m["nx"]
apix = m["apix_x"]
options.shrink = 1
pinfo = []
nptcl = lst.n
if options.maxshift < 0:
options.maxshift = bxsz // 2
print("Initializing parallelism...")
etc = EMTaskCustomer(options.parallel, module="e2spa_align.SpaAlignTask")
num_cpus = etc.cpu_est()
print("{} particles".format(nptcl))
print("{} total CPUs available".format(num_cpus))
if options.debug:
nptcl = min(4 * num_cpus, nptcl)
print("Debugging mode. running on one thread with 8 particles")
for i in range(nptcl):
pinfo.append(lst.read(i))
lst = None
nbatch = min(nptcl // 4, num_cpus)
infos = [[] for i in range(nbatch)]
for i, info in enumerate(pinfo):
infos[i % nbatch].append([i, info])
print("{} jobs, each with {:.1f} particles".format(
len(infos), np.mean([len(i) for i in infos])))
tids = []
for info in infos:
task = SpaAlignTask(info, threedname, options)
if options.debug:
task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
dics = [0] * nptcl
for i in tids:
ret = etc.get_results(i)[1]
for r in ret:
ii = r.pop("idx")
dics[ii] = r
del etc
allscr = [d["score"] for d in dics]
maxl = np.max([len(s) for s in allscr])
maxv = np.max(np.concatenate(allscr))
for s in allscr:
s.extend([maxv] * (maxl - len(s)))
allscr = np.array(allscr)
try:
os.remove(lname)
except:
pass
lout = LSXFile(lname, False)
for i, dc in enumerate(dics):
lc = ""
for j, xf in enumerate(dc["xform.align3d"]):
d = xf.get_params("eman")
d["score"] = float(allscr[i, j])
lc = lc + str(d) + ';'
l = pinfo[i]
lout.write(-1, l[0], l[1], lc[:-1])
lout = None
E2end(logid)
