def execute(self,alignment_jobs,files,caller): ''' The main function @param alignment_jobs a list of alignment pair indices like this [[0,1],[2,1],[2,3],[0,5],...] etc the indices pair represent images to be aligned and correspond to the order of the files argument @param files a list of filenames - used to read image based on the indices present in alignment_jobs @param caller - the calling object - it needs to have a function called process_output that takes a dictionary as the argument ''' options = self.options align_data = EMAN2.parsemodopt(options.align) align_cmp_data = EMAN2.parsemodopt(options.aligncmp) cmp_data = EMAN2.parsemodopt(options.cmp) ralign_data = None if options.ralign != None: ralign_data = EMAN2.parsemodopt(options.ralign) ralign_cmp_data = EMAN2.parsemodopt(options.raligncmp) data = {} data["align"] = align_data data["aligncmp"] = align_cmp_data data["cmp"] = cmp_data if ralign_data: data["ralign"] = ralign_data data["raligncmp"] = ralign_cmp_data data["using_cuda"] = self.using_cuda data["nsoln"] = self.nsoln if self.options.parallel : task_customers = [] tids = [] if options.shrink: scratch_name_1 = numbered_bdb("bdb:tomo_scratch#scratch_shrink") scratch_name_2 = numbered_bdb("bdb:tomo_scratch##scratch_shrink") else: print "no shrink" for i,j in alignment_jobs: if options.shrink or options.filter: a = EMData(files[i],0) if options.filter: filter_params = EMAN2.parsemodopt(options.filter) a.process_inplace(filter_params[0],filter_params[1]) if options.shrink: a.process_inplace("math.meanshrink",{"n":options.shrink}) a.set_attr("src_image",files[i]) a.write_image(scratch_name_1,0) a = EMData(files[j],0) if options.filter: filter_params = EMAN2.parsemodopt(options.filter) a.process_inplace(filter_params[0],filter_params[1]) if options.shrink: a.process_inplace("math.meanshrink",{"n":options.shrink}) a.set_attr("src_image",files[j]) a.write_image(scratch_name_2,0) data["probe"] = ("cache",scratch_name_1,0) data["target"] = ("cache",scratch_name_2,0) else: data["probe"] = ("cache",files[i],0) data["target"] = ("cache",files[j],0) data["target_idx"] = j data["probe_idx"] = i task = EMTomoAlignTaskDC(data=data) from EMAN2PAR import EMTaskCustomer etc=EMTaskCustomer(self.options.parallel) #print "Est %d CPUs"%etc.cpu_est() tid=etc.send_task(task) #print "Task submitted tid=",tid task_customers.append(etc) tids.append(tid) self.dc_monitor(task_customers,tids,caller) else: n = len(alignment_jobs) p = 0.0 for i,j in alignment_jobs: probe = EMData(files[i],0) target = EMData(files[j],0) if options.filter: print "filtered" filter_params = EMAN2.parsemodopt(options.filter) probe.process_inplace(filter_params[0],filter_params[1]) target.process_inplace(filter_params[0],filter_params[1]) if options.shrink: probe.process_inplace("math.meanshrink",{"n":options.shrink}) target.process_inplace("math.meanshrink",{"n":options.shrink}) else: print "no shrink" data["target"] = target data["probe"] = probe data["target_idx"] = j data["probe_idx"] = i task = EMTomoAlignTask(data=data) rslts = task.execute(self.progress_callback) if options.shrink: self.correction_translation(rslts,options.shrink) caller.process_output(rslts) p += 1.0
class EMParallelProject3D: def __init__(self, options, fsp, sym, start, modeln=0, logger=None): """ @param options the options produced by (options, args) = parser.parse_args() @param args the options produced by (options, args) = parser.parse_args() @param logger and EMAN2 logger, i.e. logger=E2init(sys.argv) assumes you have already called the check function. """ self.options = options self.args = fsp self.sym = sym self.logger = logger self.start = start self.modeln = modeln from EMAN2PAR import EMTaskCustomer self.etc = EMTaskCustomer(options.parallel) print "Precache ", fsp self.etc.precache([fsp]) self.num_cpus = self.etc.cpu_est() print self.num_cpus, " total CPUs available" if self.num_cpus > 64: # upper limit self.num_cpus = 64 self.__task_options = None def __init_memory(self, options): """ """ sym_object = parsesym(self.sym) [og_name, og_args] = parsemodopt(options.orientgen) self.eulers = sym_object.gen_orientations(og_name, og_args) def __get_task_options(self, options): if self.__task_options == None: d = {} d["projector"] = parsemodopt(options.projector) d["prethreshold"] = options.prethreshold self.__task_options = d return self.__task_options def execute(self): # from EMAN2PAR import EMTaskCustomer if len(self.options.parallel) > 1: self.__init_memory(self.options) num_tasks = self.num_cpus # In the worst case we can only spawn as many tasks as there are eulers if self.num_cpus > len(self.eulers): num_tasks = len(self.eulers) eulers_per_task = len(self.eulers) / num_tasks resid_eulers = len(self.eulers) - eulers_per_task * num_tasks # we can distribute the residual evenly first = 0 task_customers = [] tids = [] # self.etc=EMTaskCustomer(self.options.parallel) for i in xrange(0, num_tasks): last = first + eulers_per_task if resid_eulers > 0: last += 1 resid_eulers -= 1 tmp_eulers = self.eulers[first:last] indices = range(first, last) data = {} data["input"] = ("cache", self.args, 0) data["eulers"] = tmp_eulers data["indices"] = indices task = EMProject3DTaskDC(data=data, options=self.__get_task_options(self.options)) # print "Est %d CPUs"%etc.cpu_est() tid = self.etc.send_task(task) # print "Task submitted tid=",tid tids.append(tid) first = last print "Task ids are", tids while 1: print len(tids), "projection tasks left in main loop" st_vals = self.etc.check_task(tids) for i in xrange(len(tids) - 1, -1, -1): st = st_vals[i] if st == 100: tid = tids[i] rslts = self.etc.get_results(tid) if not self.__write_output_data(rslts[1]): print "There was a problem with the task of id", tid if self.logger != None: E2progress(self.logger, 1.0 - len(tids) / float(num_tasks)) if self.options.verbose > 0: print "%d/%d\r" % (num_tasks - len(tids), num_tasks) sys.stdout.flush() print "Task", tids.pop(i), "completed" print "These tasks are remaining:", tids if len(tids) == 0: break time.sleep(5) return len(self.eulers) else: raise NotImplementedError( "The parallelism option you specified (%s) is not suppored" % self.options.parallel ) def __write_output_data(self, rslts): for idx, image in rslts.items(): if not isinstance(image, EMData): continue # this is here because we get the dimensions of the database as a key (e.g. '40x40x1'). image["model_id"] = self.modeln if self.options.append: image.write_image(self.options.outfile, -1) else: image.write_image(self.options.outfile, idx + self.start) return True
def main(): progname = os.path.basename(sys.argv[0]) usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference> This program is part of the 'new' hierarchy of e2spt_ programs. It performs one iteration of a classical subtomogram refinement, ie - aligning particles with missing wedge to a reference in 3-D The reference may be <volume> or <volume>,<n> """ parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_argument( "--threads", default=4, type=int, help= "Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.", guitype='intbox', row=24, col=2, rowspan=1, colspan=1, mode="refinement") parser.add_argument( "--iter", type=int, help="Iteration number within path. Default = start a new iteration", default=0) parser.add_argument( "--goldcontinue", action="store_true", help= "Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again", default=False) parser.add_argument( "--path", type=str, default=None, help= "Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)" ) parser.add_argument( "--sym", type=str, default="c1", help= "Symmetry of the input. Must be aligned in standard orientation to work properly." ) parser.add_argument( "--maxres", type=float, help="Maximum resolution to consider in alignment (in A, not 1/A)", default=0) parser.add_argument( "--minres", type=float, help="Minimum resolution to consider in alignment (in A, not 1/A)", default=0) parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help= "verbose level [0-9], higher number means higher level of verboseness") parser.add_argument( "--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-1) parser.add_argument("--maxshift", type=int, help="maximum shift for subtilt refine", default=8) parser.add_argument("--parallel", type=str, help="Thread/mpi parallelism to use", default=None) parser.add_argument("--fromscratch", action="store_true", help=".", default=False) parser.add_argument("--skipali3d", action="store_true", help="", default=False) parser.add_argument("--skipali2d", action="store_true", help="", default=False) parser.add_argument("--debug", action="store_true", help=".", default=False) parser.add_argument( "--plst", type=str, default=None, help= "list of 2d particle with alignment parameters. will reconstruct before alignment." ) parser.add_argument( "--smooth", type=float, help= "smooth local motion by this factor. smoother local motion with larger numbers", default=-1) parser.add_argument( "--smoothN", type=float, help="number of neighboring particles used for smoothing. default 15", default=15) (options, args) = parser.parse_args() if options.path == None: fls = [ int(i[-2:]) for i in os.listdir(".") if i[:4] == "spt_" and len(i) == 6 and str.isdigit(i[-2:]) ] if len(fls) == 0: fls = [0] options.path = "spt_{:02d}".format(max(fls) + 1) try: os.mkdir(options.path) except: pass if options.iter <= 0: fls = [ int(i[15:17]) for i in os.listdir(options.path) if i[:15] == "particle_parms_" and str.isdigit(i[15:17]) ] if len(fls) == 0: options.iter = 1 else: options.iter = max(fls) + 1 if options.parallel == None: options.parallel = "thread:{}".format(options.threads) # file may be "name" or "name,#" reffile = args[1].split(",")[0] try: refn = int(args[1].split(",")[1]) except: refn = 0 NTHREADS = max(options.threads + 1, 2) # we have one thread just writing results logid = E2init(sys.argv, options.ppid) refnames = [reffile, reffile] if options.goldcontinue: ref = [] try: refnames = [reffile[:-4] + "_even.hdf", reffile[:-4] + "_odd.hdf"] ref.append(EMData(refnames[0], 0)) ref.append(EMData(refnames[1], 0)) except: print("Error: cannot find one of reference files, eg: ", EMData(reffile[:-4] + "_even.hdf", 0)) n = -1 tasks = [] readjson = False if args[0].endswith(".lst") or args[0].endswith(".hdf"): nptcl = EMUtil.get_image_count(args[0]) tasks.extend([(args[0], i, refnames, i % 2) for i in range(nptcl)]) elif args[0].endswith(".json"): js = js_open_dict(args[0]) readjson = True jsinput = dict(js) keys = sorted(js.keys()) nptcl = len(keys) for k in keys: src, ii = eval(k) dic = js[k] xf = dic["xform.align3d"] tasks.append([src, ii, refnames, ii % 2, xf]) if options.plst: xinfo = [[] for i in range(nptcl)] xfkey = [ "type", "alt", "az", "phi", "tx", "ty", "tz", "alpha", "scale" ] lst = LSXFile(options.plst) for i in range(lst.n): l = lst.read(i) dc = eval(l[2]) pid = dc["pid"] dxf = Transform({k: dc[k] for k in dc.keys() if k in xfkey}) xinfo[pid].append([dc["tid"], i, dxf]) lst = None for i, xn in enumerate(xinfo): sid = np.argsort([x[0] for x in xn]) xinfo[i] = [[xn[s][1], xn[s][2]] for s in sid] for i in range(nptcl): ii = tasks[i][1] tasks[i].append(xinfo[ii]) from EMAN2PAR import EMTaskCustomer etc = EMTaskCustomer(options.parallel, module="e2spt_align_subtlt.SptAlignTask") num_cpus = etc.cpu_est() options.nowtime = time.time() if options.debug: tasks = tasks[:num_cpus * 4] print("{} jobs on {} CPUs".format(len(tasks), num_cpus)) njob = num_cpus tids = [] for i in range(njob): t = tasks[i::njob] task = SptAlignTask(t, options) if options.debug: ret = task.execute(print) return tid = etc.send_task(task) tids.append(tid) while 1: st_vals = etc.check_task(tids) if -100 in st_vals: print("Error occurs in parallelism. Exit") return E2progress(logid, np.mean(st_vals) / 100.) if np.min(st_vals) == 100: break time.sleep(5) output = [] for i in tids: rets = etc.get_results(i)[1] for ret in rets: fsp, n, dic = ret output.append([fsp, n, dic]) angs = {} data = [] for out in output: fsp, n, dc = out angs[(fsp, n)] = { "xform.align3d": dc["xform.align3d"], "score": dc["score"] } ts = dc.pop("transscr") for i, ii in enumerate(dc["imgidx"]): dxf = dc["imgxfs"][i].get_params("eman") #dxf["score"]=float(dc["imgscore"][i]) dxf["pid"] = n dxf["tid"] = i dxf["class"] = n % 2 tsmult = dc["tsmult"] data.append({ "src": dc["imgsrc"], "srci": ii, "pid": n, "tid": i, "xf": dxf, "coord": dc["coord"], "tsscr": ts[i] }) m = options.maxshift trans = np.indices((m * 2 + 1, m * 2 + 1)).reshape((2, -1)).T - m alldtx = np.zeros((len(data), 3)) fnames = np.unique([d["src"] for d in data]) maxtid = np.max([d["tid"] for d in data]) + 1 print("Smoothing trajectories...") for fname in fnames: print(fname) for tid in range(maxtid): idx = [ i for i, d in enumerate(data) if d["src"] == fname and d["tid"] == tid ] if len(idx) == 0: continue coord = np.array([data[i]["coord"] for i in idx]) scrs = np.array([data[i]["tsscr"] for i in idx]) if options.smooth <= 0: s = np.argmin(scrs, 1) sv = np.min(scrs, axis=1) newdtx = np.hstack([trans[s], sv[:, None]]) if tid == 0: print(" Skip smoothing") else: newdtx = [] for i, crd in enumerate(coord): dst = np.linalg.norm(coord - crd, axis=1) srtid = np.argsort(dst)[:options.smoothN] dst = dst[srtid] wt = np.exp(-dst / options.smooth) scr = scrs[srtid].copy() scr = np.sum(scr * wt[:, None], axis=0) pos = trans[np.argmin(scr)].copy() pos = pos.astype(float) * tsmult newdtx.append([pos[0], pos[1], np.min(scr)]) sys.stdout.write("\r {}: {:.4f}".format(tid, np.mean(scr))) sys.stdout.flush() newdtx = np.array(newdtx) alldtx[idx] = newdtx print() for i, d in enumerate(data): dxf = d["xf"] dxf["tx"] += alldtx[i][0] dxf["ty"] += alldtx[i][1] dxf["score"] = alldtx[i][2] d["xf"] = dxf fm = "{}/aliptcls_{:02d}.lst".format(options.path, options.iter) if os.path.isfile(fm): os.remove(fm) lout = LSXFile(fm, False) for i, d in enumerate(data): lout.write(-1, d["srci"], d["src"], d["xf"]) lout = None f = "{}/aliptcls_ts_{:02d}.hdf".format(options.path, options.iter) if os.path.isfile(f): os.remove(f) t = np.array([d["tsscr"] for d in data]) m = from_numpy(t).copy() m.write_image(f) #print(f, t.shape) out = "{}/particle_parms_{:02d}.json".format(options.path, options.iter) if os.path.isfile(out): os.remove(out) js = js_open_dict(out) js.update(angs) js.close() del etc E2end(logid)
def main(): parser = EMArgumentParser(usage="") parser.add_argument("--output", default="threed.hdf", help="Output reconstructed volume file name.") parser.add_argument( "--input", default=None, help= "The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion" ) parser.add_argument( "--sym", dest="sym", default="c1", help= "Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct." ) parser.add_argument("--pad", default=-1, type=int, help="Will zero-pad images to the specifed size. ") parser.add_argument( "--outsize", default=-1, type=int, help="Defines the dimensions of the final volume written to disk") parser.add_argument( "--keep", type=float, dest="keep", help= "The fraction of slices to keep, based on quality scores (1.0 = use all slices). See keepsig.", default=.9) parser.add_argument( "--no_wt", action="store_true", dest="no_wt", default=False, help= "This argument turns automatic weighting off causing all images to be weighted by 1." ) parser.add_argument( "--mode", type=str, default="trilinear", help= "Fourier reconstruction 'mode' to use. The default should not normally be changed." ) parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help= "verbose level [0-9], higher number means higher level of verboseness") parser.add_argument( "--apix", metavar="A/pix", type=float, help="A/pix value for output, overrides automatic values", default=None) parser.add_argument("--ref", type=str, help="ref", default=None) parser.add_argument("--tidrange", type=str, help="range of tilt id to include", default="-1,-1") parser.add_argument("--minres", type=float, help="", default=200) parser.add_argument("--maxres", type=float, help="", default=5) parser.add_argument("--parallel", type=str, help="Thread/mpi parallelism to use", default="thread:1") parser.add_argument("--debug", action="store_true", default=False, help="") parser.add_argument("--clsid", default=-1, type=int, help="only reconstruct a class of particles") parser.add_argument( "--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-1) (options, args) = parser.parse_args() logger = E2init(sys.argv, options.ppid) # get basic image parameters tmp = EMData(options.input, 0, True) boxsz = tmp["nx"] if options.apix != None: apix = options.apix else: apix = tmp["apix_x"] if options.pad < 0: options.pad = good_size(boxsz * 1.5) if options.outsize < 0: options.outsize = boxsz options.tidrange = [int(i) for i in options.tidrange.split(',')] if options.tidrange[0] >= 0: print("including tilt ids from {} to {}".format( options.tidrange[0], options.tidrange[1])) data = initialize_data(options.input, options) padvol = options.pad from EMAN2PAR import EMTaskCustomer if options.ref: print("weighting by reference...") ref = EMData(options.ref) ref = ref.do_fft() ref.process_inplace("xform.phaseorigin.tocenter") ref.process_inplace("xform.fourierorigin.tocenter") etc = EMTaskCustomer(options.parallel, module="e2spa_make3d.WeightptclTask") num_cpus = etc.cpu_est() print("{} total CPUs available".format(num_cpus)) tasks = [data[i::num_cpus] for i in range(num_cpus)] print("{} jobs".format(len(tasks))) tids = [] for t in tasks: task = WeightptclTask(t, ref, options) if options.debug: task.execute(print) return tid = etc.send_task(task) tids.append(tid) while 1: st_vals = etc.check_task(tids) E2progress(logger, .5 * np.mean(st_vals) / 100.) if np.min(st_vals) == 100: break time.sleep(5) wts = [0] * len(data) for i in tids: wt = etc.get_results(i)[1] for w in wt: try: wts[w[0]] = w[1] except: print(len(wts), w) wts = np.array(wts) del etc r0 = int(apix * boxsz / options.minres) r1 = int(apix * boxsz / options.maxres) print(r0, r1) scrs = np.mean(wts[:, r0:r1], axis=1) if options.keep < 1: thr = np.sort(scrs)[int(len(scrs) * (1 - options.keep)) - 1] scrs[scrs < thr] = -1 for i, d in enumerate(data): d["curve"] = wts[i] d["weight"] = float(scrs[i]) etc = EMTaskCustomer(options.parallel, module="e2spa_make3d.Make3dTask") num_cpus = etc.cpu_est() print("{} total CPUs available".format(num_cpus)) tasks = [data[i::num_cpus] for i in range(num_cpus)] print("{} jobs".format(len(tasks))) tids = [] for t in tasks: task = Make3dTask(t, options) if options.debug: task.execute(print) return tid = etc.send_task(task) tids.append(tid) while 1: st_vals = etc.check_task(tids) E2progress(logger, .5 + .5 * np.mean(st_vals) / 100.) if np.min(st_vals) == 100: break time.sleep(5) output = EMData(padvol, padvol, padvol) normvol = EMData(padvol // 2 + 1, padvol, padvol) output.to_zero() output.do_fft_inplace() normvol.to_zero() for i in tids: threed, norm = etc.get_results(i)[1] threed.process_inplace("math.multamplitude", {"amp": norm}) output.add(threed) normvol.add(norm) normvol.process_inplace("math.reciprocal") output.process_inplace("math.multamplitude", {"amp": normvol}) output.do_ift_inplace() output.depad() output.process_inplace("xform.phaseorigin.tocenter") del etc if options.verbose > 0: print("Finished Reconstruction") output["apix_x"] = output["apix_y"] = output["apix_z"] = apix sz = options.outsize output.clip_inplace( Region((padvol - sz) // 2, (padvol - sz) // 2, (padvol - sz) // 2, sz, sz, sz)) if os.path.isfile(options.output): os.remove(options.output) output.write_image(options.output, 0) if options.verbose > 0: print("Output File: " + options.output) E2end(logger) print("Exiting")
def main(): parser = EMArgumentParser(usage=get_usage()) parser.add_argument("--output", default="threed.hdf", help="Output reconstructed volume file name.") parser.add_argument( "--input", default=None, help= "The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion" ) parser.add_argument( "--input_model", default=None, help= "If the class-averages have the model_id parameter (produced by e2refinemulti.py), this will use only class-averages with the specified model_id for the reconstruction." ) parser.add_argument( "--tlt", help= "An imod tlt file containing alignment angles. If specified slices will be inserted using these angles in the IMOD convention", type=str, default=None) parser.add_argument( "--sym", dest="sym", default="c1", help= "Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct." ) parser.add_argument( "--fillangle", type=float, dest="fillangle", help= "An angular range used for both alt & az over which the projection should be averaged. Generally the angular step used when making projections.", default=0) parser.add_argument( "--pad", metavar="x or x,y", default=None, type=str, help= "Will zero-pad images to the specifed size (x,y) or (x,x) prior to reconstruction. If not specified or 0 no padding occurs. If a negative value is specified automatic padding is performed. " ) parser.add_argument( "--padvol", metavar="x or x,y,z", default=None, type=str, help= "Defines the dimensions (x,y,z) or (x,x,x) of the reconstructed volume. If ommitted, implied value based on padded 2D images is used." ) parser.add_argument( "--outsize", metavar="x or x,y,z", default=None, type=str, help= "Defines the dimensions (x,y,z) or (x,x,x) of the final volume written to disk, if ommitted, size will be based on unpadded input size" ) parser.add_argument( "--compressbits", type=int, help= "Bits to keep when writing volumes with compression. 0->lossless floating point. Default 10 (3 significant figures)", default=10) #parser.add_argument("--clipz", default=None, type=int, help="Extract a specified number of central z-slices. Option disabled by default.") parser.add_argument( "--savenorm", default=None, type=str, help= "If set, will save the normalization volume showing Fourier space filling to the specified file" ) parser.add_argument( "--keep", type=float, dest="keep", help= "The fraction of slices to keep, based on quality scores (1.0 = use all slices). See keepsig.", default=1.0) parser.add_argument( "--keepsig", action="store_true", default=False, dest="keepsig", help= "If set, keep will be interpreted as a standard deviation coefficient instead of as a percentage." ) parser.add_argument( "--keepabs", action="store_true", default=False, dest="keepabs", help= "If set, keep will refer to the absolute quality of the class-average, not a local quality relative to other similar sized classes." ) parser.add_argument( "--altedgemask", action="store_true", default=False, dest="altedgemask", help= "If set, assumes tomographic data with a thin specimen, and masks out the +-X edges based on the alt Euler angle." ) parser.add_argument( "--no_wt", action="store_true", dest="no_wt", default=False, help= "This argument turns automatic weighting off causing all images to be weighted by 1. If this argument is not specified images inserted into the reconstructed volume are weighted by the number of particles that contributed to them (i.e. as in class averages), which is extracted from the image header (as the ptcl_repr attribute)." ) parser.add_argument( "--sqrt_wt", action="store_true", default=False, help= "Normally class-averages are weighted into the reconstruction based on the number of particles in the average. This option causes the sqrt of the number of particles to be used instead." ) parser.add_argument( "--iterative", action="store_true", default=False, help= "Uses iterative interpolation in Fourier space instead of single step gridding or interpolation. --mode and --usessnr are ignored with this option." ) parser.add_argument( "--itermask", type=str, default=None, help= "Used in iterative mode to define a mask to apply between iterations. The resulting reconstruction won't be masked per-se, but will be biased towards lower densities outside the mask." ) parser.add_argument( "--usessnr", action="store_true", default=False, help= "Makes use of the class_ssnr header data to weight each slice during insertion, instead of the default behavior of just using the number of particles in the average as a global weight." ) parser.add_argument( "--mode", type=str, default="gauss_var", help= "Fourier reconstruction 'mode' to use. The default should not normally be changed. default='gauss_var'" ) parser.add_argument( "--noradcor", action="store_true", default=False, help= "Normally a radial correction will be applied based on the --mode used. This option disables that correction." ) parser.add_argument( "--seedmap", type=str, default=None, help= "If specified this volume will be used as a starting point for the reconstruction, filling any missing values in Fourier space. experimental." ) parser.add_argument( "--seedweight", type=float, default=1.0, help= "If seedmap specified, this is how strongly the seedmap will bias existing values. 1 is default, and is equivalent to a one particle weight." ) parser.add_argument( "--seedweightmap", type=str, default=None, help= "Specify a full map of weights for the seed. This must be in the same format as the --savenorm output map." ) parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help= "verbose level [0-9], higher number means higher level of verboseness") parser.add_argument( "--threads", default=4, type=int, help= "Number of threads to run in parallel on a single computer. This is the only parallelism supported by e2make3dpar", guitype='intbox', row=24, col=2, rowspan=1, colspan=1, mode="refinement") parser.add_argument( "--preprocess", metavar="processor_name(param1=value1:param2=value2)", type=str, action="append", help= "preprocessor to be applied to the projections prior to 3D insertion. There can be more than one preprocessor and they are applied in the order in which they are specifed. Applied before padding occurs. See e2help.py processors for a complete list of available processors." ) parser.add_argument( "--setsf", type=str, help= "Force the structure factor to match a 'known' curve prior to postprocessing (<filename>, auto or none). default=none", default="none") parser.add_argument( "--postprocess", metavar="processor_name(param1=value1:param2=value2)", type=str, action="append", help= "postprocessor to be applied to the 3D volume once the reconstruction is completed. There can be more than one postprocessor, and they are applied in the order in which they are specified. See e2help.py processors for a complete list of available processors." ) parser.add_argument( "--apix", metavar="A/pix", type=float, help="A/pix value for output, overrides automatic values", default=None) parser.add_argument( "--useseedasref", action="store_true", default=False, help="use seed map as reference to weight the particles.") parser.add_argument("--parallel", type=str, help="Thread/mpi parallelism to use", default=None) # Database Metadata storage parser.add_argument( "--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-1) (options, args) = parser.parse_args() # options.fillangle=options.fillangle*pi/180.0 # make sure that the user has atleast specified an input file # if len(args) <1: # parser.error("No input file specified") # options.input = args[0] if (not options.keep and not options.keepsig): print( "Warning, neither the keep nor the keepsig argument was specified. Setting keep=1 (keeping 100% of inserted slices)" ) options.keep = 1 if options.input_model != None: options.input_model = int(options.input_model) print("e2make3dpar.py") logger = E2init(sys.argv, options.ppid) # update the no_wt parameter if options.sqrt_wt: options.no_wt = 2 elif options.no_wt: options.no_wt = 1 else: options.no_wt = 0 # get basic image parameters tmp = EMData() tmp.read_image(options.input, 0, True) if options.no_wt != 1: try: n = 1 while tmp["ptcl_repr"] == 0: tmp.read_image(options.input, n, True) n += 1 except: pass nx = tmp["nx"] ny = tmp["ny"] nslice = tmp["nz"] if options.apix != None: apix = options.apix else: apix = tmp["apix_x"] if options.verbose > 0: print("Image dimensions %d x %d" % (nx, ny)) # parse the padding options, to make sure we have a 2 or 3 tuple for each try: if options.pad == None or options.pad == 0: options.pad = (max(nx, ny), max(nx, ny)) elif options.pad[0] == '-': sz = max(nx, ny) sz = good_size(sz * 1.25) options.pad = (sz, sz) print("padding to {}x{}".format(sz, sz)) elif "," in options.pad: s = options.pad.split(",") options.pad = (int(s[0]), int(s[1])) else: options.pad = (int(options.pad), int(options.pad)) if options.verbose > 0: print("Pad to %d x %d" % (options.pad[0], options.pad[1])) except: print("Couldn't parse pad option :", options.pad) exit(1) try: if options.padvol == None: padvol = (max(options.pad), max(options.pad), max(options.pad)) elif "," in options.padvol: s = options.padvol.split(",") padvol = (int(s[0]), int(s[1]), int(s[2])) else: padvol = (int(options.padvol), int(options.padvol), int(options.padvol)) if options.verbose > 0: print("Padded volume to reconstruct %d x %d x %d" % (padvol[0], padvol[1], padvol[2])) except: print("Couldn't parse padvol option :", options.padvol) exit(1) try: if options.outsize == None: outsize = (max(nx, ny), max(nx, ny), max(nx, ny)) elif "," in options.outsize: s = options.outsize.split(",") outsize = (int(s[0]), int(s[1]), int(s[2])) else: outsize = (int(options.outsize), int(options.outsize), int(options.outsize)) if options.verbose > 0: print("Final output volume %d x %d x %d" % (outsize[0], outsize[1], outsize[2])) except: print("Couldn't parse outsize option :", options.outsize) exit(1) data = initialize_data(options.input, options.input_model, options.tlt, options.pad, options.no_wt, options.preprocess) # Filter out averages/images which aren't good enough if options.verbose: print("Filtering data, %d images ->" % len(data), end=' ') # we have an absolute threshold if options.keepabs: thr = options.keep else: quals = np.array(sorted([i["quality"] for i in data])) if np.std(quals) == 0: thr = max(quals) + 1.0 else: if options.keepsig: thr = np.mean(quals) + options.keep * np.std(quals) else: try: thr = quals[int(floor(len(quals) * options.keep))] except: thr = max(quals) + 1.0 excluded = [ max(i["fileslice"], i["filenum"]) for i in data if i["quality"] > thr ] included = [ max(i["fileslice"], i["filenum"]) for i in data if i["quality"] <= thr ] data = [i for i in data if i["quality"] <= thr] ptclcount = sum([i["weight"] for i in data]) if options.verbose: print("After filter, %d images" % len(data)) if options.itermask: itmask = EMData(options.itermask) itmask = itmask.get_clip( Region(old_div((itmask["nx"] - options.pad[1]), 2), old_div((itmask["ny"] - options.pad[1]), 2), old_div((itmask["nz"] - options.pad[1]), 2), options.pad[1], options.pad[1], options.pad[1])) # Get the reconstructor and initialize it correctly if options.iterative: a = { "size": padvol, "sym": options.sym, "verbose": options.verbose - 1 } if options.savenorm != None: a["savenorm"] = options.savenorm recon = Reconstructors.get("fourier_iter", a) niter = 4 else: a = { "size": padvol, "sym": options.sym, "mode": options.mode, "usessnr": options.usessnr, "verbose": options.verbose - 1 } if options.savenorm != None: a["savenorm"] = options.savenorm recon = Reconstructors.get("fourier", a) niter = 1 ######################################################### # The actual reconstruction options.padvol3 = padvol if options.parallel != None: par = options.parallel.split(':') if par[0].startswith("thread"): #options.parallel=None options.threads = int(par[1]) elif par[0] == "mpi": nthr = int(par[1]) ppt = len(data) / nthr if ppt > 16: print("Inserting {:.1f} images per thread...".format(ppt)) ### prepare some options else: print( "Too few images in input ({:.1f} images per thread). Switching back to threading" .format(ppt)) options.parallel = None options.threads = nthr else: options.parallel = None refmap = None if options.seedmap != None: seed = EMData(options.seedmap) #seed.clip_inplace(Region(old_div((nx-padvol[0]),2),old_div((ny-padvol[1]),2),old_div((nslice-padvol[2]),2),padvol[0],padvol[1],padvol[2])) seed = seed.get_clip( Region((seed["nx"] - padvol[0]) // 2, (seed["ny"] - padvol[1]) // 2, (seed["nz"] - padvol[2]) // 2, padvol[0], padvol[1], padvol[2])) if options.useseedasref: refmap = seed.copy() refmap = refmap.do_fft() refmap.process_inplace("xform.phaseorigin.tocenter") refmap.process_inplace("xform.fourierorigin.tocenter") #seed.process_inplace("threshold.belowtozero",{"minval":0}) seed.process_inplace("xform.phaseorigin.tocenter") seed.do_fft_inplace() else: seed = None for it in range(niter): if options.parallel: print( "running in mpi mode. This is experimental, so please switch back to threading if anything goes wrong..." ) if it > 0: seed = output from EMAN2PAR import EMTaskCustomer etc = EMTaskCustomer(options.parallel, module="e2make3dpar.Make3dTask") num_cpus = etc.cpu_est() print("{} total CPUs available".format(num_cpus)) tasks = [data[i::num_cpus] for i in range(num_cpus)] print("{} jobs".format(len(tasks))) tids = [] for t in tasks: task = Make3dTask(t, seed, refmap, options) tid = etc.send_task(task) tids.append(tid) while 1: st_vals = etc.check_task(tids) E2progress(logger, np.mean(st_vals) / 100.) #print("{:.1f}/{} finished".format(np.mean(st_vals), 100)) #print(tids) if np.min(st_vals) == 100: break time.sleep(5) #dics=[0]*nptcl output = EMData(padvol[0], padvol[1], padvol[2]) normvol = EMData(padvol[0] // 2 + 1, padvol[1], padvol[2]) output.to_zero() output.do_fft_inplace() normvol.to_zero() for i in tids: threed, norm = etc.get_results(i)[1] threed.process_inplace("math.multamplitude", {"amp": norm}) output.add(threed) normvol.add(norm) normvol.process_inplace("math.reciprocal") output.process_inplace("math.multamplitude", {"amp": normvol}) output.do_ift_inplace() output.depad() output.process_inplace("xform.phaseorigin.tocenter") del etc else: threads = [ threading.Thread(target=reconstruct, args=(data[i::options.threads], recon, options.preprocess, options.pad, options.fillangle, options.altedgemask, max(options.verbose - 1, 0), options.input.endswith(".lst"), refmap)) for i in range(options.threads) ] if it == 0: if options.seedmap != None: if options.seedweightmap == None: recon.setup_seed(seed, options.seedweight) else: seedweightmap = EMData(seedweightmap, 0) recon.setup_seedandweights(seed, seedweightmap) else: recon.setup() else: recon.setup_seed(output, 1.0) for i, t in enumerate(threads): if options.verbose > 1: print("started thread ", i) t.start() for t in threads: t.join() # output = recon.finish(it==niter-1) # only return real-space on the final pass output = recon.finish(True) if options.verbose: print("Iteration ", it) if options.verbose > 2: output.write_image("it{}.hdf".format(it)) #if options.verbose>2 and it!=niter-1: #output.process("xform.phaseorigin.tocenter").do_ift().write_image("it{}.hdf".format(it)) if it < niter - 1: if options.itermask: output.mult(itmask) else: output.process_inplace( "threshold.compress", { "range": output["sigma"], "value": output["mean"], "clamponly": True }) if options.verbose: print("Iteration ", it) output.write_image("it{}.hdf".format(it)) #if options.verbose>2 and it!=niter-1: #output.process("xform.phaseorigin.tocenter").do_ift().write_image("it{}.hdf".format(it)) if options.verbose > 0: print("Finished Reconstruction") try: output.set_attr("ptcl_repr", ptclcount) if len(included) > 0: output.set_attr("threed_ptcl_idxs", included) if len(excluded) > 0: output.set_attr("threed_excl_ptcl_idxs", excluded) output.set_attr("threed_ptcl_src", data[0]["filename"]) except: print("Warning, error setting reconstruction attributes") # progress += 10 # E2progress(logid,float(progress)/total_progress) # ########################################################3 # Correct for Gaussian falloff if not options.noradcor: cor = { "gauss_2": 4.0, "gauss_3": 6.4, "gauss_4": 8.8, "gauss_5": 10.4, "gauss_5_slow": 10.4 } # Gaussian widths for different reconstruction modes if options.mode in cor: output.process_inplace("math.gausskernelfix", {"gauss_width": cor[options.mode]}) elif options.mode in ["gridding_5", "gridding_7"]: output.process_inplace("math.gridkernelfix", {"mode": options.mode}) else: print("Warning: no radial correction applied for this mode") # clip to the requested final dimensions if output["nx"] != outsize[0] or output["ny"] != outsize[1] or output[ "nz"] != outsize[2]: output.clip_inplace( Region(old_div((output["nx"] - outsize[0]), 2), old_div((output["ny"] - outsize[1]), 2), old_div((output["nz"] - outsize[2]), 2), outsize[0], outsize[1], outsize[2])) #if options.clipz != None: # output.clip_inplace(Region(0,0,(output["nz"]-options.clipz)//2, output["nx"], output["ny"], options.clipz)) if options.apix != None: apix = options.apix output["apix_x"] = apix output["apix_y"] = apix output["apix_z"] = apix # Structure factor setting if options.setsf.lower() != "none": if options.setsf.lower() != "auto": try: sfcurve = XYData() sfcurve.read_file(options.setsf) for i in range(sfcurve.get_size()): v = sfcurve.get_y(i) if v <= 0: print( "Warning values <=0 found in structure factor file. Please remove." ) sfcurve.update() except: print( "ERROR: Specified structure factor ({}) not found.".format( options.setsf)) sys.exit(1) else: try: sfcurve = XYData() sfcurve.read_file("strucfac.txt") for i in range(sfcurve.get_size()): v = sfcurve.get_y(i) if v <= 0: print( "Warning values <=0 found in structure factor file. Please remove." ) sfcurve.update() except: sfcurve = None if sfcurve == None: print( "ERROR : Structure factor read failed. Not applying structure factor" ) else: # need to be really careful about the corners # for i in range(sfcurve.get_size()): # if sfcurve.get_x(i)>1.0/(2.0*apix) : sfcurve.set_y(i,0.0) output.process_inplace("filter.setstrucfac", { "apix": apix, "strucfac": sfcurve }) # output.process_inplace("filter.lowpass.tophat",{"apix":apix,"cutoff_abs":0.49}) if options.postprocess != None: for p in options.postprocess: try: (processorname, param_dict) = parsemodopt(p) if not param_dict: param_dict = {} output.process_inplace(str(processorname), param_dict) except: print("warning - application of the post processor", p, " failed. Continuing anyway") # output.process_inplace("normalize.circlemean") # just remove the output file if file_exists(options.output): remove_file(options.output) # write the reconstruction to disk # output.write_image(options.output,0) output.write_compressed(options.output, 0, options.compressbits) if options.verbose > 0: print("Output File: " + options.output) E2end(logger) print("Exiting")
def main(): usage = " " parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_pos_argument( name="tomograms", help= "Specify tomograms from which you wish to extract boxed particles.", default="", guitype='filebox', browser="EMTomoBoxesTable(withmodal=True,multiselect=True)", row=0, col=0, rowspan=1, colspan=2, mode="extract") parser.add_argument("--boxsz_unbin", type=int, help="box size in unbinned tomogram", default=-1, guitype='intbox', row=2, col=0, rowspan=1, colspan=1, mode="extract") parser.add_argument( "--label", type=str, help= "Only extract particle with this name. Leave blank to extract all particles.", default=None, guitype='strbox', row=2, col=1, rowspan=1, colspan=1, mode="extract") parser.add_argument( "--newlabel", type=str, help= "Label of output particles. Same as original particle label by default.", default="", guitype='strbox', row=6, col=0, rowspan=1, colspan=1, mode="extract") parser.add_header(name="orblock1", help='Just a visual separation', title="Options", row=3, col=0, rowspan=1, colspan=1, mode="extract") parser.add_argument("--threads", type=int, help="threads", default=12, guitype='intbox', row=4, col=1, rowspan=1, colspan=1, mode="extract") parser.add_argument("--maxtilt", type=int, help="max tilt", default=100, guitype='intbox', row=4, col=0, rowspan=1, colspan=1, mode="extract") parser.add_argument("--padtwod", type=float, help="padding factor", default=2.0, guitype='floatbox', row=5, col=0, rowspan=1, colspan=1, mode="extract") parser.add_argument("--noctf", action="store_true", default=False, help="skip ctf correction.") parser.add_argument( "--wiener", action="store_true", default=False, help="wiener filter the particles using ctf information..", guitype='boolbox', row=6, col=1, rowspan=1, colspan=1, mode="extract") parser.add_argument("--alltomograms", action="store_true", default=False, help="use all tomograms.", guitype='boolbox', row=1, col=1, rowspan=1, colspan=1, mode="extract") parser.add_argument("--dotest", action="store_true", default=False, help="only make 1 batch of subtomograms for testing") parser.add_argument( "--shrink", type=float, help= "Shrinking factor for output particles. 1.5 or integers allowed. Default is 1 (no shrink).", default=1, guitype='floatbox', row=8, col=0, rowspan=1, colspan=1, mode="extract") parser.add_argument( "--tltkeep", type=float, help= "keep a fraction of tilt images with good score determined from tomogram reconstruction", default=1.0, guitype='floatbox', row=8, col=1, rowspan=1, colspan=1, mode="extract") parser.add_argument( "--rmbeadthr", type=float, help= "remove 2d particles with high contrast object beyond N sigma at 100A. Note that this may result in generating fewer particles than selected. Default is -1 (include all particles). 0.5 might be a good choice for removing gold beads but may need some testing...", default=-1, guitype='floatbox', row=9, col=0, rowspan=1, colspan=1, mode="extract") parser.add_header(name="orblock2", help='Just a visual separation', title="Extract from curves", row=10, col=0, rowspan=1, colspan=1, mode="extract") parser.add_argument( "--curves", type=int, default=-1, help="specify curve id to extract particles from saved curves. ", guitype='intbox', row=11, col=0, rowspan=1, colspan=1, mode="extract") parser.add_argument( "--curves_overlap", type=float, help= "fraction of overlap when generating particle along curves. default is 0.5", default=0.5, guitype='floatbox', row=12, col=0, rowspan=1, colspan=1, mode="extract") parser.add_header(name="orblock3", help='Just a visual separation', title="Re-extraction from spt", row=13, col=0, rowspan=1, colspan=1, mode="extract") parser.add_argument( "--jsonali", type=str, help= "re-extract particles using a particle_param_xx json file from a spt alignment", default="", guitype='filebox', browser="EMBrowserWidget(withmodal=True,multiselect=False)", row=14, col=0, rowspan=1, colspan=2, mode="extract") parser.add_argument( "--mindist", type=float, help="minimum distance between particles in A. for reextraction only", default=10) parser.add_argument( "--keep", type=float, help= "fraction of particles to keep fron previous alignment. for reextraction only.", default=.9) parser.add_argument("--postproc", type=str, help="processor after 3d particle reconstruction", default="") parser.add_argument( "--postmask", type=str, help= "masking after 3d particle reconstruction. The mask is transformed if json ", default="") parser.add_argument( "--textin", type=str, help="text file for particle coordinates. do not use..", default=None) parser.add_argument( "--compressbits", type=int, help= "Bits to keep for compression. default is -1 meaning uncompressed floating point. 8 bit seems fine...", default=-1, guitype='intbox', row=5, col=1, rowspan=1, colspan=1, mode="extract[8]") # parser.add_argument("--saveint", action="store_true", default=False ,help="save particles in uint8 format to save space. still under testing.") parser.add_argument("--norewrite", action="store_true", default=False, help="skip existing files. do not rewrite.") parser.add_argument("--parallel", type=str, help="parallel", default="") #parser.add_argument("--alioffset", type=str,help="coordinate offset when re-extract particles. (x,y,z)", default="0,0,0", guitype='strbox', row=12, col=0,rowspan=1, colspan=1, mode="extract") parser.add_argument( "--postxf", type=str, help= "a file listing post transforms (see http://eman2.org/e2tomo_more), or for simple symmetry, <sym>,<cx>,<cy>,<cz> where the coordinates specify the center of a single subunit", default=None) parser.add_argument( "--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-2) parser.add_argument( "--skip3d", action="store_true", default=False, help= "do not make 3d particles. only generate 2d particles and 3d header. ") #parser.add_argument("--shrink3d", type=int, help="Only shrink 3d particles by x factor",default=-1) parser.add_argument("--verbose", type=int, help="verbose", default=1) (options, args) = parser.parse_args() logid = E2init(sys.argv) allxfs = {} allinfo = {} if options.textin: allxfs, allinfo = parse_text(options) if len(options.jsonali) > 0: allxfs, allinfo = parse_json(options) if options.alltomograms: fld = "tomograms/" args = [fld + f for f in os.listdir(fld) if f.endswith(".hdf")] ### do not count a tomogram twice when multiple versions exist uq, uid = np.unique([base_name(a) for a in args], return_index=True) args = [args[i] for i in uid] if len(args) == 0 and len(allxfs) == 0: print("No input. Exit.") return if options.parallel != "": from EMAN2PAR import EMTaskCustomer etc = EMTaskCustomer(options.parallel, module="e2spt_extract.SptExtractTask") num_cpus = etc.cpu_est() options.nowtime = time.time() print( "Running in parallel mode with {} workers, each running {} threads" .format(num_cpus, options.threads)) njob = num_cpus options.verbose = 0 if len(allxfs) > 0: fnames = list(allxfs.keys()) elif len(args) > 0: fnames = args tids = [] for i in range(njob): task = SptExtractTask(fnames[i::njob], options, allxfs, allinfo) tid = etc.send_task(task) tids.append(tid) while 1: st_vals = etc.check_task(tids) if np.min(st_vals) == 100: break #print(st_vals) time.sleep(1) else: if len(allxfs) > 0: for fname in allxfs.keys(): #### it seems options is changed inplace somewhere... (options, args) = parser.parse_args() do_extraction(fname, options, allxfs[fname], allinfo[fname]) elif len(args) > 0: for a in args: do_extraction(a, options) return print("Particle extraction finished.") E2end(logid) return
def main(): usage = """ Alternative to e2spt_align.py, but uses 2D subtilt images instead of 3D particle volume for alignment. Also a modified scipy minimizer is used in place of EMAN2 aligners. e2spt_align_subtlt.py sets/ptcls.lst reference.hdf --path spt_xx --iter x """ parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_argument( "--path", type=str, default=None, help= "Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)" ) parser.add_argument("--iter", type=int, help="Iteration number within path. Default = 0", default=0) parser.add_argument( "--goldcontinue", action="store_true", help= "Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again", default=False) parser.add_argument( "--sym", type=str, default="c1", help= "Symmetry of the input. Must be aligned in standard orientation to work properly." ) parser.add_argument( "--maxres", type=float, help="Maximum resolution to consider in alignment (in A, not 1/A)", default=0) parser.add_argument( "--minres", type=float, help="Minimum resolution to consider in alignment (in A, not 1/A)", default=0) #parser.add_argument("--maxtilt",type=float,help="Excluding tilt images beyond the angle",default=-1) parser.add_argument("--parallel", type=str, help="Thread/mpi parallelism to use", default="thread:4") parser.add_argument( "--fromscratch", action="store_true", help= "Start from exhaustive coarse alignment. Otherwise will use alignment from the previous rounds and do local search only.", default=False) parser.add_argument( "--use3d", action="store_true", help="use projection of 3d particles instead of 2d sub tilt series", default=False) parser.add_argument( "--preprocess", metavar="processor_name:param1=value1:param2=value2", type=str, default=None, help="Preprocess each 2-D subtilt while loading (alignment only)") parser.add_argument( "--debug", action="store_true", help= "Debug mode. Will run a small number of particles directly without parallelism with lots of print out. ", default=False) parser.add_argument( "--plst", type=str, default=None, help= "list of 2d particle with alignment parameters. The program will reconstruct before alignment so it can be slower." ) parser.add_argument( "--maxshift", type=int, help= "Maximum shift from the center of the box or the previous alignment. default box size//6", default=-1) parser.add_argument( "--maxang", type=int, help= "Maximum angle difference from starting point. Ignored when --fromscratch is on.", default=30) parser.add_argument( "--curve", action="store_true", help= "Mode for filament structure refinement. Still under testing. Ignored when --fromscratch is on.", default=False) parser.add_argument( "--skipali", action="store_true", help= "Skip alignment and only calculate the score. Incompatible with --fromscratch, but --breaksym will still be considered.", default=False) parser.add_argument( "--breaksym", type=str, default=None, help="Specify symmetry to break. Ignored when --fromscratch is on.") parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help= "verbose level [0-9], higher number means higher level of verboseness") parser.add_argument( "--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-1) (options, args) = parser.parse_args() logid = E2init(sys.argv, options.ppid) options.ref = args[1] #### we assume the particle_info files are in the right place so the bookkeeping is easier ## they should be created by e2spt_refine_new or e2spt_refine_multi_new options.info3dname = "{}/particle_info_3d.lst".format(options.path) options.info2dname = "{}/particle_info_2d.lst".format(options.path) if options.preprocess != None: options.preprocess = parsemodopt(options.preprocess) #### this should generate a list of dictionaries (one dictionary per 3d particle) tasks = load_lst_params(args[0]) #### keep a copy of the particle index here, so we can make sure the output list has the same order as the input list for i, t in enumerate(tasks): t["ii"] = i from EMAN2PAR import EMTaskCustomer etc = EMTaskCustomer(options.parallel, module="e2spt_align_subtlt.SptAlignTask") num_cpus = etc.cpu_est() options.nowtime = time.time() #### only run a few particles for debugging if options.debug: tasks = tasks[:num_cpus * 4] print("{} jobs on {} CPUs".format(len(tasks), num_cpus)) njob = num_cpus #### send out jobs tids = [] for i in range(njob): t = tasks[i::njob] task = SptAlignTask(t, options) if options.debug: ret = task.execute(print) return tid = etc.send_task(task) tids.append(tid) while 1: st_vals = etc.check_task(tids) if -100 in st_vals: print("Error occurs in parallelism. Exit") return E2progress(logid, np.mean(st_vals) / 100.) if np.min(st_vals) == 100: break time.sleep(5) #### each job returns a 3-object tuple: ## (index of 3d particle, 3d particle align info dict, list of dict for 2d particles align info) output3d = [None] * len(tasks) output2d = [None] * len(tasks) for i in tids: rets = etc.get_results(i)[1] for r in rets: output3d[r[0]] = r[1] output2d[r[0]] = r[2] del etc #### merge the 2d particle alignment info into one list output2d = sum(output2d, []) #### just dump the list of dictionaries to list files fm3d = f"{options.path}/aliptcls3d_{options.iter:02d}.lst" save_lst_params(output3d, fm3d) fm2d = f"{options.path}/aliptcls2d_{options.iter:02d}.lst" save_lst_params(output2d, fm2d) E2end(logid)
class EMParallelProject3D(object): def __init__(self, options, fsp, sym, start, modeln=0, logger=None): ''' @param options the options produced by (options, args) = parser.parse_args() @param args the options produced by (options, args) = parser.parse_args() @param logger and EMAN2 logger, i.e. logger=E2init(sys.argv) assumes you have already called the check function. ''' self.options = options self.args = fsp self.sym = sym self.logger = logger self.start = start self.modeln = modeln from EMAN2PAR import EMTaskCustomer self.etc = EMTaskCustomer(options.parallel, module="e2project3d.EMProject3DTaskDC") print("Precache ", fsp) self.etc.precache([fsp]) self.num_cpus = self.etc.cpu_est() print(self.num_cpus, " total CPUs available") if self.num_cpus > 64: # upper limit self.num_cpus = 64 self.__task_options = None def __init_memory(self, options): ''' ''' sym_object = parsesym(self.sym) [og_name, og_args] = parsemodopt(options.orientgen) self.eulers = sym_object.gen_orientations(og_name, og_args) def __get_task_options(self, options): if self.__task_options == None: d = {} d["projector"] = parsemodopt(options.projector) d["prethreshold"] = options.prethreshold self.__task_options = d return self.__task_options def execute(self): # from EMAN2PAR import EMTaskCustomer if len(self.options.parallel) > 1: self.__init_memory(self.options) num_tasks = self.num_cpus # In the worst case we can only spawn as many tasks as there are eulers if self.num_cpus > len(self.eulers): num_tasks = len(self.eulers) eulers_per_task = old_div(len(self.eulers), num_tasks) resid_eulers = len( self.eulers ) - eulers_per_task * num_tasks # we can distribute the residual evenly first = 0 task_customers = [] tids = [] # self.etc=EMTaskCustomer(self.options.parallel) for i in range(0, num_tasks): last = first + eulers_per_task if resid_eulers > 0: last += 1 resid_eulers -= 1 tmp_eulers = self.eulers[first:last] indices = list(range(first, last)) data = {} data["input"] = ("cache", self.args, 0) data["eulers"] = tmp_eulers data["indices"] = indices task = EMProject3DTaskDC(data=data, options=self.__get_task_options( self.options)) #print "Est %d CPUs"%etc.cpu_est() tid = self.etc.send_task(task) #print "Task submitted tid=",tid tids.append(tid) first = last print("Task ids are", tids) while 1: print(len(tids), "projection tasks left in main loop") st_vals = self.etc.check_task(tids) for i in range(len(tids) - 1, -1, -1): st = st_vals[i] if st == 100: tid = tids[i] rslts = self.etc.get_results(tid) if not self.__write_output_data(rslts[1]): print("There was a problem with the task of id", tid) if self.logger != None: E2progress( self.logger, 1.0 - old_div(len(tids), float(num_tasks))) if self.options.verbose > 0: print("%d/%d\r" % (num_tasks - len(tids), num_tasks)) sys.stdout.flush() print("Task", tids.pop(i), "completed") print("These tasks are remaining:", tids) if len(tids) == 0: break time.sleep(5) return len(self.eulers) else: raise NotImplementedError( "The parallelism option you specified (%s) is not suppored" % self.options.parallel) def __write_output_data(self, rslts): for idx, image in list(rslts.items()): if not isinstance(image, EMData): continue # this is here because we get the dimensions of the database as a key (e.g. '40x40x1'). image["model_id"] = self.modeln #if self.options.append : image.write_image(self.options.outfile,-1) #else : image.write_image(self.options.outfile,idx+self.start) if self.options.append: image.write_compressed(self.options.outfile, -1, self.options.compressbits) else: image.write_compressed(self.options.outfile, idx + self.start, self.options.compressbits) return True
def execute(self,alignment_jobs,files,caller): ''' The main function @param alignment_jobs a list of alignment pair indices like this [[0,1],[2,1],[2,3],[0,5],...] etc the indices pair represent images to be aligned and correspond to the order of the files argument @param files a list of filenames - used to read image based on the indices present in alignment_jobs @param caller - the calling object - it needs to have a function called process_output that takes a dictionary as the argument ''' options = self.options align_data = EMAN2.parsemodopt(options.align) align_cmp_data = EMAN2.parsemodopt(options.aligncmp) cmp_data = EMAN2.parsemodopt(options.cmp) ralign_data = None if options.ralign != None: ralign_data = EMAN2.parsemodopt(options.ralign) ralign_cmp_data = EMAN2.parsemodopt(options.raligncmp) data = {} data["align"] = align_data data["aligncmp"] = align_cmp_data data["cmp"] = cmp_data if ralign_data: data["ralign"] = ralign_data data["raligncmp"] = ralign_cmp_data data["using_cuda"] = self.using_cuda data["nsoln"] = self.nsoln if self.options.parallel : task_customers = [] tids = [] if options.shrink: scratch_name_1 = numbered_bdb("bdb:tomo_scratch#scratch_shrink") scratch_name_2 = numbered_bdb("bdb:tomo_scratch##scratch_shrink") else: print("no shrink") for i,j in alignment_jobs: if options.shrink or options.filter: a = EMData(files[i],0) if options.filter: filter_params = EMAN2.parsemodopt(options.filter) a.process_inplace(filter_params[0],filter_params[1]) if options.shrink: a.process_inplace("math.meanshrink",{"n":options.shrink}) a.set_attr("src_image",files[i]) a.write_image(scratch_name_1,0) a = EMData(files[j],0) if options.filter: filter_params = EMAN2.parsemodopt(options.filter) a.process_inplace(filter_params[0],filter_params[1]) if options.shrink: a.process_inplace("math.meanshrink",{"n":options.shrink}) a.set_attr("src_image",files[j]) a.write_image(scratch_name_2,0) data["probe"] = ("cache",scratch_name_1,0) data["target"] = ("cache",scratch_name_2,0) else: data["probe"] = ("cache",files[i],0) data["target"] = ("cache",files[j],0) data["target_idx"] = j data["probe_idx"] = i task = EMTomoAlignTaskDC(data=data) from EMAN2PAR import EMTaskCustomer etc=EMTaskCustomer(self.options.parallel) #print "Est %d CPUs"%etc.cpu_est() tid=etc.send_task(task) #print "Task submitted tid=",tid task_customers.append(etc) tids.append(tid) self.dc_monitor(task_customers,tids,caller) else: n = len(alignment_jobs) p = 0.0 for i,j in alignment_jobs: probe = EMData(files[i],0) target = EMData(files[j],0) if options.filter: print("filtered") filter_params = EMAN2.parsemodopt(options.filter) probe.process_inplace(filter_params[0],filter_params[1]) target.process_inplace(filter_params[0],filter_params[1]) if options.shrink: probe.process_inplace("math.meanshrink",{"n":options.shrink}) target.process_inplace("math.meanshrink",{"n":options.shrink}) else: print("no shrink") data["target"] = target data["probe"] = probe data["target_idx"] = j data["probe_idx"] = i task = EMTomoAlignTask(data=data) rslts = task.execute(self.progress_callback) if options.shrink: self.correction_translation(rslts,options.shrink) caller.process_output(rslts) p += 1.0
def main(): progname = os.path.basename(sys.argv[0]) usage = """ """ parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_argument( "--path", type=str, help= "Path to a folder where results should be stored, following standard naming conventions", default="subtlt_00") parser.add_argument("--ref", type=str, help="reference map", default=None) parser.add_argument( "--iter", type=int, help="Iteration number within path. Default = start a new iteration", default=0) parser.add_argument( "--goldcontinue", action="store_true", help= "Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again", default=False) parser.add_argument( "--maxres", type=float, help="Maximum resolution to consider in alignment (in A, not 1/A)", default=0) parser.add_argument( "--minres", type=float, help="Minimum resolution to consider in alignment (in A, not 1/A)", default=0) parser.add_argument( "--smooth", type=float, help= "smooth local motion by this factor. smoother local motion with larger numbers. default 100", default=100) parser.add_argument( "--smoothN", type=int, help="number of neighboring particles used for smoothing. default 15", default=15) parser.add_argument("--maxshift", type=float, help="max shift in pixel. default default box size/6", default=-1) parser.add_argument("--refine_trans", action="store_true", help="do translational alignment.", default=False) parser.add_argument( "--refine_rot", action="store_true", help= "do translational-rotational alignment. better to start from an existing translational alignment.", default=False) parser.add_argument( "--refine_defocus", action="store_true", help= "do defocus refinement. need aliptcls input. doesn't work with refine_trans or rot yet..", default=False) parser.add_argument( "--use3d", action="store_true", help="use projection of 3d particles instead of 2d ones..", default=False) parser.add_argument( "--aliptcls2d", type=str, help= "optional aliptcls input. the program can start search from the position from last run.", default="") parser.add_argument("--aliptcls3d", type=str, help="optional aliptcls input.", default="") parser.add_argument("--parallel", type=str, help="Thread/mpi parallelism to use", default="thread:4") parser.add_argument("--debug", action="store_true", help="for testing.", default=False) parser.add_argument( "--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help= "verbose level [0-9], higher number means higher level of verboseness") parser.add_argument( "--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-1) (options, args) = parser.parse_args() logid = E2init(sys.argv, options.ppid) options.info2dname = "{}/particle_info_2d.lst".format(options.path) options.info3dname = "{}/particle_info_3d.lst".format(options.path) n = EMUtil.get_image_count(options.info2dname) tasks = list(range(n)) from EMAN2PAR import EMTaskCustomer etc = EMTaskCustomer(options.parallel, module="e2spt_subtlt_local.SptAlignTask") num_cpus = etc.cpu_est() if options.debug: tasks = tasks[:num_cpus * 4] print("{} jobs on {} CPUs".format(len(tasks), num_cpus)) njob = num_cpus tids = [] for i in range(njob): t = tasks[i::njob] task = SptAlignTask(t, options) if options.debug: ret = task.execute(print) return tid = etc.send_task(task) tids.append(tid) while 1: st_vals = etc.check_task(tids) if -100 in st_vals: print("Error occurs in parallelism. Exit") return E2progress(logid, np.mean(st_vals) / 100.) if np.min(st_vals) == 100: break time.sleep(5) output = [None] * len(tasks) for i in tids: rets = etc.get_results(i)[1] for r in rets: output[r[0]] = r[1] del etc fm = "{}/aliptcls2d_{:02d}.lst".format(options.path, options.iter) save_lst_params(output, fm) E2end(logid)
def main(): usage = " " parser = EMArgumentParser(usage=usage, version=EMANVERSION) parser.add_argument("--ptclin", type=str, help="particle input", default=None) parser.add_argument("--ptclout", type=str, help="particle output", default=None) parser.add_argument("--ref", type=str, help="reference input", default=None) parser.add_argument("--keep", type=float, help="propotion of tilts to keep. default is 0.8", default=0.8) parser.add_argument( "--parallel", type=str, help="Thread/mpi parallelism to use. Default is thread:12", default="thread:12") parser.add_argument( "--debug", action="store_true", default=False, help= "Turn on debug mode. This will only process a small subset of the data" ) parser.add_argument("--maxshift", type=int, help="maximum shift allowed", default=-1) parser.add_argument("--localrefine", action="store_true", default=False, help="local refinement") #parser.add_argument("--ctfweight", action="store_true", default=False ,help="weight by ctf. not used yet...") parser.add_argument("--slow", action="store_true", default=False, help="slow but finer search") parser.add_argument("--maxres", type=float, default=-1, help="max resolution for cmp") parser.add_argument("--minrespx", type=int, default=4, help="skip the first x pixel in fourier space") parser.add_argument("--sym", type=str, help="symmetry. ", default="c1") parser.add_argument("--ppid", type=int, help="ppid...", default=-1) #parser.add_argument("--nkeep", type=int,help="", default=1) parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0) (options, args) = parser.parse_args() logid = E2init(sys.argv) lstname = options.ptclin threedname = options.ref lname = options.ptclout lst = LSXFile(lstname, True) m = EMData(threedname) bxsz = m["nx"] apix = m["apix_x"] options.shrink = 1 pinfo = [] nptcl = lst.n if options.maxshift < 0: options.maxshift = bxsz // 2 print("Initializing parallelism...") etc = EMTaskCustomer(options.parallel, module="e2spa_align.SpaAlignTask") num_cpus = etc.cpu_est() print("{} particles".format(nptcl)) print("{} total CPUs available".format(num_cpus)) if options.debug: nptcl = min(4 * num_cpus, nptcl) print("Debugging mode. running on one thread with 8 particles") for i in range(nptcl): pinfo.append(lst.read(i)) lst = None nbatch = min(nptcl // 4, num_cpus) infos = [[] for i in range(nbatch)] for i, info in enumerate(pinfo): infos[i % nbatch].append([i, info]) print("{} jobs, each with {:.1f} particles".format( len(infos), np.mean([len(i) for i in infos]))) tids = [] for info in infos: task = SpaAlignTask(info, threedname, options) if options.debug: task.execute(print) return tid = etc.send_task(task) tids.append(tid) while 1: st_vals = etc.check_task(tids) if -100 in st_vals: print("Error occurs in parallelism. Exit") return E2progress(logid, np.mean(st_vals) / 100.) if np.min(st_vals) == 100: break time.sleep(5) dics = [0] * nptcl for i in tids: ret = etc.get_results(i)[1] for r in ret: ii = r.pop("idx") dics[ii] = r del etc allscr = [d["score"] for d in dics] maxl = np.max([len(s) for s in allscr]) maxv = np.max(np.concatenate(allscr)) for s in allscr: s.extend([maxv] * (maxl - len(s))) allscr = np.array(allscr) try: os.remove(lname) except: pass lout = LSXFile(lname, False) for i, dc in enumerate(dics): lc = "" for j, xf in enumerate(dc["xform.align3d"]): d = xf.get_params("eman") d["score"] = float(allscr[i, j]) lc = lc + str(d) + ';' l = pinfo[i] lout.write(-1, l[0], l[1], lc[:-1]) lout = None E2end(logid)