class EMParallelProject3D:
def __init__(self, options, fsp, sym, start, modeln=0, logger=None):
'''
@param options the options produced by (options, args) = parser.parse_args()
@param args the options produced by (options, args) = parser.parse_args()
@param logger and EMAN2 logger, i.e. logger=E2init(sys.argv)
assumes you have already called the check function.
'''
self.options = options
self.args = fsp
self.sym = sym
self.logger = logger
self.start = start
self.modeln = modeln
from EMAN2PAR import EMTaskCustomer
self.etc = EMTaskCustomer(options.parallel)
print "Precache ", fsp
self.etc.precache([fsp])
self.num_cpus = self.etc.cpu_est()
print self.num_cpus, " total CPUs available"
if self.num_cpus > 64: # upper limit
self.num_cpus = 64
self.__task_options = None
def __init_memory(self, options):
'''
'''
sym_object = parsesym(self.sym)
[og_name, og_args] = parsemodopt(options.orientgen)
self.eulers = sym_object.gen_orientations(og_name, og_args)
def __get_task_options(self, options):
if self.__task_options == None:
d = {}
d["projector"] = parsemodopt(options.projector)
d["prethreshold"] = options.prethreshold
self.__task_options = d
return self.__task_options
def execute(self):
# from EMAN2PAR import EMTaskCustomer
if len(self.options.parallel) > 1:
self.__init_memory(self.options)
num_tasks = self.num_cpus
# In the worst case we can only spawn as many tasks as there are eulers
if self.num_cpus > len(self.eulers): num_tasks = len(self.eulers)
eulers_per_task = len(self.eulers) / num_tasks
resid_eulers = len(
self.eulers
) - eulers_per_task * num_tasks # we can distribute the residual evenly
first = 0
task_customers = []
tids = []
# self.etc=EMTaskCustomer(self.options.parallel)
for i in xrange(0, num_tasks):
last = first + eulers_per_task
if resid_eulers > 0:
last += 1
resid_eulers -= 1
tmp_eulers = self.eulers[first:last]
indices = range(first, last)
data = {}
data["input"] = ("cache", self.args, 0)
data["eulers"] = tmp_eulers
data["indices"] = indices
task = EMProject3DTaskDC(data=data,
options=self.__get_task_options(
self.options))
#print "Est %d CPUs"%etc.cpu_est()
tid = self.etc.send_task(task)
#print "Task submitted tid=",tid
tids.append(tid)
first = last
print "Task ids are", tids
while 1:
print len(tids), "projection tasks left in main loop"
st_vals = self.etc.check_task(tids)
for i in xrange(len(tids) - 1, -1, -1):
st = st_vals[i]
if st == 100:
tid = tids[i]
rslts = self.etc.get_results(tid)
if not self.__write_output_data(rslts[1]):
print "There was a problem with the task of id", tid
if self.logger != None:
E2progress(self.logger,
1.0 - len(tids) / float(num_tasks))
if self.options.verbose > 0:
print "%d/%d\r" % (num_tasks - len(tids),
num_tasks)
sys.stdout.flush()
print "Task", tids.pop(i), "completed"
print "These tasks are remaining:", tids
if len(tids) == 0: break
time.sleep(5)
return len(self.eulers)
else:
raise NotImplementedError(
"The parallelism option you specified (%s) is not suppored" %
self.options.parallel)
def __write_output_data(self, rslts):
for idx, image in rslts.items():
if not isinstance(image, EMData):
continue # this is here because we get the dimensions of the database as a key (e.g. '40x40x1').
image["model_id"] = self.modeln
if self.options.append: image.write_image(self.options.outfile, -1)
else: image.write_image(self.options.outfile, idx + self.start)
return True
class EMParallelSimMX:
def __init__(self,options,args,logger=None):
'''
@param options the options produced by (options, args) = parser.parse_args()
@param args the options produced by (options, args) = parser.parse_args()
@param logger and EMAN2 logger, i.e. logger=E2init(sys.argv)
assumes you have already called the check function.
'''
self.options = options
self.args = args
self.logger = logger
from EMAN2PAR import EMTaskCustomer
self.etc=EMTaskCustomer(options.parallel)
if options.colmasks!=None : self.etc.precache([args[0],args[1],options.colmasks])
else : self.etc.precache([args[0],args[1]])
self.num_cpus = self.etc.cpu_est()
if self.num_cpus < 32: # lower limit
self.num_cpus = 32
self.__task_options = None
def __get_task_options(self,options):
'''
Get the options required by each task as a dict
@param options is always self.options - the initialization argument. Could be changed.
'''
if self.__task_options == None:
d = {}
d["align"] = parsemodopt(options.align)
d["aligncmp"] = parsemodopt(options.aligncmp)
d["cmp"] = parsemodopt(options.cmp)
if hasattr(options,"ralign") and options.ralign != None:
d["ralign"] = parsemodopt(options.ralign)
d["raligncmp"] = parsemodopt(options.raligncmp) # raligncmp must be specified if using ralign
else:
d["ralign"] = None
d["raligncmp"] = None
d["prefilt"]=options.prefilt
if hasattr(options,"shrink") and options.shrink != None: d["shrink"] = options.shrink
else: d["shrink"] = None
self.__task_options = d
return self.__task_options
def __init_memory(self,options):
'''
@param options is always self.options - the initialization argument. Could be changed.
Establishes several important attributes they are:
----
self.clen - the number of images in the image defined by args[0], the number of columns in the similarity matrix
self.rlen - the number of images in the image defined by args[1], the number of rows in the similarity matrix
----
Also, since we adopted region output writing as our preferred approach, this function makes sure the output
image(s) exists on disk and has the correct dimensions - seeing as this is the way region writing works (the image
has to exist on disk and have its full dimensions)
'''
self.clen=EMUtil.get_image_count(self.args[0])
self.rlen=EMUtil.get_image_count(self.args[1])
output = self.args[2]
if file_exists(output) and not options.fillzero:
if options.force: remove_file(output)
else: raise RuntimeError("The output file exists. Please remove it or specify the force option")
e = EMData(self.clen,self.rlen)
e.to_zero()
e.set_attr(PROJ_FILE_ATTR,self.args[0])
e.set_attr(PART_FILE_ATTR,self.args[1])
n = 1
if self.options.saveali: n = 6 # the total number of images written to disk
if not options.fillzero : e.write_image(output,0)
for i in range(1,n):
e.write_image(output,i)
def __get_blocks(self):
'''
Gets the blocks that will be processed in parallel, these are essentially ranges
'''
steve_factor = 3 # increase number of jobs a bit for better distribution
total_jobs = steve_factor*self.num_cpus
[col_div,row_div] = opt_rectangular_subdivision(self.clen,self.rlen,total_jobs)
block_c = self.clen/col_div
block_r = self.rlen/row_div
residual_c = self.clen-block_c*col_div # residual left over by integer division
blocks = []
current_c = 0
for c in xrange(0,col_div):
last_c = current_c + block_c
if residual_c > 0:
last_c += 1
residual_c -= 1
current_r = 0
residual_r = self.rlen-block_r*row_div # residual left over by integer division
for r in xrange(0,row_div) :
last_r = current_r + block_r
if residual_r > 0:
last_r += 1
residual_r -= 1
blocks.append([current_c,last_c,current_r,last_r])
current_r = last_r
current_c = last_c
# print col_div,row_div,col_div*row_div
# print self.clen,self.rlen,residual_c,residual_r
return blocks
def execute(self):
'''
The main function to be called
'''
if len(self.options.parallel) > 1 :
self.__init_memory(self.options)
blocks = self.__get_blocks()
# print blocks
# self.check_blocks(blocks) # testing function can be removed at some point
tasks=[]
for bn,block in enumerate(blocks):
data = {}
data["references"] = ("cache",self.args[0],block[0],block[1])
data["particles"] = ("cache",self.args[1],block[2],block[3])
if self.options.colmasks!=None : data["colmasks"] = ("cache",self.options.colmasks,block[0],block[1])
if self.options.mask!=None : data["mask"] = ("cache",self.options.mask,0,1)
if self.options.fillzero :
# for each particle check to see which portion of the matrix we need to fill
if (bn%10==0) : print "%d/%d \r"%(bn,len(blocks)),
sys.stdout.flush()
rng=[]
for i in range(block[2],block[3]):
c=EMData()
c.read_image(self.args[2],0,False,Region(block[0],i,block[1]-block[0]+1,1))
inr=0
st=0
for j in range(c["nx"]):
if c[j]==0 and not inr:
st=j
inr=1
if c[j]!=0 and inr:
rng.append((i,st+block[0],j-1+block[0]))
inr=0
if inr :
rng.append((i,st+block[0],j+block[0]))
data["partial"]=rng
# print "%d) %s\t"%(bn,str(block)),rng
if self.options.fillzero and len(data["partial"])==0 : continue # nothing to compute in this block, skip it completely
else :
task = EMSimTaskDC(data=data,options=self.__get_task_options(self.options))
#print "Est %d CPUs"%etc.cpu_est()
tasks.append(task)
# This just verifies that all particles have at least one class
#a=set()
#for i in tasks:
#for k in i.data["partial"] : a.add(k[0])
#b=set(range(self.rlen))
#b-=a
#print b
print "%d/%d "%(bn,len(blocks))
self.tids=self.etc.send_tasks(tasks)
print len(self.tids)," tasks submitted"
#
while 1:
if len(self.tids) == 0: break
print len(self.tids),"simmx tasks left in main loop \r",
sys.stdout.flush()
st_vals = self.etc.check_task(self.tids)
for i in xrange(len(self.tids)-1,-1,-1):
st = st_vals[i]
if st==100:
tid = self.tids[i]
try:
rslts = self.etc.get_results(tid)
# display(rslts[1]["rslt_data"][0])
self.__store_output_data(rslts[1])
except:
traceback.print_exc()
print "ERROR storing results for task %d. Rerunning."%tid
self.etc.rerun_task(tid)
continue
if self.logger != None:
E2progress(self.logger,1.0-len(self.tids)/float(len(blocks)))
if self.options.verbose>0:
print "%d/%d\r"%(len(self.tids),len(blocks))
sys.stdout.flush()
self.tids.pop(i)
print len(self.tids),"simmx tasks left in main loop \r",
sys.stdout.flush()
time.sleep(10)
print "\nAll simmx tasks complete "
# if using fillzero, we must fix the -1.0e38 values placed into empty cells
if self.options.fillzero :
l=EMData(self.args[2],0,True)
rlen=l["ny"]
clen=l["nx"]
# launch_childprocess("e2proc2d.py %s %s"%(self.args[2],self.args[2]+"_x"))
print "Filling noncomputed regions in similarity matrix (%dx%d)"%(clen,rlen)
l=EMData()
for r in range(rlen):
l.read_image(self.args[2],0,False,Region(0,r,clen,1))
fill=l["maximum"]+.0001
l.process_inplace("threshold.belowtominval",{"minval":-1.0e37,"newval":fill})
l.write_image(self.args[2],0,EMUtil.ImageType.IMAGE_UNKNOWN,False,Region(0,r,clen,1))
print "Filling complete"
else: raise NotImplementedError("The parallelism option you specified (%s) is not supported" %self.options.parallel )
def __store_output_data(self,rslts):
'''
Store output data to internal images (matrices)
@param a dictionary return by the EMSimTaskDC
'''
result_data = rslts["rslt_data"]
output = self.args[2]
insertion_c = rslts["min_ref_idx"]
insertion_r = rslts["min_ptcl_idx"]
result_mx = result_data[0]
r = Region(insertion_c,insertion_r,result_mx.get_xsize(),result_mx.get_ysize())
# Note this is region io - the init_memory function made sure the images exist and are the right dimensions (on disk)
for i,mxout in enumerate(result_data):
mxout.write_image(output,i,EMUtil.ImageType.IMAGE_UNKNOWN,False,r)
def main():
parser = EMArgumentParser(usage="")
parser.add_argument("--output", default="threed.hdf", help="Output reconstructed volume file name.")
parser.add_argument("--input", default=None, help="The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion")
parser.add_argument("--sym", dest="sym", default="c1", help="Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct.")
parser.add_argument("--pad", default=-1,type=int, help="Will zero-pad images to the specifed size. ")
parser.add_argument("--outsize", default=-1, type=int, help="Defines the dimensions of the final volume written to disk")
parser.add_argument("--keep", type=float, dest="keep", help="The fraction of slices to keep, based on quality scores (1.0 = use all slices). See keepsig.",default=.9)
parser.add_argument("--no_wt", action="store_true", dest="no_wt", default=False, help="This argument turns automatic weighting off causing all images to be weighted by 1.")
parser.add_argument("--mode", type=str, default="trilinear", help="Fourier reconstruction 'mode' to use. The default should not normally be changed.")
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--apix",metavar="A/pix",type=float,help="A/pix value for output, overrides automatic values",default=None)
parser.add_argument("--ref", type=str,help="ref", default=None)
parser.add_argument("--tidrange", type=str,help="range of tilt id to include", default="-1,-1")
parser.add_argument("--minres", type=float,help="", default=200)
parser.add_argument("--maxres", type=float,help="", default=5)
parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
parser.add_argument("--debug", action="store_true", default=False, help="")
parser.add_argument("--clsid", default=-1, type=int, help="only reconstruct a class of particles")
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
(options, args) = parser.parse_args()
logger=E2init(sys.argv,options.ppid)
# get basic image parameters
tmp=EMData(options.input,0,True)
boxsz=tmp["nx"]
if options.apix!=None : apix=options.apix
else : apix=tmp["apix_x"]
if options.pad<0:
options.pad=good_size(boxsz*1.5)
if options.outsize<0:
options.outsize=boxsz
options.tidrange=[int(i) for i in options.tidrange.split(',')]
if options.tidrange[0]>=0:
print("including tilt ids from {} to {}".format(options.tidrange[0], options.tidrange[1]))
data=initialize_data(options.input, options)
padvol=options.pad
from EMAN2PAR import EMTaskCustomer
if options.ref:
print("weighting by reference...")
ref=EMData(options.ref)
ref=ref.do_fft()
ref.process_inplace("xform.phaseorigin.tocenter")
ref.process_inplace("xform.fourierorigin.tocenter")
etc=EMTaskCustomer(options.parallel, module="e2spa_make3d.WeightptclTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
tasks=[data[i::num_cpus] for i in range(num_cpus)]
print("{} jobs".format(len(tasks)))
tids=[]
for t in tasks:
task = WeightptclTask(t, ref, options)
if options.debug:
task.execute(print)
return
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
E2progress(logger, .5*np.mean(st_vals)/100.)
if np.min(st_vals) == 100: break
time.sleep(5)
wts=[0]*len(data)
for i in tids:
wt=etc.get_results(i)[1]
for w in wt:
try:
wts[w[0]]=w[1]
except:
print(len(wts), w)
wts=np.array(wts)
del etc
r0=int(apix*boxsz/options.minres)
r1=int(apix*boxsz/options.maxres)
print(r0,r1)
scrs=np.mean(wts[:,r0:r1], axis=1)
if options.keep<1:
thr=np.sort(scrs)[int(len(scrs)*(1-options.keep))-1]
scrs[scrs<thr]=-1
for i,d in enumerate(data):
d["curve"]=wts[i]
d["weight"]=float(scrs[i])
etc=EMTaskCustomer(options.parallel, module="e2spa_make3d.Make3dTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
tasks=[data[i::num_cpus] for i in range(num_cpus)]
print("{} jobs".format(len(tasks)))
tids=[]
for t in tasks:
task = Make3dTask(t, options)
if options.debug:
task.execute(print)
return
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
E2progress(logger, .5+.5*np.mean(st_vals)/100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output=EMData(padvol, padvol, padvol)
normvol=EMData(padvol//2+1, padvol, padvol)
output.to_zero()
output.do_fft_inplace()
normvol.to_zero()
for i in tids:
threed, norm=etc.get_results(i)[1]
threed.process_inplace("math.multamplitude", {"amp":norm})
output.add(threed)
normvol.add(norm)
normvol.process_inplace("math.reciprocal")
output.process_inplace("math.multamplitude", {"amp":normvol})
output.do_ift_inplace()
output.depad()
output.process_inplace("xform.phaseorigin.tocenter")
del etc
if options.verbose>0 : print("Finished Reconstruction")
output["apix_x"]=output["apix_y"]=output["apix_z"]=apix
sz=options.outsize
output.clip_inplace(Region((padvol-sz)//2,(padvol-sz)//2,(padvol-sz)//2,sz,sz,sz))
if os.path.isfile(options.output):
os.remove(options.output)
output.write_image(options.output,0)
if options.verbose>0:
print("Output File: "+options.output)
E2end(logger)
print("Exiting")
def main():
usage=" "
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--path", type=str,help="path", default=None, guitype='strbox',row=0, col=0,rowspan=1, colspan=1)
parser.add_argument("--iter", type=int,help="start from iteration X", default=-1, guitype='intbox',row=0, col=1,rowspan=1, colspan=1)
parser.add_argument("--niters", type=int,help="run this many iterations. Default is 4.", default=4, guitype='intbox',row=0, col=2,rowspan=1, colspan=1)
parser.add_argument("--sym", type=str,help="symmetry. will use symmetry from spt refinement by default", default="", guitype='strbox',row=2, col=0,rowspan=1, colspan=1)
parser.add_argument("--padby", type=float,help="pad by factor. default is 2", default=2., guitype='floatbox',row=1, col=1,rowspan=1, colspan=1)
parser.add_argument("--keep", type=float,help="propotion of tilts to keep. default is 0.5", default=0.5, guitype='floatbox',row=1, col=2,rowspan=1, colspan=1)
parser.add_argument("--maxalt", type=float,help="max altitude to insert to volume", default=90.0, guitype='floatbox',row=1, col=0,rowspan=1, colspan=1)
parser.add_argument("--nogs", action="store_true", default=False ,help="skip gold standard...", guitype='boolbox',row=2, col=1,rowspan=1, colspan=1)
parser.add_argument("--localfilter", type=int, default=-1 ,help="use tophat local. specify 0 or 1 to overwrite the setting in the spt refinement")
parser.add_argument("--mask", type=str, default="None" ,help="Refinement masking. default is the same as the spt refinement. Leave this empty for automasking",guitype='strbox',row=3, col=0,rowspan=1, colspan=2)
parser.add_argument("--threads", type=int,help="Number of CPU threads to use. Default is 12.", default=12, guitype='intbox',row=2, col=2,rowspan=1, colspan=1)
parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use. Default is thread:12", default="thread:12", guitype='strbox',row=4, col=0,rowspan=1, colspan=3)
parser.add_argument("--refineastep", type=float,help="angular variation for refine alignment (gauss std)", default=8.)
parser.add_argument("--refinentry", type=int,help="number of starting points for refine alignment", default=32)
parser.add_argument("--maxshift", type=int,help="maximum shift allowed", default=10)
parser.add_argument("--buildsetonly", action="store_true", default=False ,help="build sets only")
parser.add_argument("--output", type=str,help="Write results to this directory. We do not recommend changing this.", default="subtlt")#, guitype='intbox',row=2, col=1,rowspan=1, colspan=1)
parser.add_argument("--debug", action="store_true", default=False ,help="Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)")
parser.add_argument("--ppid", type=int,help="ppid...", default=-1)
(options, args) = parser.parse_args()
logid=E2init(sys.argv)
itr = options.iter
oldpath = options.path
if not oldpath:
print("No input path. Exit.")
return
if options.iter != -1:
itr = options.iter
elif "spt" in oldpath:
for f in sorted(os.listdir(oldpath)):
if "particle_parms" in f:
itrstr = f[15:].split(".")[0]
if os.path.isfile("{}/threed_{}.hdf".format(oldpath,itrstr)):
itr = int(itrstr)
else:
for f in sorted(os.listdir(oldpath)):
if re.match("threed_[0-9][0-9].hdf",f):
itr = int(f[7:].split(".")[0])
# print(oldpath)
fromspt=True
if "0_subtlt_params.json" in os.listdir(oldpath):
print("Continuing from a subtilt refinement...")
fromspt=False
path = make_path(options.output)
if not os.path.isfile("{}/threed_{:02d}.hdf".format(oldpath,itr)):
print("Could not locate {}/threed_{:02d}.hdf".format(oldpath,itr))
print("Please specify the iteration number (--iter) of a completed subtomogram refinement.")
sys.exit(1)
#elif not os.path.isfile("{}/particle_parms_{:02d}.json".format(oldpath,itr)):
#print("Could not locate {}/particle_parms_{:02d}.json".format(oldpath,itr))
#print("Please specify the iteration number (--iter) of a completed subtomogram refinement.")
#sys.exit(1)
else:
#copy2("{}/0_spt_params.json".format(oldpath),"{}/0_subtlt_params.json".format(path))
oldmap = os.path.join(oldpath,"threed_{:02d}.hdf".format(itr))
oem = os.path.join(oldpath,"threed_{:02d}_even.hdf".format(itr))
oom = os.path.join(oldpath,"threed_{:02d}_odd.hdf".format(itr))
oldfsc = os.path.join(oldpath, "fsc_masked_{:02d}.txt".format(itr))
copy2(oldmap,os.path.join(path,"threed_00.hdf"))
copy2(oldfsc, os.path.join(path, "fsc_masked_00.txt"))
copy2(oem,os.path.join(path,"threed_00_even.hdf"))
copy2(oom,os.path.join(path,"threed_00_odd.hdf"))
if fromspt:
oldparm = os.path.join(oldpath,"particle_parms_{:02d}.json".format(itr))
copy2(oldparm,os.path.join(path,"particle_parms_00.json"))
else:
for eo in ["even", "odd"]:
oali = os.path.join(oldpath,"ali_ptcls_{:02d}_{}.lst".format(itr, eo))
copy2(oali,os.path.join(path,"ali_ptcls_00_{}.lst".format(eo)))
e=EMData(os.path.join(path,"threed_00.hdf"))
bxsz=e["nx"]
apix=e["apix_x"]
jd = js_open_dict("{}/0_subtlt_params.json".format(path))
jd.update(vars(options))
jd["cmd"] = " ".join(sys.argv)
jd["path"] = oldpath
jd["iter"] = itr
jd["output"] = path
if fromspt:
sptparms = os.path.join(oldpath,"0_spt_params.json")
else:
sptparms = os.path.join(oldpath,"0_subtlt_params.json")
if os.path.isfile(sptparms):
oldjd = js_open_dict(sptparms)
#print(oldjd.keys())
jd["mass"] = oldjd["mass"]
jd["setsf"] = oldjd["setsf"]
jd["sym"] = oldjd["sym"]
jd["localfilter"]=oldjd["localfilter"]
jd["mask"]=oldjd["mask"]
oldjd.close()
else:
print("Cannot find {}. exit.".format(sptparms))
if options.mask.lower()!="none":
print("Overwritting masking")
jd["mask"]=options.mask
if options.localfilter==0:
jd["localfilter"]=False
elif options.localfilter==1:
jd["localfilter"]=True
if len(options.sym)>0:
jd["sym"]=options.sym
jsparams=jd.data
jd.close()
jd = jsparams
if fromspt:
js=js_open_dict(os.path.join(path,"particle_parms_00.json"))
k=list(js.keys())[0]
src=eval(k)[0]
print("loading 3D particles from {}".format(src))
print("box size {}, apix {:.2f}".format(bxsz, apix))
lname=[os.path.join(path, "ali_ptcls_00_{}.lst".format(eo)) for eo in ["even", "odd"]]
for l in lname:
try: os.remove(l)
except:pass
lst=[LSXFile(m, False) for m in lname]
n3d=len(list(js.keys()))
for ii in range(n3d):
e=EMData(src, ii, True)
fname=e["class_ptcl_src"]
ids=e["class_ptcl_idxs"]
ky="('{}', {})".format(src, ii)
dic=js[ky]
xali=dic["xform.align3d"]
for i in ids:
try:
m=EMData(fname, i, True)
except:
continue
xf=m["xform.projection"]
dc=xf.get_params("xyz")
if abs(dc["ytilt"])>options.maxalt:
continue
rot=xf*xali.inverse()
lst[ii%2].write(-1, i, fname, str(rot.get_params("eman")))
for l in lst:
l.close()
js=None
if options.buildsetonly: return
for itr in range(0,options.niters):
from EMAN2PAR import EMTaskCustomer
for eo in ["even", "odd"]:
if options.nogs:
threedname="{}/threed_{:02d}.hdf".format(path, itr)
else:
threedname="{}/threed_{:02d}_{}.hdf".format(path, itr, eo)
lstname="{}/ali_ptcls_{:02d}_{}.lst".format(path, itr, eo)
lst=LSXFile(lstname, True)
m=EMData(threedname)
m.process_inplace('normalize.edgemean')
pinfo=[]
if options.debug: nptcl=options.threads*8
else: nptcl=lst.n
for i in range(nptcl):
pinfo.append(lst.read(i))
lst=None
etc=EMTaskCustomer(options.parallel)
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
print("{} jobs".format(nptcl))
infos=[[] for i in range(num_cpus)]
for i,info in enumerate(pinfo):
infos[i%num_cpus].append([i, info])
tids=[]
for info in infos:
task = SptTltRefineTask(info, m, options)
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
#print(tids)
if np.min(st_vals) == 100: break
time.sleep(5)
dics=[0]*nptcl
for i in tids:
ret=etc.get_results(i)[1]
for r in ret:
#print(r)
ii=r.pop("idx")
dics[ii]=r
del etc
allscr=np.array([d["score"] for d in dics])
print(np.min(allscr), np.mean(allscr), np.max(allscr), np.std(allscr))
allscr*=-1
s=allscr.copy()
s-=np.mean(s)
s/=np.std(s)
clp=2
ol=abs(s)>clp
print("Removing {} outliers from {} particles..".format(np.sum(ol), len(s)))
s=old_div(old_div((s+clp),clp),2)
s[ol]=0
allscr=s
#allscr-=np.min(allscr)-1e-5
#allscr/=np.max(allscr)
lname="{}/ali_ptcls_{:02d}_{}.lst".format(path, itr+1, eo)
try: os.remove(lname)
except: pass
lout=LSXFile(lname, False)
for i, dc in enumerate(dics):
d=dc["xform.align3d"].get_params("eman")
d["score"]=float(allscr[i])
l=pinfo[i]
lout.write(-1, l[0], l[1], str(d))
lout=None
pb=options.padby
threedout="{}/threed_{:02d}_{}.hdf".format(path, itr+1, eo)
cmd="e2make3dpar.py --input {inp} --output {out} --pad {pd} --padvol {pdv} --threads {trd} --outsize {bx} --apix {apx} --mode gauss_var --keep {kp} --sym {sm}".format(
inp=lname,
out=threedout,
bx=bxsz, pd=int(bxsz*pb), pdv=int(bxsz*pb), apx=apix, kp=options.keep, sm=jd["sym"], trd=options.threads)
run(cmd)
run("e2proc3d.py {} {}".format(threedout, "{}/threed_raw_{}.hdf".format(path, eo)))
s = ""
if jd.has_key("goldstandard"):
if jd["goldstandard"] > 0:
s += " --align"
if jd.has_key("setsf"):
s += " --setsf {}".format(jd['setsf']) #options.setsf)
if jd.has_key("localfilter"):
s += " --tophat local"
msk = jd["mask"] #{}/mask_tight.hdf".format(path)
if len(msk)>0:
if os.path.isfile(msk):
msk=" --automask3d mask.fromfile:filename={}".format(msk)
else:
msk=" --automask3d {}".format(msk)
# get target resolution from last iteration map
ref=os.path.join(path, "threed_{:02d}.hdf".format(itr))
fsc=np.loadtxt(os.path.join(path, "fsc_masked_{:02d}.txt".format(itr)))
rs=1./fsc[fsc[:,1]<0.3, 0][0]
curres=rs*.5
#os.system("rm {}/mask*.hdf {}/*unmasked.hdf".format(path, path))
ppcmd="e2refine_postprocess.py --even {} --odd {} --output {} --iter {:d} --restarget {} --threads {} --sym {} --mass {} {}".format(
os.path.join(path, "threed_{:02d}_even.hdf".format(itr+1)),
os.path.join(path, "threed_{:02d}_odd.hdf".format(itr+1)),
os.path.join(path, "threed_{:02d}.hdf".format(itr+1)), itr+1, curres, options.threads, jd["sym"], jd["mass"], s)
run(ppcmd)
fsc=np.loadtxt(os.path.join(path, "fsc_masked_{:02d}.txt".format(itr+1)))
rs=1./fsc[fsc[:,1]<0.3, 0][0]
print("Resolution (FSC<0.3) is ~{:.1f} A".format(rs))
E2end(logid)
def main():
usage = """
Alternative to e2spt_align.py, but uses 2D subtilt images instead of 3D particle volume for alignment. Also a modified scipy minimizer is used in place of EMAN2 aligners.
e2spt_align_subtlt.py sets/ptcls.lst reference.hdf --path spt_xx --iter x
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
)
parser.add_argument("--iter",
type=int,
help="Iteration number within path. Default = 0",
default=0)
parser.add_argument(
"--goldcontinue",
action="store_true",
help=
"Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
default=False)
parser.add_argument(
"--sym",
type=str,
default="c1",
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly."
)
parser.add_argument(
"--maxres",
type=float,
help="Maximum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument(
"--minres",
type=float,
help="Minimum resolution to consider in alignment (in A, not 1/A)",
default=0)
#parser.add_argument("--maxtilt",type=float,help="Excluding tilt images beyond the angle",default=-1)
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default="thread:4")
parser.add_argument(
"--fromscratch",
action="store_true",
help=
"Start from exhaustive coarse alignment. Otherwise will use alignment from the previous rounds and do local search only.",
default=False)
parser.add_argument(
"--use3d",
action="store_true",
help="use projection of 3d particles instead of 2d sub tilt series",
default=False)
parser.add_argument(
"--preprocess",
metavar="processor_name:param1=value1:param2=value2",
type=str,
default=None,
help="Preprocess each 2-D subtilt while loading (alignment only)")
parser.add_argument(
"--debug",
action="store_true",
help=
"Debug mode. Will run a small number of particles directly without parallelism with lots of print out. ",
default=False)
parser.add_argument(
"--plst",
type=str,
default=None,
help=
"list of 2d particle with alignment parameters. The program will reconstruct before alignment so it can be slower."
)
parser.add_argument(
"--maxshift",
type=int,
help=
"Maximum shift from the center of the box or the previous alignment. default box size//6",
default=-1)
parser.add_argument(
"--maxang",
type=int,
help=
"Maximum angle difference from starting point. Ignored when --fromscratch is on.",
default=30)
parser.add_argument(
"--curve",
action="store_true",
help=
"Mode for filament structure refinement. Still under testing. Ignored when --fromscratch is on.",
default=False)
parser.add_argument(
"--skipali",
action="store_true",
help=
"Skip alignment and only calculate the score. Incompatible with --fromscratch, but --breaksym will still be considered.",
default=False)
parser.add_argument(
"--breaksym",
type=str,
default=None,
help="Specify symmetry to break. Ignored when --fromscratch is on.")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv, options.ppid)
options.ref = args[1]
#### we assume the particle_info files are in the right place so the bookkeeping is easier
## they should be created by e2spt_refine_new or e2spt_refine_multi_new
options.info3dname = "{}/particle_info_3d.lst".format(options.path)
options.info2dname = "{}/particle_info_2d.lst".format(options.path)
if options.preprocess != None:
options.preprocess = parsemodopt(options.preprocess)
#### this should generate a list of dictionaries (one dictionary per 3d particle)
tasks = load_lst_params(args[0])
#### keep a copy of the particle index here, so we can make sure the output list has the same order as the input list
for i, t in enumerate(tasks):
t["ii"] = i
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_align_subtlt.SptAlignTask")
num_cpus = etc.cpu_est()
options.nowtime = time.time()
#### only run a few particles for debugging
if options.debug:
tasks = tasks[:num_cpus * 4]
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob = num_cpus
#### send out jobs
tids = []
for i in range(njob):
t = tasks[i::njob]
task = SptAlignTask(t, options)
if options.debug:
ret = task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
#### each job returns a 3-object tuple:
## (index of 3d particle, 3d particle align info dict, list of dict for 2d particles align info)
output3d = [None] * len(tasks)
output2d = [None] * len(tasks)
for i in tids:
rets = etc.get_results(i)[1]
for r in rets:
output3d[r[0]] = r[1]
output2d[r[0]] = r[2]
del etc
#### merge the 2d particle alignment info into one list
output2d = sum(output2d, [])
#### just dump the list of dictionaries to list files
fm3d = f"{options.path}/aliptcls3d_{options.iter:02d}.lst"
save_lst_params(output3d, fm3d)
fm2d = f"{options.path}/aliptcls2d_{options.iter:02d}.lst"
save_lst_params(output2d, fm2d)
E2end(logid)
def main():
usage = """
Extract 2D subtilt particles from the tilt series, and reconstruct 3D subvolumes.
There are a number of modes for the program and it is probably easier to get the right commands from the GUI.
To extract particles of a certain label from one tomogram
e2spt_extract.py tomograms/tomox__bin4.hdf --boxsz_unbin 128 --label <label from particle picking>
To extract the particles of the same label from all tomograms
e2spt_extract.py --alltomograms --boxsz_unbin 128 --label <label from particle picking>
To extract particles along curves drawn using e2tomo_drawcurve.py. The direction of particles along the curve will be written to the header of particles which can be used in downstream refinement. The spacing of particles along the curve is controlled by the box size and the overlap between adjacent boxes.
e2spt_extract.py --curves <curve ID> --curves_overlap 0.75 --alltomograms --boxsz_unbin 128 --newlabel <new particle label>
To extract particls from an existing refinement (either the particle_param_xx.json from e2spt_refine, or aliptcls3d_xx.lst from e2spt_refine_new)
e2spt_extract.py --jsonali <json/lst from existing alignment> --boxsz_unbin 128 --newlabel <new particle label>
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_pos_argument(
name="tomograms",
help=
"Specify tomograms from which you wish to extract boxed particles.",
default="",
guitype='filebox',
browser="EMTomoBoxesTable(withmodal=True,multiselect=True)",
row=0,
col=0,
rowspan=1,
colspan=2,
mode="extract")
parser.add_argument("--boxsz_unbin",
type=int,
help="box size in unbinned tomogram",
default=-1,
guitype='intbox',
row=2,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--label",
type=str,
help=
"Only extract particle with this name. Leave blank to extract all particles.",
default=None,
guitype='strbox',
row=2,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--newlabel",
type=str,
help=
"Label of output particles. Same as original particle label by default.",
default="",
guitype='strbox',
row=6,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock1",
help='Just a visual separation',
title="Options",
row=3,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--threads",
type=int,
help="threads",
default=12,
guitype='intbox',
row=4,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--maxtilt",
type=int,
help="max tilt",
default=100,
guitype='intbox',
row=4,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--padtwod",
type=float,
help="padding factor",
default=2.0,
guitype='floatbox',
row=5,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--noctf",
action="store_true",
default=False,
help="skip ctf correction.")
parser.add_argument(
"--wiener",
action="store_true",
default=False,
help=
"wiener filter the particles using ctf information. Not very useful..."
)
parser.add_argument("--alltomograms",
action="store_true",
default=False,
help="use all tomograms.",
guitype='boolbox',
row=1,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--dotest",
action="store_true",
default=False,
help="only make 1 batch of subtomograms for testing")
parser.add_argument(
"--shrink",
type=float,
help=
"Shrinking factor for output particles. 1.5 or integers allowed. Default is 1 (no shrink).",
default=1,
guitype='floatbox',
row=8,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--tltkeep",
type=float,
help=
"keep a fraction of tilt images with good score determined from tomogram reconstruction",
default=1.0,
guitype='floatbox',
row=8,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--rmbeadthr",
type=float,
help=
"remove 2d particles with high contrast object beyond N sigma at 100A. Note that this may result in generating fewer particles than selected. Default is -1 (include all particles). 0.5 might be a good choice for removing gold beads but may need some testing...",
default=-1,
guitype='floatbox',
row=9,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock2",
help='Just a visual separation',
title="Extract from curves",
row=10,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--curves",
type=int,
default=-1,
help="specify curve id to extract particles from saved curves. ",
guitype='intbox',
row=11,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--curves_overlap",
type=float,
help=
"fraction of overlap when generating particle along curves. default is 0.5",
default=0.5,
guitype='floatbox',
row=12,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock3",
help='Just a visual separation',
title="Re-extraction from spt",
row=13,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--jsonali",
type=str,
help=
"re-extract particles using a particle_param_xx json file from a spt alignment",
default="",
guitype='filebox',
browser="EMBrowserWidget(withmodal=True,multiselect=False)",
row=14,
col=0,
rowspan=1,
colspan=2,
mode="extract")
parser.add_argument(
"--mindist",
type=float,
help="minimum distance between particles in A. for reextraction only",
default=10)
parser.add_argument(
"--keep",
type=float,
help=
"fraction of particles to keep fron previous alignment. for reextraction only.",
default=1)
parser.add_argument("--postproc",
type=str,
help="processor after 3d particle reconstruction",
default="")
parser.add_argument(
"--postmask",
type=str,
help=
"masking after 3d particle reconstruction. The mask is transformed if json ",
default="")
#parser.add_argument("--textin", type=str,help="text file for particle coordinates. do not use..", default=None)
parser.add_argument(
"--compressbits",
type=int,
help=
"Bits to keep for compression. default is -1 meaning uncompressed floating point. 8 bit seems fine...",
default=8,
guitype='intbox',
row=5,
col=1,
rowspan=1,
colspan=1,
mode="extract[8]")
parser.add_argument("--norewrite",
action="store_true",
default=False,
help="skip existing files. do not rewrite.")
parser.add_argument("--append",
action="store_true",
default=False,
help="append to existing files.")
parser.add_argument("--parallel", type=str, help="parallel", default="")
parser.add_argument(
"--postxf",
type=str,
help=
"a file listing post transforms (see http://eman2.org/e2tomo_more), or for simple symmetry, <sym>,<cx>,<cy>,<cz> where the coordinates specify the center of a single subunit",
default=None)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-2)
parser.add_argument(
"--skip3d",
action="store_true",
default=False,
help=
"do not make 3d particles. only generate 2d particles and 3d header. ")
#parser.add_argument("--shrink3d", type=int, help="Only shrink 3d particles by x factor",default=-1)
parser.add_argument("--verbose", type=int, help="verbose", default=1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
allxfs = {}
allinfo = {}
if len(options.jsonali) > 0:
allxfs, allinfo = parse_json(options)
if options.alltomograms:
fld = "tomograms/"
args = [fld + f for f in os.listdir(fld) if f.endswith(".hdf")]
### do not count a tomogram twice when multiple versions exist
uq, uid = np.unique([base_name(a) for a in args], return_index=True)
args = [args[i] for i in uid]
if len(args) == 0 and len(allxfs) == 0:
print("No input. Exit.")
return
if options.parallel != "":
#### in theory we allow doing this via MPI in addition to threading, and it is possible to spawn one job per node, and each worker will do reconstruction with multiple threads using shared memory
## in practice it is not well tested and it is not quite clear how well it performs on different clusters.
## since the major speed limiting factor of the program is still in disk I/O, it is not much slowerer with just --threads
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_extract.SptExtractTask")
num_cpus = etc.cpu_est()
options.nowtime = time.time()
print(
"Running in parallel mode with {} workers, each running {} threads"
.format(num_cpus, options.threads))
njob = num_cpus
options.verbose = 0
if len(allxfs) > 0:
fnames = list(allxfs.keys())
elif len(args) > 0:
fnames = args
tids = []
for i in range(njob):
task = SptExtractTask(fnames[i::njob], options, allxfs, allinfo)
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if np.min(st_vals) == 100: break
#print(st_vals)
time.sleep(1)
else:
#### do extraction sequentially for each tomogram
if len(allxfs) > 0:
for fname in allxfs.keys():
#### it seems options is changed inplace somewhere...
(options, args) = parser.parse_args()
do_extraction(fname, options, allxfs[fname], allinfo[fname])
elif len(args) > 0:
for a in args:
do_extraction(a, options)
return
print("Particle extraction finished.")
E2end(logid)
return
class EMParallelSimMX(object):
def __init__(self, options, args, logger=None):
'''
@param options the options produced by (options, args) = parser.parse_args()
@param args the options produced by (options, args) = parser.parse_args()
@param logger and EMAN2 logger, i.e. logger=E2init(sys.argv)
assumes you have already called the check function.
'''
self.options = options
self.args = args
self.logger = logger
from EMAN2PAR import EMTaskCustomer
self.etc = EMTaskCustomer(options.parallel,
module="e2simmx.EMSimTaskDC")
if options.colmasks != None:
self.etc.precache([args[0], args[1], options.colmasks])
else:
self.etc.precache([args[0], args[1]])
self.num_cpus = self.etc.cpu_est()
if self.num_cpus < 32: # lower limit
self.num_cpus = 32
self.__task_options = None
def __get_task_options(self, options):
'''
Get the options required by each task as a dict
@param options is always self.options - the initialization argument. Could be changed.
'''
if self.__task_options == None:
d = {}
d["align"] = parsemodopt(options.align)
d["aligncmp"] = parsemodopt(options.aligncmp)
d["cmp"] = parsemodopt(options.cmp)
if hasattr(options, "ralign") and options.ralign != None:
d["ralign"] = parsemodopt(options.ralign)
d["raligncmp"] = parsemodopt(
options.raligncmp
) # raligncmp must be specified if using ralign
else:
d["ralign"] = None
d["raligncmp"] = None
d["prefilt"] = options.prefilt
if hasattr(options, "shrink") and options.shrink != None:
d["shrink"] = options.shrink
else:
d["shrink"] = None
self.__task_options = d
return self.__task_options
def __init_memory(self, options):
'''
@param options is always self.options - the initialization argument. Could be changed.
Establishes several important attributes they are:
----
self.clen - the number of images in the image defined by args[0], the number of columns in the similarity matrix
self.rlen - the number of images in the image defined by args[1], the number of rows in the similarity matrix
----
Also, since we adopted region output writing as our preferred approach, this function makes sure the output
image(s) exists on disk and has the correct dimensions - seeing as this is the way region writing works (the image
has to exist on disk and have its full dimensions)
'''
self.clen = EMUtil.get_image_count(self.args[0])
self.rlen = EMUtil.get_image_count(self.args[1])
output = self.args[2]
if file_exists(output) and not options.fillzero:
remove_file(output)
e = EMData(self.clen, self.rlen)
e.to_zero()
e.set_attr(PROJ_FILE_ATTR, self.args[0])
e.set_attr(PART_FILE_ATTR, self.args[1])
n = 1
if self.options.saveali:
n = 6 # the total number of images written to disk
if not options.fillzero: e.write_image(output, 0)
for i in range(1, n):
e.write_image(output, i)
def __get_blocks(self):
'''
Gets the blocks that will be processed in parallel, these are essentially ranges
'''
steve_factor = 3 # increase number of jobs a bit for better distribution
total_jobs = steve_factor * self.num_cpus
[col_div,
row_div] = opt_rectangular_subdivision(self.clen, self.rlen,
total_jobs)
block_c = old_div(self.clen, col_div)
block_r = old_div(self.rlen, row_div)
residual_c = self.clen - block_c * col_div # residual left over by integer division
blocks = []
current_c = 0
for c in range(0, col_div):
last_c = current_c + block_c
if residual_c > 0:
last_c += 1
residual_c -= 1
current_r = 0
residual_r = self.rlen - block_r * row_div # residual left over by integer division
for r in range(0, row_div):
last_r = current_r + block_r
if residual_r > 0:
last_r += 1
residual_r -= 1
blocks.append([current_c, last_c, current_r, last_r])
current_r = last_r
current_c = last_c
# print col_div,row_div,col_div*row_div
# print self.clen,self.rlen,residual_c,residual_r
return blocks
def execute(self):
'''
The main function to be called
'''
if len(self.options.parallel) > 1:
self.__init_memory(self.options)
blocks = self.__get_blocks()
# print blocks
# self.check_blocks(blocks) # testing function can be removed at some point
tasks = []
for bn, block in enumerate(blocks):
data = {}
data["references"] = ("cache", self.args[0], block[0],
block[1])
data["particles"] = ("cache", self.args[1], block[2], block[3])
if self.options.colmasks != None:
data["colmasks"] = ("cache", self.options.colmasks,
block[0], block[1])
if self.options.mask != None:
data["mask"] = ("cache", self.options.mask, 0, 1)
if self.options.fillzero:
# for each particle check to see which portion of the matrix we need to fill
if (bn % 10 == 0):
print("%d/%d \r" % (bn, len(blocks)), end=' ')
sys.stdout.flush()
rng = []
for i in range(block[2], block[3]):
c = EMData()
c.read_image(
self.args[2], 0, False,
Region(block[0], i, block[1] - block[0] + 1, 1))
inr = 0
st = 0
for j in range(c["nx"]):
if c[j] == 0 and not inr:
st = j
inr = 1
if c[j] != 0 and inr:
rng.append(
(i, st + block[0], j - 1 + block[0]))
inr = 0
if inr:
rng.append((i, st + block[0], j + block[0]))
data["partial"] = rng
# print "%d) %s\t"%(bn,str(block)),rng
if self.options.fillzero and len(data["partial"]) == 0:
continue # nothing to compute in this block, skip it completely
else:
task = EMSimTaskDC(data=data,
options=self.__get_task_options(
self.options))
#print "Est %d CPUs"%etc.cpu_est()
tasks.append(task)
# This just verifies that all particles have at least one class
#a=set()
#for i in tasks:
#for k in i.data["partial"] : a.add(k[0])
#b=set(range(self.rlen))
#b-=a
#print b
print("%d/%d " % (bn, len(blocks)))
self.tids = self.etc.send_tasks(tasks)
print(len(self.tids), " tasks submitted")
#
while 1:
if len(self.tids) == 0: break
print(len(self.tids),
"simmx tasks left in main loop \r",
end=' ')
sys.stdout.flush()
st_vals = self.etc.check_task(self.tids)
for i in range(len(self.tids) - 1, -1, -1):
st = st_vals[i]
if st == 100:
tid = self.tids[i]
try:
rslts = self.etc.get_results(tid)
# display(rslts[1]["rslt_data"][0])
self.__store_output_data(rslts[1])
except:
traceback.print_exc()
print(
"ERROR storing results for task %d. Rerunning."
% tid)
self.etc.rerun_task(tid)
continue
if self.logger != None:
E2progress(
self.logger, 1.0 -
old_div(len(self.tids), float(len(blocks))))
if self.options.verbose > 0:
print("%d/%d\r" %
(len(self.tids), len(blocks)))
sys.stdout.flush()
self.tids.pop(i)
print(len(self.tids),
"simmx tasks left in main loop \r",
end=' ')
sys.stdout.flush()
time.sleep(10)
print("\nAll simmx tasks complete ")
# if using fillzero, we must fix the -1.0e38 values placed into empty cells
if self.options.fillzero:
l = EMData(self.args[2], 0, True)
rlen = l["ny"]
clen = l["nx"]
# launch_childprocess("e2proc2d.py %s %s"%(self.args[2],self.args[2]+"_x"))
print(
"Filling noncomputed regions in similarity matrix (%dx%d)"
% (clen, rlen))
l = EMData()
for r in range(rlen):
l.read_image(self.args[2], 0, False, Region(0, r, clen, 1))
fill = l["maximum"] + .0001
l.process_inplace("threshold.belowtominval", {
"minval": -1.0e37,
"newval": fill
})
l.write_image(self.args[2], 0,
EMUtil.ImageType.IMAGE_UNKNOWN, False,
Region(0, r, clen, 1))
print("Filling complete")
else:
raise NotImplementedError(
"The parallelism option you specified (%s) is not supported" %
self.options.parallel)
def __store_output_data(self, rslts):
'''
Store output data to internal images (matrices)
@param a dictionary return by the EMSimTaskDC
'''
result_data = rslts["rslt_data"]
output = self.args[2]
insertion_c = rslts["min_ref_idx"]
insertion_r = rslts["min_ptcl_idx"]
result_mx = result_data[0]
r = Region(insertion_c, insertion_r, result_mx.get_xsize(),
result_mx.get_ysize())
# Note this is region io - the init_memory function made sure the images exist and are the right dimensions (on disk)
for i, mxout in enumerate(result_data):
mxout.write_image(output, i, EMUtil.ImageType.IMAGE_UNKNOWN, False,
r)
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--ptclin",
type=str,
help="particle input",
default=None)
parser.add_argument("--ptclout",
type=str,
help="particle output",
default=None)
parser.add_argument("--ref",
type=str,
help="reference input",
default=None)
parser.add_argument("--threedout",
type=str,
help="map output",
default=None)
parser.add_argument("--keep",
type=float,
help="propotion of tilts to keep. default is 0.5",
default=0.5)
parser.add_argument("--threads",
type=int,
help="Number of CPU threads to use. Default is 12.",
default=12)
parser.add_argument(
"--parallel",
type=str,
help="Thread/mpi parallelism to use. Default is thread:12",
default="thread:12")
parser.add_argument(
"--debug",
action="store_true",
default=False,
help=
"Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)"
)
parser.add_argument("--transonly",
action="store_true",
default=False,
help="only refine translation")
parser.add_argument("--savepath",
action="store_true",
default=False,
help="save alignment path in a json file for testing.")
#parser.add_argument("--scipytest", action="store_true", default=False ,help="test scipy optimizer")
parser.add_argument(
"--fromscratch",
action="store_true",
default=False,
help=
"align from scratch and ignore previous particle transforms. for spt mostly. will include mirror"
)
parser.add_argument("--refineastep",
type=float,
help="Mean angular variation for refine alignment",
default=2.)
parser.add_argument("--refinentry",
type=int,
help="number of starting points for refine alignment",
default=4)
parser.add_argument("--maxshift",
type=int,
help="maximum shift allowed",
default=8)
parser.add_argument("--localrefine",
action="store_true",
default=False,
help="local refinement")
parser.add_argument("--defocus",
action="store_true",
default=False,
help="refine defocus. Still under development")
parser.add_argument("--seedmap",
action="store_true",
default=False,
help="seed")
parser.add_argument("--ctfweight",
action="store_true",
default=False,
help="weight by ctf")
parser.add_argument("--skipm3d",
action="store_true",
default=False,
help="skip make3d. only output aligned list")
parser.add_argument("--padby",
type=float,
default=2.0,
help="pad by factor. default is 2")
parser.add_argument("--maxres",
type=float,
default=-1,
help="max resolution for cmp")
parser.add_argument("--minres",
type=float,
default=-1,
help="min resolution for cmp")
parser.add_argument(
"--sym",
type=str,
help="symmetry. will use symmetry from spt refinement by default",
default="c1")
parser.add_argument(
"--smooth",
type=int,
help=
"Smooth trajectory per image based on nearby particles. Still under development",
default=-1)
parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
parser.add_argument("--nkeep", type=int, help="", default=1)
parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
lstname = options.ptclin
threedname = options.ref
lname = options.ptclout
threedout = options.threedout
lst = LSXFile(lstname, True)
m = EMData(threedname)
bxsz = m["nx"]
apix = m["apix_x"]
options.shrink = 1
pinfo = []
if options.debug: nptcl = options.threads * 8
else: nptcl = lst.n
for i in range(nptcl):
pinfo.append(lst.read(i))
lst = None
print("Initializing parallelism...")
etc = EMTaskCustomer(options.parallel,
module="e2spt_tiltrefine_oneiter.SptTltRefineTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
print("{} jobs".format(nptcl))
infos = [[] for i in range(num_cpus)]
for i, info in enumerate(pinfo):
infos[i % num_cpus].append([i, info])
tids = []
for info in infos:
task = SptTltRefineTask(info, threedname, options)
if options.debug:
task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
dics = [0] * nptcl
for i in tids:
ret = etc.get_results(i)[1]
for r in ret:
#print(r)
ii = r.pop("idx")
dics[ii] = r
del etc
allscr = [d["score"] for d in dics]
if options.smooth > 0 or options.defocus:
### need to add per tilt smoothing later...
s = np.array(allscr)
oname = lname.replace(".lst", "_score.hdf")
ss = from_numpy(s).copy()
ss.write_image(oname)
return
maxl = np.max([len(s) for s in allscr])
maxv = np.max(np.concatenate(allscr))
for s in allscr:
s.extend([maxv] * (maxl - len(s)))
allscr = np.array(allscr)
#print(np.min(allscr), np.mean(allscr), np.max(allscr), np.std(allscr))
if options.skipm3d:
pass
else:
allscr = 2 - allscr
allscr -= np.min(allscr)
allscr /= np.max(allscr)
if maxl > 1:
mx = np.max(allscr, axis=1)[:, None]
allscr = np.exp(allscr * 20)
allscr = allscr / np.sum(allscr, axis=1)[:, None]
allscr *= mx
try:
os.remove(lname)
except:
pass
lout = LSXFile(lname, False)
for i, dc in enumerate(dics):
lc = ""
if isinstance(dc["xform.align3d"], list):
alilist = dc["xform.align3d"]
scorelist = dc["score"]
else:
alilist = [dc["xform.align3d"]]
scorelist = [dc["score"]]
for j, xf in enumerate(alilist):
d = xf.get_params("eman")
d["score"] = float(allscr[i, j])
if d["score"] > .05 or j == 0:
lc = lc + str(d) + ';'
l = pinfo[i]
lout.write(-1, l[0], l[1], lc[:-1])
lout = None
pb = options.padby
if options.parallel.startswith("mpi") and len(dics) > 10000:
m3dpar = "--parallel {}".format(options.parallel)
else:
m3dpar = ""
if options.seedmap:
seed = "--seedmap " + threedname
else:
seed = ""
cmd = "e2make3dpar.py --input {inp} --output {out} --pad {pd} --padvol {pdv} --threads {trd} --outsize {bx} --apix {apx} --mode trilinear --keep {kp} --sym {sm} {seed} {par}".format(
inp=lname,
out=threedout,
bx=bxsz,
pd=int(bxsz * pb),
pdv=int(bxsz * pb),
apx=apix,
kp=options.keep,
sm=options.sym,
trd=options.threads,
par=m3dpar,
seed=seed)
if options.skipm3d:
print("Skipping 3D reconstruction")
else:
run(cmd)
E2end(logid)
class EMParallelProject3D:
def __init__(self, options, fsp, sym, start, modeln=0, logger=None):
"""
@param options the options produced by (options, args) = parser.parse_args()
@param args the options produced by (options, args) = parser.parse_args()
@param logger and EMAN2 logger, i.e. logger=E2init(sys.argv)
assumes you have already called the check function.
"""
self.options = options
self.args = fsp
self.sym = sym
self.logger = logger
self.start = start
self.modeln = modeln
from EMAN2PAR import EMTaskCustomer
self.etc = EMTaskCustomer(options.parallel)
print "Precache ", fsp
self.etc.precache([fsp])
self.num_cpus = self.etc.cpu_est()
print self.num_cpus, " total CPUs available"
if self.num_cpus > 64: # upper limit
self.num_cpus = 64
self.__task_options = None
def __init_memory(self, options):
"""
"""
sym_object = parsesym(self.sym)
[og_name, og_args] = parsemodopt(options.orientgen)
self.eulers = sym_object.gen_orientations(og_name, og_args)
def __get_task_options(self, options):
if self.__task_options == None:
d = {}
d["projector"] = parsemodopt(options.projector)
d["prethreshold"] = options.prethreshold
self.__task_options = d
return self.__task_options
def execute(self):
# from EMAN2PAR import EMTaskCustomer
if len(self.options.parallel) > 1:
self.__init_memory(self.options)
num_tasks = self.num_cpus
# In the worst case we can only spawn as many tasks as there are eulers
if self.num_cpus > len(self.eulers):
num_tasks = len(self.eulers)
eulers_per_task = len(self.eulers) / num_tasks
resid_eulers = len(self.eulers) - eulers_per_task * num_tasks # we can distribute the residual evenly
first = 0
task_customers = []
tids = []
# self.etc=EMTaskCustomer(self.options.parallel)
for i in xrange(0, num_tasks):
last = first + eulers_per_task
if resid_eulers > 0:
last += 1
resid_eulers -= 1
tmp_eulers = self.eulers[first:last]
indices = range(first, last)
data = {}
data["input"] = ("cache", self.args, 0)
data["eulers"] = tmp_eulers
data["indices"] = indices
task = EMProject3DTaskDC(data=data, options=self.__get_task_options(self.options))
# print "Est %d CPUs"%etc.cpu_est()
tid = self.etc.send_task(task)
# print "Task submitted tid=",tid
tids.append(tid)
first = last
print "Task ids are", tids
while 1:
print len(tids), "projection tasks left in main loop"
st_vals = self.etc.check_task(tids)
for i in xrange(len(tids) - 1, -1, -1):
st = st_vals[i]
if st == 100:
tid = tids[i]
rslts = self.etc.get_results(tid)
if not self.__write_output_data(rslts[1]):
print "There was a problem with the task of id", tid
if self.logger != None:
E2progress(self.logger, 1.0 - len(tids) / float(num_tasks))
if self.options.verbose > 0:
print "%d/%d\r" % (num_tasks - len(tids), num_tasks)
sys.stdout.flush()
print "Task", tids.pop(i), "completed"
print "These tasks are remaining:", tids
if len(tids) == 0:
break
time.sleep(5)
return len(self.eulers)
else:
raise NotImplementedError(
"The parallelism option you specified (%s) is not suppored" % self.options.parallel
)
def __write_output_data(self, rslts):
for idx, image in rslts.items():
if not isinstance(image, EMData):
continue # this is here because we get the dimensions of the database as a key (e.g. '40x40x1').
image["model_id"] = self.modeln
if self.options.append:
image.write_image(self.options.outfile, -1)
else:
image.write_image(self.options.outfile, idx + self.start)
return True
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_average.py [options]
Note that this program is not part of the original e2spt hierarchy, but is part of an experimental refactoring.
Will read metadata from the specified spt_XX directory, as produced by e2spt_align.py, and average a selected subset of subtomograms in the predetermined orientation.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--threads",
default=4,
type=int,
help=
"Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present."
)
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=-1)
parser.add_argument(
"--simthr",
default=-0.1,
type=float,
help=
"Similarity is smaller for better 'quality' particles. Specify the highest value to include from e2spt_hist.py. Default -0.1"
)
parser.add_argument(
"--keep",
default=-1,
type=float,
help="fraction of particles to keep. will overwrite simthr if set.")
parser.add_argument(
"--replace",
type=str,
default=None,
help=
"Replace the input subtomograms used for alignment with the specified file (used when the aligned particles were masked or filtered)"
)
parser.add_argument(
"--outfile",
type=str,
default=None,
help=
"Normally even/odd and overall outputs automatically generated. If specified, only overall file written with this specified filename. Suppresses postprocessing."
)
parser.add_argument(
"--wedgesigma",
type=float,
help=
"Threshold for identifying missing data in Fourier space in terms of standard deviation of each Fourier shell. Default 3.0",
default=3.0)
parser.add_argument(
"--minalt",
type=float,
help="Minimum alignment altitude to include. Default=0",
default=0)
parser.add_argument(
"--maxalt",
type=float,
help="Maximum alignment altitude to include. Deafult=180",
default=180)
parser.add_argument(
"--maxtilt",
type=float,
help=
"Explicitly zeroes data beyond specified tilt angle. Assumes tilt axis exactly on Y and zero tilt in X-Y plane. Default 90 (no limit).",
default=90.0)
parser.add_argument(
"--listfile",
type=str,
help=
"Specify a filename containing a list of integer particle numbers to include in the average, one per line, first is 0. Additional exclusions may apply.",
default=None)
parser.add_argument(
"--automaskexpand",
default=-1,
type=int,
help=
"Default=boxsize/20. Specify number of voxels to expand mask before soft edge. Use this if low density peripheral features are cut off by the mask.",
guitype='intbox',
row=12,
col=1,
rowspan=1,
colspan=1,
mode="refinement[-1]")
parser.add_argument(
"--symalimasked",
type=str,
default=None,
help=
"This will translationally realign each asymmetric unit to the specified (usually masked) reference "
)
parser.add_argument(
"--sym",
type=str,
default="c1",
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly."
)
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder containing current results (default = highest spt_XX)"
)
parser.add_argument("--skippostp",
action="store_true",
default=False,
help="Skip post process steps (fsc, mask and filters)")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default=None)
(options, args) = parser.parse_args()
if options.path == None:
fls = [
int(i[-2:]) for i in os.listdir(".")
if i[:4] == "spt_" and len(i) == 6 and str.isdigit(i[-2:])
]
if len(fls) == 0:
print("Error, cannot find any spt_XX folders")
sys.exit(2)
options.path = "spt_{:02d}".format(max(fls))
if options.verbose: print("Working in : ", options.path)
if options.iter < 0:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0:
print("Cannot find a {}/particle_parms* file".format(options.path))
sys.exit(2)
options.iter = max(fls)
if options.verbose: print("Using iteration ", options.iter)
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, options.iter))
else:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0:
print("Cannot find a {}/particle_parms* file".format(options.path))
sys.exit(2)
mit = max(fls)
if options.iter > mit:
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, mit))
print(
"WARNING: no particle_parms found for iter {}, using parms from {}"
.format(options.iter, mit))
else:
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, options.iter))
if options.listfile != None:
plist = set([int(i) for i in open(options.listfile, "r")])
NTHREADS = max(options.threads + 1,
2) # we have one thread just writing results
logid = E2init(sys.argv, options.ppid)
# jsd=Queue.Queue(0)
# filter the list of particles to include
keys = list(angs.keys())
if options.listfile != None:
keys = [i for i in keys if eval(i)[1] in plist]
if options.verbose:
print("{}/{} particles based on list file".format(
len(keys), len(list(angs.keys()))))
newkey = []
newang = {}
if options.keep > 0 and options.keep <= 1:
score = [float(angs[k]["score"]) for k in keys]
options.simthr = np.sort(score)[int(len(score) * options.keep) - 1]
print("Keeping {:.0f}% particles with score below {:.2f}".format(
options.keep * 100, options.simthr))
for k in keys:
val = angs[k]
if type(val) == list:
val = val[0]
if val["score"] <= options.simthr and inrange(
options.minalt, val["xform.align3d"].get_params("eman")["alt"],
options.maxalt):
newkey.append(k)
newang[k] = val
if options.verbose:
print("{}/{} particles after filters".format(len(newkey),
len(list(angs.keys()))))
keys = newkey
angs = newang
if options.parallel and options.symalimasked == None:
#print("running in mpi mode. This is experimental, so please switch back to threading if anything goes wrong...")
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_average.SptavgTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
data = [[], []] ## even/odd
if "eo" in angs[keys[0]]:
print("Reading even/odd subset from json file...")
for i, k in enumerate(keys):
src, ii = eval(k)[0], eval(k)[1]
eo = int(angs[k]["eo"])
data[eo].append([src, ii, angs[k]["xform.align3d"]])
else:
for i, k in enumerate(keys):
src, ii = eval(k)[0], eval(k)[1]
data[ii % 2].append([src, ii, angs[k]["xform.align3d"]])
#### check and save size of particle
fsp, i, xf = data[0][0]
b = EMData(fsp, i, True)
sz = options.boxsz = b["ny"]
avgs = []
for ieo, eo in enumerate(["even", "odd"]):
print("Averaging {}...".format(eo))
nbatch = min(len(data[ieo]) // 4, num_cpus)
tasks = [data[ieo][i::nbatch] for i in range(nbatch)]
print("{} particles in {} jobs".format(len(data[ieo]), len(tasks)))
tids = []
for t in tasks:
task = SptavgTask(t, options)
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if np.min(st_vals) == 100: break
time.sleep(5)
output = EMData(sz, sz, sz)
normvol = EMData((sz // 2 + 1) * 2, sz, sz)
output.to_zero()
output.do_fft_inplace()
avg = Averagers.get("mean")
normvol.to_zero()
for i in tids:
threed, norm = etc.get_results(i)[1]
#print(i, threed["mean"], threed["sigma"], norm["mean"], norm["sigma"])
#norm.div(len(tids))
threed.process_inplace("math.multamplitude", {"amp": norm})
avg.add_image(threed)
normvol.add(norm)
output = avg.finish()
normvol.process_inplace("math.reciprocal")
output.process_inplace("math.multamplitude", {"amp": normvol})
output.process_inplace("xform.phaseorigin.tocenter")
output.do_ift_inplace()
output.depad()
avgs.append(output)
ave, avo = avgs
else:
avg = [0, 0]
avg[0] = Averagers.get(
"mean.tomo",
{"thresh_sigma": options.wedgesigma}) #,{"save_norm":1})
avg[1] = Averagers.get("mean.tomo",
{"thresh_sigma": options.wedgesigma})
# Rotation and insertion are slow, so we do it with threads.
if options.symalimasked != None:
if options.replace != None:
print("Error: --replace cannot be used with --symalimasked")
sys.exit(1)
alimask = EMData(options.symalimasked)
thrds = [
threading.Thread(target=rotfnsym,
args=(avg[eval(k)[1] % 2], eval(k)[0],
eval(k)[1], angs[k]["xform.align3d"],
options.sym, alimask, options.maxtilt,
options.verbose))
for i, k in enumerate(keys)
]
else:
# Averager isn't strictly threadsafe, so possibility of slight numerical errors with a lot of threads
if options.replace != None:
thrds = [
threading.Thread(target=rotfn,
args=(avg[eval(k)[1] % 2],
options.replace, eval(k)[1],
angs[k]["xform.align3d"],
options.maxtilt, options.verbose))
for i, k in enumerate(keys)
]
else:
thrds = [
threading.Thread(target=rotfn,
args=(avg[eval(k)[1] % 2], eval(k)[0],
eval(k)[1],
angs[k]["xform.align3d"],
options.maxtilt, options.verbose))
for i, k in enumerate(keys)
]
print(len(thrds), " threads")
thrtolaunch = 0
while thrtolaunch < len(thrds) or threading.active_count() > 1:
# If we haven't launched all threads yet, then we wait for an empty slot, and launch another
# note that it's ok that we wait here forever, since there can't be new results if an existing
# thread hasn't finished.
if thrtolaunch < len(thrds):
while (threading.active_count() == NTHREADS):
time.sleep(.1)
if options.verbose:
print("Starting thread {}/{}".format(
thrtolaunch, len(thrds)))
thrds[thrtolaunch].start()
thrtolaunch += 1
else:
time.sleep(1)
#while not jsd.empty():
#fsp,n,ptcl=jsd.get()
#avg[n%2].add_image(ptcl)
for t in thrds:
t.join()
ave = avg[0].finish() #.process("xform.phaseorigin.tocenter").do_ift()
avo = avg[1].finish() #.process("xform.phaseorigin.tocenter").do_ift()
# impose symmetry on even and odd halves if appropriate
if options.sym != None and options.sym.lower(
) != "c1" and options.symalimasked == None:
ave.process_inplace("xform.applysym", {
"averager": "mean.tomo",
"sym": options.sym
})
avo.process_inplace("xform.applysym", {
"averager": "mean.tomo",
"sym": options.sym
})
av = ave + avo
av.mult(0.5)
if options.outfile:
av.write_image(options.outfile)
sys.exit(0)
evenfile = "{}/threed_{:02d}_even.hdf".format(options.path, options.iter)
oddfile = "{}/threed_{:02d}_odd.hdf".format(options.path, options.iter)
combfile = "{}/threed_{:02d}.hdf".format(options.path, options.iter)
ave.write_image(evenfile, 0)
avo.write_image(oddfile, 0)
av.write_image(combfile, 0)
cmd = "e2proc3d.py {evenfile} {path}/fsc_unmasked_{itr:02d}.txt --calcfsc={oddfile}".format(
path=options.path,
itr=options.iter,
evenfile=evenfile,
oddfile=oddfile)
launch_childprocess(cmd)
#### skip post process in case we want to do this elsewhere...
if options.skippostp:
E2end(logid)
return
# final volume at this point is Wiener filtered
launch_childprocess(
"e2proc3d.py {combfile} {combfile} --process=filter.wiener.byfsc:fscfile={path}/fsc_unmasked_{itr:02d}.txt:snrmult=2"
.format(path=options.path, itr=options.iter, combfile=combfile))
# New version of automasking based on a more intelligent interrogation of the volume
vol = EMData(combfile)
nx = vol["nx"]
apix = vol["apix_x"]
md = vol.calc_radial_dist(old_div(nx, 2), 0, 1,
3) # radial max value per shell in real space
rmax = int(old_div(nx, 2.2)) # we demand at least 10% padding
vmax = max(md[:rmax]) # max value within permitted radius
# this finds the first radius where the max value @ r falls below overall max/4
# this becomes the new maximum mask radius
act = 0
mv = 0, 0
for i in range(rmax):
if md[i] > mv[0]:
mv = md[i], i # find the radius of the max val in range
if not act and md[i] < 0.9 * vmax: continue
act = True
if md[i] < 0.2 * vmax:
rmax = i
break
rmaxval = mv[1]
vmax = mv[0]
# excludes any spurious high values at large radius
vol.process_inplace("mask.sharp", {"outer_radius": rmax})
# automask
mask = vol.process(
"mask.auto3d", {
"threshold": vmax * .15,
"radius": 0,
"nshells":
int(nx * 0.05 + 0.5 + old_div(20, apix)) + options.automaskexpand,
"nmaxseed": 24,
"return_mask": 1
})
mask.process_inplace("filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, (40.0))})
mask.write_image("{path}/mask.hdf".format(path=options.path), 0)
# compute masked fsc and refilter
ave.mult(mask)
ave.write_image("{path}/tmp_even.hdf".format(path=options.path), 0)
avo.mult(mask)
avo.write_image("{path}/tmp_odd.hdf".format(path=options.path), 0)
av.mult(mask)
av.write_image(combfile, 0)
cmd = "e2proc3d.py {path}/tmp_even.hdf {path}/fsc_masked_{itr:02d}.txt --calcfsc={path}/tmp_odd.hdf".format(
path=options.path, itr=options.iter)
launch_childprocess(cmd)
# final volume is premasked and Wiener filtered based on the masked FSC
launch_childprocess(
"e2proc3d.py {combfile} {combfile} --process=filter.wiener.byfsc:fscfile={path}/fsc_masked_{itr:02d}.txt:snrmult=2"
.format(path=options.path, itr=options.iter, combfile=combfile))
E2end(logid)
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--ptclin",
type=str,
help="particle input",
default=None)
parser.add_argument("--ptclout",
type=str,
help="particle output",
default=None)
parser.add_argument("--ref",
type=str,
help="reference input",
default=None)
parser.add_argument("--threedout",
type=str,
help="map output",
default=None)
parser.add_argument("--keep",
type=float,
help="propotion of tilts to keep. default is 0.5",
default=0.5)
parser.add_argument("--threads",
type=int,
help="Number of CPU threads to use. Default is 12.",
default=12)
parser.add_argument(
"--parallel",
type=str,
help="Thread/mpi parallelism to use. Default is thread:12",
default="thread:12")
parser.add_argument(
"--debug",
action="store_true",
default=False,
help=
"Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)"
)
parser.add_argument("--transonly",
action="store_true",
default=False,
help="only refine translation")
parser.add_argument(
"--fromscratch",
action="store_true",
default=False,
help=
"align from scratch and ignore previous particle transforms. for spt mostly. will include mirror"
)
parser.add_argument("--refineastep",
type=float,
help="Mean angular variation for refine alignment",
default=2.)
parser.add_argument("--refinentry",
type=int,
help="number of starting points for refine alignment",
default=4)
parser.add_argument("--maxshift",
type=int,
help="maximum shift allowed",
default=8)
parser.add_argument("--padby",
type=float,
default=2.0,
help="pad by factor. default is 2")
parser.add_argument("--maxres",
type=float,
default=-1,
help="max resolution for cmp")
parser.add_argument(
"--sym",
type=str,
help="symmetry. will use symmetry from spt refinement by default",
default="c1")
parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
lstname = options.ptclin
threedname = options.ref
lname = options.ptclout
threedout = options.threedout
lst = LSXFile(lstname, True)
m = EMData(threedname)
bxsz = m["nx"]
apix = m["apix_x"]
if options.maxres > 0:
options.shrink = max(1, int(options.maxres / apix * .3))
options.shrink = min(options.shrink, bxsz // 48)
print("Will shrink by {} and filter to {:.0f} A. Box size {}".format(
options.shrink, options.maxres, bxsz // options.shrink))
else:
options.shrink = 1
#m.process_inplace('normalize.edgemean')
pinfo = []
if options.debug: nptcl = options.threads * 8
else: nptcl = lst.n
for i in range(nptcl):
pinfo.append(lst.read(i))
lst = None
print("Initializing parallelism...")
etc = EMTaskCustomer(options.parallel,
module="e2spt_tiltrefine_oneiter.SptTltRefineTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
print("{} jobs".format(nptcl))
infos = [[] for i in range(num_cpus)]
for i, info in enumerate(pinfo):
infos[i % num_cpus].append([i, info])
tids = []
for info in infos:
task = SptTltRefineTask(info, threedname, options)
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print(st_vals)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
dics = [0] * nptcl
for i in tids:
ret = etc.get_results(i)[1]
for r in ret:
#print(r)
ii = r.pop("idx")
dics[ii] = r
del etc
allscr = np.array([d["score"] for d in dics])
print(np.min(allscr), np.mean(allscr), np.max(allscr), np.std(allscr))
allscr = 2 - allscr
s = allscr.copy()
s -= np.mean(s)
s /= np.std(s)
clp = 2
ol = abs(s) > clp
print("Removing {} outliers from {} particles..".format(
np.sum(ol), len(s)))
s = (s + clp) / clp / 2
s[ol] = 0
allscr = s
allscr -= np.min(allscr) - 1e-5
allscr /= np.max(allscr)
try:
os.remove(lname)
except:
pass
lout = LSXFile(lname, False)
for i, dc in enumerate(dics):
d = dc["xform.align3d"].get_params("eman")
d["score"] = float(allscr[i])
l = pinfo[i]
lout.write(-1, l[0], l[1], str(d))
lout = None
pb = options.padby
if options.parallel.startswith("mpi") and len(dics) > 10000:
m3dpar = "--parallel {}".format(options.parallel)
else:
m3dpar = ""
cmd = "e2make3dpar.py --input {inp} --output {out} --pad {pd} --padvol {pdv} --threads {trd} --outsize {bx} --apix {apx} --mode gauss_2 --keep {kp} --sym {sm} {par}".format(
inp=lname,
out=threedout,
bx=bxsz,
pd=int(bxsz * pb),
pdv=int(bxsz * pb),
apx=apix,
kp=options.keep,
sm=options.sym,
trd=options.threads,
par=m3dpar)
run(cmd)
E2end(logid)
def main():
parser = EMArgumentParser(usage="")
parser.add_argument("--output", default="threed.hdf", help="Output reconstructed volume file name.")
parser.add_argument("--input", default=None, help="The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion")
parser.add_argument("--sym", dest="sym", default="c1", help="Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct.")
parser.add_argument("--pad", default=-1,type=int, help="Will zero-pad images to the specifed size. ")
parser.add_argument("--outsize", default=-1, type=int, help="Defines the dimensions of the final volume written to disk")
parser.add_argument("--keep", type=str, dest="keep", help="The fraction of slices to keep, based on quality scores (1.0 = use all slices).",default=".9")
parser.add_argument("--no_wt", action="store_true", dest="no_wt", default=False, help="This argument turns automatic weighting off causing all images to be weighted by 1.")
parser.add_argument("--mode", type=str, default="trilinear", help="Fourier reconstruction 'mode' to use. The default should not normally be changed.")
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--apix",metavar="A/pix",type=float,help="A/pix value for output, overrides automatic values",default=None)
parser.add_argument("--tidrange", type=str,help="Range of tilt id to include for particles from tilt series. Specify two integers separated by ','.", default="-1,-1")
parser.add_argument("--ref", type=str,help="Weight each particle using a specified reference map.", default=None)
parser.add_argument("--minres", type=float,help="minimum resolution to compare when weighting by a reference map.", default=50)
parser.add_argument("--maxres", type=float,help="maximum resolution to compare when weighting by a reference map.", default=-1)
parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use without shared memory. Each worker will reconstruct a map with a subset of particles and the results from workers will be averaged together with the corresponding Fourier weighting. Along with --threads, this allows having one worker per node using multiple threads.", default="thread:1")
parser.add_argument("--threads", type=int,help="Number of threads using shared memory.", default=1)
parser.add_argument("--setsf", type=str,help="Set structure factor from text file", default=None)
parser.add_argument("--debug", action="store_true", default=False, help="Turn on debug mode. This will only process a small subset of the data.")
parser.add_argument("--clsid", default=None, type=str, help="Only reconstruct a class of particles. Also take even/odd to reconstruct subsets of particles.")
parser.add_argument("--listsel", default=None, type=str, help="only reconstruct particles of indices from the given list in a text file.")
parser.add_argument("--p3did", type=int, help="only reconstruct images for one 3d particle of the given ID.",default=-1)
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
(options, args) = parser.parse_args()
logger=E2init(sys.argv,options.ppid)
time0=time.time()
# so it recognize even/odd or 0/1
if options.clsid:
options.clsid=options.clsid.replace("even","0").replace("odd","1")
try: options.clsid=int(options.clsid)
except:options.clsid=-1
else:
options.clsid=-1
if options.sym.startswith("h"):
options.sym="c1"
options.keep=[float(k) for k in options.keep.split(',')]
if len(options.keep)<3: options.keep=[options.keep[0]]*3
# get basic image parameters
tmp=EMData(options.input,0,True)
boxsz=tmp["nx"]
if options.apix!=None : apix=options.apix
else : apix=tmp["apix_x"]
if options.pad<0:
options.pad=good_size(boxsz*1.5)
if options.outsize<0:
options.outsize=boxsz
options.tidrange=[int(i) for i in options.tidrange.split(',')]
if options.tidrange[0]>=0:
print("including tilt ids from {} to {}".format(options.tidrange[0], options.tidrange[1]))
data=initialize_data(options.input, options)
padvol=options.pad
from EMAN2PAR import EMTaskCustomer
if options.ref:
print("weighting by reference...")
ref=EMData(options.ref)
ny=options.pad
by=ref["ny"]
ref=ref.get_clip(Region((by-ny)/2, (by-ny)/2,(by-ny)/2, ny, ny,ny))
ref=ref.do_fft()
ref.process_inplace("xform.phaseorigin.tocenter")
ref.process_inplace("xform.fourierorigin.tocenter")
etc=EMTaskCustomer(options.parallel, module="e2spa_make3d.WeightptclTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
tasks=[data[i::num_cpus] for i in range(num_cpus)]
print("{} jobs".format(len(tasks)))
tids=[]
for t in tasks:
task = WeightptclTask(t, ref, options)
if options.debug:
task.execute(print)
return
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
E2progress(logger, .5*np.mean(st_vals)/100.)
if np.min(st_vals) == 100: break
time.sleep(5)
wts=[0]*len(data)
for i in tids:
wt=etc.get_results(i)[1]
for w in wt:
try:
wts[w[0]]=w[1]
except:
print(w)
wts=np.array(wts)
wts[wts<0]=0
if options.minres>0:
r0=int(apix*ny/options.minres)
wts[:,:r0]=np.mean(wts[:,:r0], axis=0)
if options.maxres>0:
r1=int(apix*ny/options.maxres)
wts[:,r1:]=np.mean(wts[:,r1:], axis=0)
print(wts.shape)
del etc
#print(r0,r1)
#scrs=np.mean(wts[:,r0:r1], axis=1)
#if options.keep<1:
#thr=np.sort(scrs)[int(len(scrs)*(1-options.keep))-1]
#scrs[scrs<thr]=-1
for i,d in enumerate(data):
d["curve"]=wts[i]
#d["weight"]=float(scrs[i])
etc=EMTaskCustomer(options.parallel, module="e2spa_make3d.Make3dTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
tasks=[data[i::num_cpus] for i in range(num_cpus)]
print("{} jobs".format(len(tasks)))
tids=[]
for t in tasks:
task = Make3dTask(t, options)
if options.debug:
task.execute(print)
return
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
E2progress(logger, .5+.5*np.mean(st_vals)/100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output=EMData(padvol, padvol, padvol)
normvol=EMData(padvol//2+1, padvol, padvol)
output.to_zero()
output.do_fft_inplace()
normvol.to_zero()
for i in tids:
threed, norm=etc.get_results(i)[1]
threed.process_inplace("math.multamplitude", {"amp":norm})
output.add(threed)
normvol.add(norm)
normvol.process_inplace("math.reciprocal")
output.process_inplace("math.multamplitude", {"amp":normvol})
output.do_ift_inplace()
output.depad()
output.process_inplace("xform.phaseorigin.tocenter")
del etc
if options.verbose>0 : print("Finished Reconstruction")
output["apix_x"]=output["apix_y"]=output["apix_z"]=apix
sz=options.outsize
output.clip_inplace(Region((padvol-sz)//2,(padvol-sz)//2,(padvol-sz)//2,sz,sz,sz))
if os.path.isfile(options.output):
os.remove(options.output)
output.write_image(options.output,0)
if options.setsf:
launch_childprocess("e2proc3d.py {} {} --setsf {}".format(options.output,options.output,options.setsf))
if options.verbose>0:
print("Output File: "+options.output)
E2end(logger)
print("Reconstruction finishend ({:.1f} s)".format(time.time()-time0))
def main():
usage = """
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
)
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=0)
parser.add_argument(
"--goldcontinue",
action="store_true",
help=
"Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
default=False)
parser.add_argument(
"--sym",
type=str,
default="c1",
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly."
)
parser.add_argument(
"--maxres",
type=float,
help="Maximum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument(
"--minres",
type=float,
help="Minimum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default="thread:4")
parser.add_argument("--fromscratch",
action="store_true",
help=".",
default=False)
parser.add_argument(
"--use3d",
action="store_true",
help="use projection of 3d particles instead of 2d sub tilt series",
default=False)
parser.add_argument("--debug",
action="store_true",
help=".",
default=False)
parser.add_argument(
"--plst",
type=str,
default=None,
help=
"list of 2d particle with alignment parameters. will reconstruct before alignment."
)
parser.add_argument("--maxshift",
type=int,
help="maximum shift. default box size/6",
default=-1)
parser.add_argument(
"--maxang",
type=int,
help=
"maximum angle difference from starting point. ignored when fromscratch is on",
default=30)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv, options.ppid)
options.ref = args[1]
options.info3dname = "{}/particle_info_3d.lst".format(options.path)
options.info2dname = "{}/particle_info_2d.lst".format(options.path)
tasks = load_lst_params(args[0])
for i, t in enumerate(tasks):
t["ii"] = i
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_align_subtlt.SptAlignTask")
num_cpus = etc.cpu_est()
options.nowtime = time.time()
if options.debug:
tasks = tasks[:num_cpus * 4]
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob = num_cpus
tids = []
for i in range(njob):
t = tasks[i::njob]
task = SptAlignTask(t, options)
if options.debug:
ret = task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output3d = [None] * len(tasks)
output2d = [None] * len(tasks)
for i in tids:
rets = etc.get_results(i)[1]
for r in rets:
output3d[r[0]] = r[1]
output2d[r[0]] = r[2]
del etc
output2d = sum(output2d, [])
fm3d = f"{options.path}/aliptcls3d_{options.iter:02d}.lst"
save_lst_params(output3d, fm3d)
fm2d = f"{options.path}/aliptcls2d_{options.iter:02d}.lst"
save_lst_params(output2d, fm2d)
E2end(logid)
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_pos_argument(
name="tomograms",
help=
"Specify tomograms from which you wish to extract boxed particles.",
default="",
guitype='filebox',
browser="EMTomoBoxesTable(withmodal=True,multiselect=True)",
row=0,
col=0,
rowspan=1,
colspan=2,
mode="extract")
parser.add_argument("--boxsz_unbin",
type=int,
help="box size in unbinned tomogram",
default=-1,
guitype='intbox',
row=2,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--label",
type=str,
help=
"Only extract particle with this name. Leave blank to extract all particles.",
default=None,
guitype='strbox',
row=2,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--newlabel",
type=str,
help=
"Label of output particles. Same as original particle label by default.",
default="",
guitype='strbox',
row=6,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock1",
help='Just a visual separation',
title="Options",
row=3,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--threads",
type=int,
help="threads",
default=12,
guitype='intbox',
row=4,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--maxtilt",
type=int,
help="max tilt",
default=100,
guitype='intbox',
row=4,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--padtwod",
type=float,
help="padding factor",
default=2.0,
guitype='floatbox',
row=5,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--noctf",
action="store_true",
default=False,
help="skip ctf correction.")
parser.add_argument(
"--wiener",
action="store_true",
default=False,
help="wiener filter the particles using ctf information..",
guitype='boolbox',
row=6,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--alltomograms",
action="store_true",
default=False,
help="use all tomograms.",
guitype='boolbox',
row=1,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument("--dotest",
action="store_true",
default=False,
help="only make 1 batch of subtomograms for testing")
parser.add_argument(
"--shrink",
type=float,
help=
"Shrinking factor for output particles. 1.5 or integers allowed. Default is 1 (no shrink).",
default=1,
guitype='floatbox',
row=8,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--tltkeep",
type=float,
help=
"keep a fraction of tilt images with good score determined from tomogram reconstruction",
default=1.0,
guitype='floatbox',
row=8,
col=1,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--rmbeadthr",
type=float,
help=
"remove 2d particles with high contrast object beyond N sigma at 100A. Note that this may result in generating fewer particles than selected. Default is -1 (include all particles). 0.5 might be a good choice for removing gold beads but may need some testing...",
default=-1,
guitype='floatbox',
row=9,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock2",
help='Just a visual separation',
title="Extract from curves",
row=10,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--curves",
type=int,
default=-1,
help="specify curve id to extract particles from saved curves. ",
guitype='intbox',
row=11,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--curves_overlap",
type=float,
help=
"fraction of overlap when generating particle along curves. default is 0.5",
default=0.5,
guitype='floatbox',
row=12,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_header(name="orblock3",
help='Just a visual separation',
title="Re-extraction from spt",
row=13,
col=0,
rowspan=1,
colspan=1,
mode="extract")
parser.add_argument(
"--jsonali",
type=str,
help=
"re-extract particles using a particle_param_xx json file from a spt alignment",
default="",
guitype='filebox',
browser="EMBrowserWidget(withmodal=True,multiselect=False)",
row=14,
col=0,
rowspan=1,
colspan=2,
mode="extract")
parser.add_argument(
"--mindist",
type=float,
help="minimum distance between particles in A. for reextraction only",
default=10)
parser.add_argument(
"--keep",
type=float,
help=
"fraction of particles to keep fron previous alignment. for reextraction only.",
default=.9)
parser.add_argument("--postproc",
type=str,
help="processor after 3d particle reconstruction",
default="")
parser.add_argument(
"--postmask",
type=str,
help=
"masking after 3d particle reconstruction. The mask is transformed if json ",
default="")
parser.add_argument(
"--textin",
type=str,
help="text file for particle coordinates. do not use..",
default=None)
parser.add_argument(
"--compress",
type=int,
help=
"Bits to keep for compression. Not compatible with saveint. default is -1 meaning no compression. 8 bit seems fine...",
default=-1)
parser.add_argument(
"--saveint",
action="store_true",
default=False,
help=
"save particles in uint8 format to save space. still under testing.")
parser.add_argument("--norewrite",
action="store_true",
default=False,
help="skip existing files. do not rewrite.")
parser.add_argument("--parallel", type=str, help="parallel", default="")
#parser.add_argument("--alioffset", type=str,help="coordinate offset when re-extract particles. (x,y,z)", default="0,0,0", guitype='strbox', row=12, col=0,rowspan=1, colspan=1, mode="extract")
parser.add_argument("--postxf",
type=str,
help="a file listing post transforms",
default="")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-2)
parser.add_argument("--shrink3d",
type=int,
help="Only shrink 3d particles by x factor",
default=-1)
parser.add_argument("--verbose", type=int, help="verbose", default=1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
allxfs = {}
allinfo = {}
if options.textin:
allxfs, allinfo = parse_text(options)
if len(options.jsonali) > 0:
allxfs, allinfo = parse_json(options)
if options.alltomograms:
fld = "tomograms/"
args = [fld + f for f in os.listdir(fld) if f.endswith(".hdf")]
### do not count a tomogram twice when multiple versions exist
uq, uid = np.unique([base_name(a) for a in args], return_index=True)
args = [args[i] for i in uid]
if len(args) == 0 and len(allxfs) == 0:
print("No input. Exit.")
return
if options.parallel != "":
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_extract.SptExtractTask")
num_cpus = etc.cpu_est()
options.nowtime = time.time()
print(
"Running in parallel mode with {} workers, each running {} threads"
.format(num_cpus, options.threads))
njob = num_cpus
options.verbose = 0
if len(allxfs) > 0:
fnames = list(allxfs.keys())
elif len(args) > 0:
fnames = args
tids = []
for i in range(njob):
task = SptExtractTask(fnames[i::njob], options, allxfs, allinfo)
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if np.min(st_vals) == 100: break
#print(st_vals)
time.sleep(1)
else:
if len(allxfs) > 0:
for fname in allxfs.keys():
#### it seems options is changed inplace somewhere...
(options, args) = parser.parse_args()
do_extraction(fname, options, allxfs[fname], allinfo[fname])
elif len(args) > 0:
for a in args:
do_extraction(a, options)
return
print("Particle extraction finished.")
E2end(logid)
return
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--ptclin",
type=str,
help="particle input",
default=None)
parser.add_argument("--ptclout",
type=str,
help="particle output",
default=None)
parser.add_argument("--ref",
type=str,
help="reference input",
default=None)
parser.add_argument(
"--parallel",
type=str,
help="Thread/mpi parallelism to use. Default is thread:12",
default="thread:12")
parser.add_argument(
"--debug",
action="store_true",
default=False,
help=
"Turn on debug mode. This will only process a small subset of the data"
)
#parser.add_argument("--maxshift", type=int,help="maximum shift allowed", default=-1)
parser.add_argument(
"--localrefine",
type=int,
default=-1,
help="local refinement. larger value correspond to smaller local region"
)
parser.add_argument("--goldcontinue",
action="store_true",
default=False,
help="split even/odd subset and references.")
#parser.add_argument("--ctfweight", action="store_true", default=False ,help="weight by ctf. not used yet...")
#parser.add_argument("--slow", action="store_true", default=False ,help="slow but finer search")
parser.add_argument("--maxres",
type=float,
default=-1,
help="max resolution for cmp")
parser.add_argument("--minrespx",
type=int,
default=4,
help="skip the first x pixel in fourier space")
parser.add_argument("--sym", type=str, help="symmetry. ", default="c1")
parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
#m=EMData(options.ref)
#bxsz=m["nx"]
#apix=m["apix_x"]
options.shrink = 1
pinfo = load_lst_params(options.ptclin)
nptcl = len(pinfo)
#if options.maxshift<0:
#options.maxshift=bxsz//2
print("Initializing parallelism...")
etc = EMTaskCustomer(options.parallel, module="e2spa_align.SpaAlignTask")
num_cpus = etc.cpu_est()
print("{} particles".format(nptcl))
print("{} total CPUs available".format(num_cpus))
if options.debug:
nptcl = min(4 * num_cpus, nptcl)
print("Debugging mode. running on one thread with 8 particles")
nbatch = min(nptcl // 4, num_cpus)
infos = [[] for i in range(nbatch)]
for i, info in enumerate(pinfo):
infos[i % nbatch].append([i, info])
print("{} jobs, each with {:.1f} particles".format(
len(infos), np.mean([len(i) for i in infos])))
tids = []
for info in infos:
task = SpaAlignTask(info, options)
if options.debug:
task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output = [None] * nptcl
for i in tids:
ret = etc.get_results(i)[1]
for r in ret:
output[r[0]] = r[1]
del etc
fm = options.ptclout
save_lst_params(output, fm)
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_average.py <ref1> <ref2> ... [options]
Note that this program is not part of the original e2spt hierarchy, but is part of an experimental refactoring.
Will read metadata from the specified spt_XX directory, as produced by e2spt_align.py, and average a selected subset of subtomograms in the predetermined orientation.
This version of the program competes each particle against N reference volumes, and only averages it with the best match. Alignment parameters from a previous
e2spt_align run are used to define the coarse orientation, so the references must be similar and in the same orientation. Alignments are translationally adjusted only.
If --sym is specified, each possible symmetric orientation is tested starting with the exisiting alignment parameters, and only the best is kept.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=0)
parser.add_argument(
"--simthr",
default=-0.1,
type=float,
help=
"Similarity is smaller for better 'quality' particles. Specify the highest value to include from e2spt_hist.py. Default -0.1"
)
parser.add_argument(
"--simthr2",
default=0,
type=float,
help=
"Simlarity score for the best matching final alignment. Scaling may be different due to resolution limit. Default 0"
)
parser.add_argument(
"--replace",
type=str,
default=None,
help=
"Replace the input subtomograms used for alignment with the specified file (used when the aligned particles were masked or filtered)"
)
parser.add_argument(
"--wedgesigma",
type=float,
help=
"Threshold for identifying missing data in Fourier space in terms of standard deviation of each Fourier shell. Default 3.0",
default=3.0)
parser.add_argument(
"--minalt",
type=float,
help="Minimum alignment altitude to include. Default=0",
default=0)
parser.add_argument(
"--maxalt",
type=float,
help="Maximum alignment altitude to include. Deafult=180",
default=180)
parser.add_argument(
"--maxtilt",
type=float,
help=
"Explicitly zeroes data beyond specified tilt angle. Assumes tilt axis exactly on Y and zero tilt in X-Y plane. Default 90 (no limit).",
default=90.0)
parser.add_argument(
"--maxres",
type=float,
help=
"Lowpass filter applied to particles prior to alignment/averaging, resolution in A. Default disabled",
default=-1)
parser.add_argument(
"--listfile",
type=str,
help=
"Specify a filename containing a list of integer particle numbers to include in the average, one per line, first is 0. Additional exclusions may apply.",
default=None)
parser.add_argument(
"--shrinkcompare",
type=int,
help="Shrink factor for classification only (for speed)",
default=0)
parser.add_argument(
"--sym",
type=str,
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly. The structure will be expanded from this symmetry to c1.",
default="c1")
parser.add_argument(
"--applysym",
type=str,
help="Symmetry to apply to the structure after classification.",
default="c1")
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder containing current results (default = highest spt_XX)"
)
parser.add_argument(
"--parallel",
type=str,
default=None,
help="parallel mode. Not all functions are implemented yet..")
parser.add_argument(
"--threads",
default=4,
type=int,
help=
"Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present."
)
parser.add_argument("--maskclass",
type=str,
default=None,
help="Mask each reference before classification")
parser.add_argument("--mask",
type=str,
default=None,
help="Mask applied to final averages")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--noali",
action="store_true",
default=False,
help="Skip translational alignment.")
parser.add_argument(
"--symcopy",
action="store_true",
default=False,
help=
"Copy each particle for each asymetrical unit. need a maskclass to focus on one unit. do not work with applysym"
)
parser.add_argument("--nolstout",
action="store_true",
default=False,
help="Skip writting lst output.")
parser.add_argument("--sample",
type=int,
help="use only N samples.",
default=-1)
parser.add_argument("--randnclass",
type=int,
help="split into N random classes. ignore refs",
default=-1)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
if options.parallel != None and options.parallel[:6] == "thread":
options.threads = int(options.parallel.split(":")[1])
options.parallel = None
print("--parallel converted to --threads for efficiency")
if options.path == None:
options.path = num_path_last("spt_")
if options.verbose: print("Working in : ", options.path)
options.path = options.path.strip('/\\')
if options.iter <= 0:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0:
print("Cannot find a {}/particle_parms* file".format(options.path))
sys.exit(2)
options.iter = max(fls)
if options.verbose: print("Using iteration ", options.iter)
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, options.iter))
else:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0:
print("Cannot find a {}/particle_parms* file".format(options.path))
sys.exit(2)
mit = max(fls)
if options.iter > mit:
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, mit))
print(
"WARNING: no particle_parms found for iter {}, using parms from {}"
.format(options.iter, mit))
else:
angs = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, options.iter))
if options.listfile != None:
plist = set([int(i) for i in open(options.listfile, "r")])
NTHREADS = max(options.threads + 1,
2) # we have one thread just writing results
logid = E2init(sys.argv, options.ppid)
# filter the list of particles to include
keys = list(angs.keys())
if options.listfile != None:
keys = [i for i in keys if eval(i)[1] in plist]
if options.verbose:
print("{}/{} particles based on list file".format(
len(keys), len(list(angs.keys()))))
keys = [
k for k in keys if angs[k]["score"] <= options.simthr
and inrange(options.minalt, angs[k]["xform.align3d"].get_params("eman")
["alt"], options.maxalt)
]
if options.verbose:
print("{}/{} particles after filters".format(len(keys),
len(list(angs.keys()))))
if options.sample > 0:
rnd = np.arange(len(keys))
np.random.shuffle(rnd)
rnd = rnd[:options.sample]
keys = [keys[i] for i in rnd.tolist()]
print("using {} samples...".format(len(keys)))
if options.parallel:
print(
"Running in parallel mode. Warning, memory consumption may be high!"
)
if options.randnclass > 0:
nref = options.randnclass
else:
nref = len(args)
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_average_multi.SptavgmultTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
data = [] ## even/odd
for i, k in enumerate(keys):
src, ii = eval(k)[0], eval(k)[1]
data.append([src, ii, angs[k]["xform.align3d"]])
#### check and save size of particle
fsp, i, xf = data[0]
b = EMData(fsp, i, True)
sz = options.boxsz = b["ny"]
avgs = []
tasks = [data[i::num_cpus] for i in range(num_cpus)]
print("{} particles in {} jobs".format(len(data), len(tasks)))
tids = []
for t in tasks:
task = SptavgmultTask(t, args, options)
#task.execute(print)
#exit()
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
#print(tids)
if np.min(st_vals) == 100: break
time.sleep(5)
#dics=[0]*nptcl
threeds = []
normvols = []
stats = []
print("collecting from workers...")
for i in tids:
threed, norm, stat = etc.get_results(i)[1]
#print(len(threed), len(norm), threed[0]["sigma"])
threeds.append(threed)
normvols.append(norm)
stats.append(stat)
stats = np.vstack(stats)
stats = stats[np.argsort(stats[:, 0]), :]
np.savetxt(
"{}/avg_multi_{:02d}.txt".format(options.path, options.iter),
stats)
if not options.nolstout:
lsts = [
LSXFile(f"sets/{options.path}_{options.iter:02d}_{i:02d}.lst")
for i in range(nref)
]
for n, score, cls, x in stats:
lsts[int(cls)].write(-1, int(n), data[0][0])
lsts = None
avs = []
for r in range(nref):
output = EMData(sz, sz, sz)
normvol = EMData((sz // 2 + 1) * 2, sz, sz)
output.to_zero()
output.do_fft_inplace()
avg = Averagers.get("mean")
normvol.to_zero()
for k, thd in enumerate(threeds):
threed = thd[r]
norm = normvols[k][r]
threed.process_inplace("math.multamplitude", {"amp": norm})
avg.add_image(threed)
normvol.add(norm)
output = avg.finish()
normvol.process_inplace("math.reciprocal")
output.process_inplace("math.multamplitude", {"amp": normvol})
output.process_inplace("xform.phaseorigin.tocenter")
output.do_ift_inplace()
output.depad()
avs.append(output)
else:
print("Running in thread mode")
n = len(args)
args = [comma(i) for i in args]
refs = [EMData(i[0], i[1]) for i in args]
if options.maxres > 0:
for r in refs:
r.process_inplace(
"filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, options.maxres)})
if options.maskclass != None:
mask = EMData(options.maskclass)
for r in refs:
r.mult(mask)
jsd = queue.Queue(0)
avgs = [
Averagers.get("mean.tomo", {"thresh_sigma": options.wedgesigma})
for i in range(n)
]
if options.shrinkcompare > 1:
shrinkrefs = [
r.process("math.meanshrink", {"n": options.shrinkcompare})
for r in refs
]
else:
shrinkrefs = refs
# Rotation and insertion are slow, so we do it with threads.
# Averager isn't strictly threadsafe, so possibility of slight numerical errors with a lot of threads
if options.replace != None:
thrds = [
threading.Thread(target=rotfncompete,
args=(jsd, avgs, options.replace, eval(k)[1],
angs[k]["xform.align3d"], options.sym,
refs, shrinkrefs, options.maxtilt,
options.wedgesigma,
options.shrinkcompare, options.maxres,
options.simthr2, options.verbose))
for i, k in enumerate(keys)
]
else:
#thrds=[threading.Thread(target=rotfncompete,args=(jsd,avgs,eval(k)[0],eval(k)[1],angs[k]["xform.align3d"],options.sym,refs,shrinkrefs,options.maxtilt,options.wedgesigma,options.shrinkcompare,options.maxres,options.simthr2,options.verbose)) for i,k in enumerate(keys)]
thrds = [
threading.Thread(target=rotfncompete,
args=(jsd, avgs, eval(k)[0], eval(k)[1],
angs[k]["xform.align3d"], refs,
shrinkrefs, options))
for i, k in enumerate(keys)
]
print(len(thrds), " threads")
thrtolaunch = 0
out = open(
"{}/avg_multi_{:02d}.txt".format(options.path, options.iter), "w")
while thrtolaunch < len(
thrds) or threading.active_count() > 1 or not jsd.empty():
# If we haven't launched all threads yet, then we wait for an empty slot, and launch another
# note that it's ok that we wait here forever, since there can't be new results if an existing
# thread hasn't finished.
if thrtolaunch < len(thrds):
while (threading.active_count() == NTHREADS):
time.sleep(.1)
if options.verbose:
print("Starting thread {}/{}".format(
thrtolaunch, len(thrds)))
thrds[thrtolaunch].start()
thrtolaunch += 1
else:
time.sleep(1)
while not jsd.empty():
fspn, score, ref, sym = jsd.get()
out.write(
"{}\t{}\t{}\t{}\n".format(fspn, score, ref, sym)
) # Output columns are img #, best score, # of best ref, # of best sym
#avg[n%2].add_image(ptcl)
out.flush()
for t in thrds:
t.join()
avs = [i.finish() for i in avgs]
if options.mask: mask = EMData(options.mask)
for i, v in enumerate(avs):
if options.mask: v.mult(mask)
# v.write_image("{}/threed_{:02d}_{:02d}.hdf".format(options.path,options.iter,i),0)
v.write_compressed(
"{}/threed_{:02d}_{:02d}.hdf".format(options.path, options.iter,
i), 0, 12)
print("Done")
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
This program is part of the 'new' hierarchy of e2spt_ programs. It performs one iteration of a classical subtomogram refinement, ie - aligning particles with missing wedge to a reference in 3-D
The reference may be <volume> or <volume>,<n>
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--threads", default=4,type=int,help="Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.", guitype='intbox', row=24, col=2, rowspan=1, colspan=1, mode="refinement")
parser.add_argument("--iter",type=int,help="Iteration number within path. Default = start a new iteration",default=0)
parser.add_argument("--goldcontinue",action="store_true",help="Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",default=False)
parser.add_argument("--path",type=str,default=None,help="Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)")
parser.add_argument("--sym",type=str,default="c1",help="Symmetry of the input. Must be aligned in standard orientation to work properly.")
parser.add_argument("--maxres",type=float,help="Maximum resolution to consider in alignment (in A, not 1/A)",default=0)
parser.add_argument("--minres",type=float,help="Minimum resolution to consider in alignment (in A, not 1/A)",default=0)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
parser.add_argument("--maxshift", type=int, help="maximum shift for subtilt refine",default=8)
parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
parser.add_argument("--fromscratch",action="store_true",help=".",default=False)
parser.add_argument("--skipali3d",action="store_true",help="",default=False)
parser.add_argument("--skipali2d",action="store_true",help="",default=False)
parser.add_argument("--debug",action="store_true",help=".",default=False)
parser.add_argument("--plst",type=str,default=None,help="list of 2d particle with alignment parameters. will reconstruct before alignment.")
parser.add_argument("--smooth",type=float,help="smooth local motion by this factor. smoother local motion with larger numbers",default=-1)
parser.add_argument("--smoothN",type=float,help="number of neighboring particles used for smoothing. default 15",default=15)
(options, args) = parser.parse_args()
if options.path == None:
fls=[int(i[-2:]) for i in os.listdir(".") if i[:4]=="spt_" and len(i)==6 and str.isdigit(i[-2:])]
if len(fls)==0 : fls=[0]
options.path = "spt_{:02d}".format(max(fls)+1)
try: os.mkdir(options.path)
except: pass
if options.iter<=0 :
fls=[int(i[15:17]) for i in os.listdir(options.path) if i[:15]=="particle_parms_" and str.isdigit(i[15:17])]
if len(fls)==0 : options.iter=1
else: options.iter=max(fls)+1
if options.parallel==None:
options.parallel="thread:{}".format(options.threads)
# file may be "name" or "name,#"
reffile=args[1].split(",")[0]
try: refn=int(args[1].split(",")[1])
except: refn=0
NTHREADS=max(options.threads+1,2) # we have one thread just writing results
logid=E2init(sys.argv, options.ppid)
refnames=[reffile, reffile]
if options.goldcontinue:
ref=[]
try:
refnames=[reffile[:-4]+"_even.hdf", reffile[:-4]+"_odd.hdf"]
ref.append(EMData(refnames[0],0))
ref.append(EMData(refnames[1],0))
except:
print("Error: cannot find one of reference files, eg: ",EMData(reffile[:-4]+"_even.hdf",0))
n=-1
tasks=[]
readjson=False
if args[0].endswith(".lst") or args[0].endswith(".hdf"):
nptcl=EMUtil.get_image_count(args[0])
tasks.extend([(args[0],i,refnames, i%2) for i in range(nptcl)])
elif args[0].endswith(".json"):
js=js_open_dict(args[0])
readjson=True
jsinput=dict(js)
keys=sorted(js.keys())
nptcl=len(keys)
for k in keys:
src, ii=eval(k)
dic=js[k]
xf=dic["xform.align3d"]
tasks.append([src, ii, refnames, ii%2, xf])
if options.plst:
xinfo=[[] for i in range(nptcl)]
xfkey=["type","alt","az","phi","tx","ty","tz","alpha","scale"]
lst=LSXFile(options.plst)
for i in range(lst.n):
l=lst.read(i)
dc=eval(l[2])
pid=dc["pid"]
dxf=Transform({k:dc[k] for k in dc.keys() if k in xfkey})
xinfo[pid].append([dc["tid"],i, dxf])
lst=None
for i,xn in enumerate(xinfo):
sid=np.argsort([x[0] for x in xn])
xinfo[i]=[[xn[s][1],xn[s][2]] for s in sid]
for i in range(nptcl):
ii=tasks[i][1]
tasks[i].append(xinfo[ii])
from EMAN2PAR import EMTaskCustomer
etc=EMTaskCustomer(options.parallel, module="e2spt_align_subtlt.SptAlignTask")
num_cpus = etc.cpu_est()
options.nowtime=time.time()
if options.debug:
tasks=tasks[:num_cpus*4]
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob=num_cpus
tids=[]
for i in range(njob):
t=tasks[i::njob]
task=SptAlignTask(t, options)
if options.debug:
ret=task.execute(print)
return
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals)/100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output=[]
for i in tids:
rets=etc.get_results(i)[1]
for ret in rets:
fsp,n,dic=ret
output.append([fsp, n, dic])
angs={}
data=[]
for out in output:
fsp,n,dc=out
angs[(fsp, n)]={
"xform.align3d":dc["xform.align3d"],
"score":dc["score"]}
ts=dc.pop("transscr")
for i, ii in enumerate(dc["imgidx"]):
dxf=dc["imgxfs"][i].get_params("eman")
#dxf["score"]=float(dc["imgscore"][i])
dxf["pid"]=n; dxf["tid"]=i; dxf["class"]=n%2
tsmult=dc["tsmult"]
data.append({
"src":dc["imgsrc"], "srci":ii, "pid":n, "tid":i,
"xf":dxf, "coord":dc["coord"], "tsscr":ts[i]
})
m=options.maxshift
trans=np.indices((m*2+1, m*2+1)).reshape((2,-1)).T-m
alldtx=np.zeros((len(data), 3))
fnames=np.unique([d["src"] for d in data])
maxtid=np.max([d["tid"] for d in data])+1
print("Smoothing trajectories...")
for fname in fnames:
print(fname)
for tid in range(maxtid):
idx=[i for i,d in enumerate(data) if d["src"]==fname and d["tid"]==tid]
if len(idx)==0:
continue
coord=np.array([data[i]["coord"] for i in idx])
scrs=np.array([data[i]["tsscr"] for i in idx])
if options.smooth<=0:
s=np.argmin(scrs, 1)
sv=np.min(scrs, axis=1)
newdtx=np.hstack([trans[s], sv[:,None]])
if tid==0:print(" Skip smoothing")
else:
newdtx=[]
for i, crd in enumerate(coord):
dst=np.linalg.norm(coord-crd, axis=1)
srtid=np.argsort(dst)[:options.smoothN]
dst=dst[srtid]
wt=np.exp(-dst/options.smooth)
scr=scrs[srtid].copy()
scr=np.sum(scr*wt[:,None], axis=0)
pos=trans[np.argmin(scr)].copy()
pos=pos.astype(float)*tsmult
newdtx.append([pos[0], pos[1], np.min(scr)])
sys.stdout.write("\r {}: {:.4f}".format(tid, np.mean(scr)))
sys.stdout.flush()
newdtx=np.array(newdtx)
alldtx[idx]=newdtx
print()
for i,d in enumerate(data):
dxf=d["xf"]
dxf["tx"]+=alldtx[i][0]
dxf["ty"]+=alldtx[i][1]
dxf["score"]=alldtx[i][2]
d["xf"]=dxf
fm="{}/aliptcls_{:02d}.lst".format(options.path, options.iter)
if os.path.isfile(fm): os.remove(fm)
lout=LSXFile(fm, False)
for i,d in enumerate(data):
lout.write(-1, d["srci"], d["src"], d["xf"])
lout=None
f="{}/aliptcls_ts_{:02d}.hdf".format(options.path,options.iter)
if os.path.isfile(f): os.remove(f)
t=np.array([d["tsscr"] for d in data])
m=from_numpy(t).copy()
m.write_image(f)
#print(f, t.shape)
out="{}/particle_parms_{:02d}.json".format(options.path,options.iter)
if os.path.isfile(out):
os.remove(out)
js=js_open_dict(out)
js.update(angs)
js.close()
del etc
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
Note that this program is not part of the original e2spt hierarchy, but is part of an experimental refactoring.
This program will take an input stack of subtomograms and a reference volume, and perform a missing-wedge aware alignment of each particle to the reference. If --goldstandard is specified, then even and odd particles will be aligned to different perturbed versions of the reference volume, phase-randomized past the specified resolution."""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--threads",
default=4,
type=int,
help=
"Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.",
guitype='intbox',
row=24,
col=2,
rowspan=1,
colspan=1,
mode="refinement")
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=0)
parser.add_argument(
"--goldstandard",
type=float,
help=
"If specified, will phase randomize the even and odd references past the specified resolution (in A, not 1/A)",
default=0)
parser.add_argument(
"--goldcontinue",
action="store_true",
help=
"Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
default=False)
parser.add_argument(
"--saveali",
action="store_true",
help=
"Save a stack file (aliptcls.hdf) containing the aligned subtomograms.",
default=False)
parser.add_argument("--savealibin",
type=int,
help="shrink aligned particles before saving",
default=1)
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
)
parser.add_argument(
"--sym",
type=str,
default="c1",
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly."
)
parser.add_argument(
"--maxres",
type=float,
help="Maximum resolution to consider in alignment (in A, not 1/A)",
default=0)
#parser.add_argument("--wtori",type=float,help="Weight for using the prior orientation in the particle header. default is -1, i.e. not used.",default=-1)
parser.add_argument(
"--nsoln",
type=int,
help="number of solutions to keep at low resolution for the aligner",
default=1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default=None)
parser.add_argument("--refine",
action="store_true",
help="local refinement from xform.init in header.",
default=False)
parser.add_argument(
"--maxang",
type=float,
help="Maximum angular difference for the refine mode. default is 30",
default=30)
(options, args) = parser.parse_args()
#task=SptAlignTask(0,1,2,options)
#from pickle import dumps,loads,dump,load
#f=open("task.tmp",'w')
#dump(task,f)
#f.close()
#print(task)
#return
if options.path == None:
fls = [
int(i[-2:]) for i in os.listdir(".")
if i[:4] == "spt_" and len(i) == 6 and str.isdigit(i[-2:])
]
if len(fls) == 0: fls = [0]
options.path = "spt_{:02d}".format(max(fls) + 1)
try:
os.mkdir(options.path)
except:
pass
if options.iter <= 0:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0: options.iter = 1
else: options.iter = max(fls) + 1
if options.parallel == None:
options.parallel = "thread:{}".format(options.threads)
reffile = args[1]
NTHREADS = max(options.threads + 1,
2) # we have one thread just writing results
logid = E2init(sys.argv, options.ppid)
if options.goldcontinue:
ref = []
try:
ref.append(EMData(reffile[:-4] + "_even.hdf", 0))
ref.append(EMData(reffile[:-4] + "_odd.hdf", 0))
except:
print("Error: cannot find one of reference files, eg: ",
EMData(reffile[:-4] + "_even.hdf", 0))
else:
ref = []
ref.append(EMData(reffile, 0))
ref.append(EMData(reffile, 0))
if options.goldstandard > 0:
ref[0].process_inplace(
"filter.lowpass.randomphase",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[0].process_inplace(
"filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[1].process_inplace(
"filter.lowpass.randomphase",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[1].process_inplace(
"filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[0].write_image("{}/align_ref.hdf".format(options.path), 0)
ref[1].write_image("{}/align_ref.hdf".format(options.path), 1)
ref[0] = ref[0].do_fft()
ref[0].process_inplace("xform.phaseorigin.tocorner")
ref[1] = ref[1].do_fft()
ref[1].process_inplace("xform.phaseorigin.tocorner")
jsd = queue.Queue(0)
n = -1
#### check if even/odd split exists
fsps = [args[0][:-4] + "__even.lst", args[0][:-4] + "__odd.lst"]
tasks = []
if os.path.isfile(fsps[0]) and os.path.isfile(fsps[1]):
print("Using particle list: \n\t {} \n\t {}".format(fsps[0], fsps[1]))
for eo, f in enumerate(fsps):
N = EMUtil.get_image_count(f)
tasks.extend([(f, i, ref[eo]) for i in range(N)])
#### split by even/odd by default
else:
N = EMUtil.get_image_count(args[0])
tasks.extend([(args[0], i, ref[i % 2]) for i in range(N)])
#thrds=[threading.Thread(target=alifn,args=(jsd,args[0],i,ref[i%2],options)) for i in range(N)]
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel, module="e2spt_align.SptAlignTask")
num_cpus = etc.cpu_est()
#tasks=tasks[:24]
print("{} total CPUs available".format(num_cpus))
print("{} jobs".format(len(tasks)))
tids = []
for t in tasks:
task = SptAlignTask(t[0], t[1], t[2], options)
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
#print(tids)
if np.min(st_vals) == 100: break
time.sleep(5)
#dics=[0]*nptcl
angs = {}
for i in tids:
ret = etc.get_results(i)[1]
fsp, n, dic = ret
if len(dic) == 1:
angs[(fsp, n)] = dic[0]
else:
angs[(fsp, n)] = dic
js = js_open_dict("{}/particle_parms_{:02d}.json".format(
options.path, options.iter))
js.update(angs)
del etc
## here we run the threads and save the results, no actual alignment done here
#print(len(thrds)," threads")
#thrtolaunch=0
#while thrtolaunch<len(thrds) or threading.active_count()>1:
## If we haven't launched all threads yet, then we wait for an empty slot, and launch another
## note that it's ok that we wait here forever, since there can't be new results if an existing
## thread hasn't finished.
#if thrtolaunch<len(thrds) :
#while (threading.active_count()==NTHREADS ) : time.sleep(.1)
#if options.verbose : print("Starting thread {}/{}".format(thrtolaunch,len(thrds)))
#thrds[thrtolaunch].start()
#thrtolaunch+=1
#else: time.sleep(1)
#while not jsd.empty():
#fsp,n,d=jsd.get()
#angs[(fsp,n)]=d
#if options.saveali:
#v=EMData(fsp,n)
#v.transform(d["xform.align3d"])
#if options.savealibin>1:
#v.process_inplace("math.meanshrink",{"n":options.savealibin})
#v.write_image("{}/aliptcls_{:02d}.hdf".format(options.path, options.iter),n)
#for t in thrds:
#t.join()
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
This program is part of the 'new' hierarchy of e2spt_ programs. It performs one iteration of a classical subtomogram refinement, ie - aligning particles with missing wedge to a reference in 3-D
The reference may be <volume> or <volume>,<n>
If --goldstandard is specified, then even and odd particles will be aligned to different perturbed versions of the reference volume, phase-randomized past the specified resolution."""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--threads",
default=4,
type=int,
help=
"Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.",
guitype='intbox',
row=24,
col=2,
rowspan=1,
colspan=1,
mode="refinement")
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=0)
parser.add_argument(
"--goldstandard",
type=float,
help=
"If specified, will phase randomize the even and odd references past the specified resolution (in A, not 1/A)",
default=0)
parser.add_argument(
"--goldcontinue",
action="store_true",
help=
"Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
default=False)
#parser.add_argument("--saveali",action="store_true",help="Save a stack file (aliptcls.hdf) containing the aligned subtomograms.",default=False)
#parser.add_argument("--savealibin",type=int,help="shrink aligned particles before saving",default=1)
parser.add_argument(
"--path",
type=str,
default=None,
help=
"Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)"
)
parser.add_argument(
"--sym",
type=str,
default="c1",
help=
"Symmetry of the input. Must be aligned in standard orientation to work properly."
)
parser.add_argument(
"--maxres",
type=float,
help="Maximum resolution to consider in alignment (in A, not 1/A)",
default=0)
#parser.add_argument("--wtori",type=float,help="Weight for using the prior orientation in the particle header. default is -1, i.e. not used.",default=-1)
parser.add_argument(
"--nsoln",
type=int,
help="number of solutions to keep at low resolution for the aligner",
default=1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default=None)
parser.add_argument("--refine",
action="store_true",
help="local refinement from xform.align3d in header.",
default=False)
parser.add_argument("--refinentry",
type=int,
help="number of tests for refine mode. default is 8",
default=8)
parser.add_argument("--randphi",
action="store_true",
help="randomize phi during refine alignment",
default=False)
parser.add_argument("--breaksym",
action="store_true",
help="symmetry breaking.",
default=False)
parser.add_argument(
"--breaksymsym",
type=str,
help=
"the symmetry to use for breaksym. setting sym to c6 and this to c2 results in a c3 structure. default is the same as sym",
default=None)
parser.add_argument(
"--rand180",
action="store_true",
help="randomly add a 180 degree rotation during refine alignment",
default=False)
parser.add_argument(
"--skipali",
action="store_true",
help=
"skip alignment. the program will do nothing. mostly for testing...",
default=False)
parser.add_argument(
"--maxang",
type=float,
help="Maximum angular difference for the refine mode. default is 30",
default=30)
parser.add_argument(
"--maxshift",
type=float,
help="Maximum shift for the refine mode. default is 16",
default=-1)
(options, args) = parser.parse_args()
#task=SptAlignTask(0,1,2,options)
#from pickle import dumps,loads,dump,load
#f=open("task.tmp",'w')
#dump(task,f)
#f.close()
#print(task)
#return
if options.path == None:
fls = [
int(i[-2:]) for i in os.listdir(".")
if i[:4] == "spt_" and len(i) == 6 and str.isdigit(i[-2:])
]
if len(fls) == 0: fls = [0]
options.path = "spt_{:02d}".format(max(fls) + 1)
try:
os.mkdir(options.path)
except:
pass
if options.iter <= 0:
fls = [
int(i[15:17]) for i in os.listdir(options.path)
if i[:15] == "particle_parms_" and str.isdigit(i[15:17])
]
if len(fls) == 0: options.iter = 1
else: options.iter = max(fls) + 1
if options.parallel == None:
options.parallel = "thread:{}".format(options.threads)
if options.breaksym:
if options.breaksymsym == None:
if options.sym == "c1":
print("cannot break a c1 symmetry. breaksym disabled.")
options.breaksym = False
else:
options.breaksymsym = options.sym
# file may be "name" or "name,#"
reffile = args[1].split(",")[0]
try:
refn = int(args[1].split(",")[1])
except:
refn = 0
NTHREADS = max(options.threads + 1,
2) # we have one thread just writing results
logid = E2init(sys.argv, options.ppid)
refnames = []
if options.goldcontinue:
ref = []
try:
refnames = [reffile[:-4] + "_even.hdf", reffile[:-4] + "_odd.hdf"]
ref.append(EMData(refnames[0], 0))
ref.append(EMData(refnames[1], 0))
except:
print("Error: cannot find one of reference files, eg: ",
EMData(reffile[:-4] + "_even.hdf", 0))
else:
ref = []
ref.append(EMData(reffile, refn))
ref.append(EMData(reffile, refn))
if options.goldstandard > 0:
ref[0].process_inplace(
"filter.lowpass.randomphase",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[0].process_inplace(
"filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[1].process_inplace(
"filter.lowpass.randomphase",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
ref[1].process_inplace(
"filter.lowpass.gauss",
{"cutoff_freq": old_div(1.0, options.goldstandard)})
refnames = [
"{}/align_ref_even.hdf".format(options.path),
"{}/align_ref_odd.hdf".format(options.path)
]
ref[0].write_image(refnames[0], 0)
ref[1].write_image(refnames[1], 0)
else:
refnames = [reffile, reffile]
#ref[0]=ref[0].do_fft()
#ref[0].process_inplace("xform.phaseorigin.tocorner")
#ref[1]=ref[1].do_fft()
#ref[1].process_inplace("xform.phaseorigin.tocorner")
#jsd=queue.Queue(0)
n = -1
tasks = []
if args[0].endswith(".lst") or args[0].endswith(".hdf"):
#### check if even/odd split exists
fsps = [args[0][:-4] + "__even.lst", args[0][:-4] + "__odd.lst"]
if os.path.isfile(fsps[0]) and os.path.isfile(fsps[1]):
print("Using particle list: \n\t {} \n\t {}".format(
fsps[0], fsps[1]))
for eo, f in enumerate(fsps):
N = EMUtil.get_image_count(f)
tasks.extend([(f, i, refnames, eo) for i in range(N)])
#### split by even/odd by default
else:
N = EMUtil.get_image_count(args[0])
tasks.extend([(args[0], i, refnames, i % 2) for i in range(N)])
#thrds=[threading.Thread(target=alifn,args=(jsd,args[0],i,ref[i%2],options)) for i in range(N)]
elif args[0].endswith(".json"):
print("Reading particles from json. This is experimental...")
js = js_open_dict(args[0])
keys = sorted(js.keys())
for k in keys:
src, ii = eval(k)
dic = js[k]
xf = dic["xform.align3d"]
tasks.append([src, ii, refnames, ii % 2, xf])
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel, module="e2spt_align.SptAlignTask")
num_cpus = etc.cpu_est()
options.nowtime = time.time()
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob = num_cpus #*4
tids = []
for i in range(njob):
t = tasks[i::njob]
task = SptAlignTask(t, options)
tid = etc.send_task(task)
tids.append(tid)
#print("starting...")
#for t in tasks:
#if len(t)>3:
#task = SptAlignTask(t[0], t[1], t[2], options, t[3])
#else:
#task = SptAlignTask(t[0], t[1], t[2], options)
#tid=etc.send_task(task)
#tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
#dics=[0]*nptcl
angs = {}
for i in tids:
rets = etc.get_results(i)[1]
for ret in rets:
fsp, n, dic = ret
if len(dic) == 1:
angs[(fsp, n)] = dic[0]
else:
angs[(fsp, n)] = dic
out = "{}/particle_parms_{:02d}.json".format(options.path, options.iter)
if os.path.isfile(out):
os.remove(out)
js = js_open_dict(out)
js.update(angs)
js.close()
del etc
## here we run the threads and save the results, no actual alignment done here
#print(len(thrds)," threads")
#thrtolaunch=0
#while thrtolaunch<len(thrds) or threading.active_count()>1:
## If we haven't launched all threads yet, then we wait for an empty slot, and launch another
## note that it's ok that we wait here forever, since there can't be new results if an existing
## thread hasn't finished.
#if thrtolaunch<len(thrds) :
#while (threading.active_count()==NTHREADS ) : time.sleep(.1)
#if options.verbose : print("Starting thread {}/{}".format(thrtolaunch,len(thrds)))
#thrds[thrtolaunch].start()
#thrtolaunch+=1
#else: time.sleep(1)
#while not jsd.empty():
#fsp,n,d=jsd.get()
#angs[(fsp,n)]=d
#if options.saveali:
#v=EMData(fsp,n)
#v.transform(d["xform.align3d"])
#if options.savealibin>1:
#v.process_inplace("math.meanshrink",{"n":options.savealibin})
#v.write_image("{}/aliptcls_{:02d}.hdf".format(options.path, options.iter),n)
#for t in thrds:
#t.join()
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """Usage: e2spt_align.py [options] <subvolume_stack> <reference>
This program is part of the 'new' hierarchy of e2spt_ programs. It performs one iteration of a classical subtomogram refinement, ie - aligning particles with missing wedge to a reference in 3-D
The reference may be <volume> or <volume>,<n>
If --goldstandard is specified, even and odd variants of the alignment reference must be provided, and even and odd particles will be aligned separately"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_argument("--threads", default=4,type=int,help="Number of alignment threads to run in parallel on a single computer. This is the only parallelism supported by e2spt_align at present.", guitype='intbox', row=24, col=2, rowspan=1, colspan=1, mode="refinement")
parser.add_argument("--iter",type=int,help="Iteration number within path. Default = start a new iteration",default=0)
parser.add_argument("--goldstandard",type=float,help="If specified, will phase randomize the even and odd references past the specified resolution (in A, not 1/A)",default=0)
parser.add_argument("--goldcontinue",action="store_true",help="Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",default=False)
#parser.add_argument("--saveali",action="store_true",help="Save a stack file (aliptcls.hdf) containing the aligned subtomograms.",default=False)
#parser.add_argument("--savealibin",type=int,help="shrink aligned particles before saving",default=1)
parser.add_argument("--mask",type=str,default=None,help="Mask file aligned to the input reference. Alignment occurs under this mask.")
parser.add_argument("--path",type=str,default=None,help="Path to a folder where results should be stored, following standard naming conventions (default = spt_XX)")
parser.add_argument("--sym",type=str,default="c1",help="Symmetry of the input. Must be aligned in standard orientation to work properly.")
parser.add_argument("--maxres",type=float,help="Maximum resolution (the smaller number) to consider in alignment (in A, not 1/A)",default=0)
parser.add_argument("--minres",type=float,help="Minimum resolution (the larger number) to consider in alignment (in A, not 1/A)",default=0)
#parser.add_argument("--wtori",type=float,help="Weight for using the prior orientation in the particle header. default is -1, i.e. not used.",default=-1)
parser.add_argument("--nsoln",type=int,help="number of solutions to keep at low resolution for the aligner",default=1)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
parser.add_argument("--transonly",action="store_true",help="translational alignment only, for prealigned particles",default=False)
parser.add_argument("--refine",action="store_true",help="local refinement from xform.align3d in header.",default=False)
parser.add_argument("--flcf",action="store_true",help="use slower aligner (experimental)",default=False)
parser.add_argument("--refinentry", type=int, help="number of tests for refine mode. default is 8",default=8)
parser.add_argument("--randphi",action="store_true",help="randomize phi during refine alignment",default=False)
parser.add_argument("--breaksym",action="store_true",help="symmetry breaking.",default=False)
parser.add_argument("--breaksymsym",type=str,help="the symmetry to use for breaksym. setting sym to c6 and this to c2 results in a c3 structure. default is the same as sym",default=None)
parser.add_argument("--rand180",action="store_true",help="randomly add a 180 degree rotation during refine alignment",default=False)
parser.add_argument("--test180",action="store_true",help="Test for improved alignment with 180 degree rotations even during refine alignment",default=False)
parser.add_argument("--skipali",action="store_true",help="skip alignment. the program will do nothing. mostly for testing...",default=False)
parser.add_argument("--maxang",type=float,help="Maximum angular difference for the refine mode. default is 30",default=30)
parser.add_argument("--maxshift",type=float,help="Maximum shift for the refine mode. default is 16",default=-1)
parser.add_argument("--scipytest",action="store_true",help="test scipy optimizer.",default=False)
parser.add_argument("--debug",action="store_true",help=".",default=False)
(options, args) = parser.parse_args()
if options.path == None:
options.path=num_path_new("spt_")
if options.iter<=0 :
fls=[int(i[15:17]) for i in os.listdir(options.path) if i[:15]=="particle_parms_" and str.isdigit(i[15:17])]
if len(fls)==0 : options.iter=1
else: options.iter=max(fls)+1
if options.parallel==None:
options.parallel="thread:{}".format(options.threads)
if options.breaksym:
if options.breaksymsym==None:
if options.sym=="c1":
print("cannot break a c1 symmetry. breaksym disabled.")
options.breaksym=False
else:
options.breaksymsym=options.sym
# file may be "name" or "name,#"
reffile=args[1].split(",")[0]
try: refn=int(args[1].split(",")[1])
except: refn=0
NTHREADS=max(options.threads+1,2) # we have one thread just writing results
logid=E2init(sys.argv, options.ppid)
refnames=[reffile,reffile]
if options.goldcontinue or options.goldstandard>0:
ref=[]
try:
refnames=[reffile[:-4]+"_even.hdf", reffile[:-4]+"_odd.hdf"]
ref.append(EMData(refnames[0],0))
ref.append(EMData(refnames[1],0))
except:
print("Error: cannot find one of reference files, eg: ",EMData(reffile[:-4]+"_even.hdf",0))
# else:
# ref=[]
# ref.append(EMData(reffile,refn))
# ref.append(EMData(reffile,refn))
#
# if options.goldstandard>0 :
# ref[0].process_inplace("filter.lowpass.randomphase",{"cutoff_freq":old_div(1.0,options.goldstandard)})
# ref[0].process_inplace("filter.lowpass.gauss",{"cutoff_freq":old_div(1.0,options.goldstandard)})
# ref[1].process_inplace("filter.lowpass.randomphase",{"cutoff_freq":old_div(1.0,options.goldstandard)})
# ref[1].process_inplace("filter.lowpass.gauss",{"cutoff_freq":old_div(1.0,options.goldstandard)})
# refnames=["{}/align_ref_even.hdf".format(options.path), "{}/align_ref_odd.hdf".format(options.path)]
# ref[0].write_image(refnames[0],0)
# ref[1].write_image(refnames[1],0)
#
# else:
# refnames=[reffile, reffile]
n=-1
tasks=[]
readjson=False
if args[0].endswith(".lst") or args[0].endswith(".hdf"):
#### check if even/odd split exists
fsps=[args[0][:-4]+"__even.lst",args[0][:-4]+"__odd.lst"]
if os.path.isfile(fsps[0]) and os.path.isfile(fsps[1]):
print("Using particle list: \n\t {} \n\t {}".format(fsps[0], fsps[1]))
for eo, f in enumerate(fsps):
N=EMUtil.get_image_count(f)
tasks.extend([(f,i,refnames, eo) for i in range(N)])
#### split by even/odd by default
else:
N=EMUtil.get_image_count(args[0])
tasks.extend([(args[0],i,refnames, i%2) for i in range(N)])
#thrds=[threading.Thread(target=alifn,args=(jsd,args[0],i,ref[i%2],options)) for i in range(N)]
elif args[0].endswith(".json"):
#print("Reading particles from json. This is experimental...")
js=js_open_dict(args[0])
readjson=True
jsinput=dict(js)
keys=sorted(js.keys())
for k in keys:
src, ii=eval(k)
dic=js[k]
xf=dic["xform.align3d"]
tasks.append([src, ii, refnames, ii%2, xf])
from EMAN2PAR import EMTaskCustomer
if options.scipytest:
etc=EMTaskCustomer(options.parallel, module="e2spt_align.ScipySptAlignTask")
else:
etc=EMTaskCustomer(options.parallel, module="e2spt_align.SptAlignTask")
num_cpus = etc.cpu_est()
options.nowtime=time.time()
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob=num_cpus#*4
tids=[]
for i in range(njob):
t=tasks[i::njob]
if options.scipytest:
task=ScipySptAlignTask(t, options)
else:
task=SptAlignTask(t, options)
if options.debug:
ret=task.execute(print)
print(ret)
return
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print(st_vals)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals)/100.)
if np.min(st_vals) == 100: break
time.sleep(5)
#dics=[0]*nptcl
angs={}
for i in tids:
rets=etc.get_results(i)[1]
for ret in rets:
fsp,n,dic=ret
if len(dic)==1:
dic=dic[0]
if readjson:
k=str((fsp,n))
if "eo" in js[k]:
dic["eo"]=jsinput[k]["eo"]
angs[(fsp,n)]=dic
out="{}/particle_parms_{:02d}.json".format(options.path,options.iter)
if os.path.isfile(out):
os.remove(out)
js=js_open_dict(out)
js.update(angs)
js.close()
del etc
E2end(logid)
def main():
parser = EMArgumentParser(usage=get_usage())
parser.add_argument("--output", default="threed.hdf", help="Output reconstructed volume file name.")
parser.add_argument("--input", default=None, help="The input projections. Project should usually have the xform.projection header attribute, which is used for slice insertion")
parser.add_argument("--input_model", default=None, help="If the class-averages have the model_id parameter (produced by e2refinemulti.py), this will use only class-averages with the specified model_id for the reconstruction.")
parser.add_argument("--tlt", help="An imod tlt file containing alignment angles. If specified slices will be inserted using these angles in the IMOD convention", type=str, default=None)
parser.add_argument("--sym", dest="sym", default="c1", help="Set the symmetry; if no value is given then the model is assumed to have no symmetry.\nChoices are: i, c, d, tet, icos, or oct.")
parser.add_argument("--fillangle", type=float, dest="fillangle", help="An angular range used for both alt & az over which the projection should be averaged. Generally the angular step used when making projections.",default=0)
parser.add_argument("--pad", metavar="x or x,y", default=None,type=str, help="Will zero-pad images to the specifed size (x,y) or (x,x) prior to reconstruction. If not specified or 0 no padding occurs. If a negative value is specified automatic padding is performed. ")
parser.add_argument("--padvol", metavar="x or x,y,z", default=None,type=str, help="Defines the dimensions (x,y,z) or (x,x,x) of the reconstructed volume. If ommitted, implied value based on padded 2D images is used.")
parser.add_argument("--outsize", metavar="x or x,y,z", default=None, type=str, help="Defines the dimensions (x,y,z) or (x,x,x) of the final volume written to disk, if ommitted, size will be based on unpadded input size")
#parser.add_argument("--clipz", default=None, type=int, help="Extract a specified number of central z-slices. Option disabled by default.")
parser.add_argument("--savenorm", default=None, type=str, help="If set, will save the normalization volume showing Fourier space filling to the specified file")
parser.add_argument("--keep", type=float, dest="keep", help="The fraction of slices to keep, based on quality scores (1.0 = use all slices). See keepsig.",default=1.0)
parser.add_argument("--keepsig",action="store_true",default=False, dest="keepsig", help="If set, keep will be interpreted as a standard deviation coefficient instead of as a percentage.")
parser.add_argument("--keepabs",action="store_true",default=False, dest="keepabs", help="If set, keep will refer to the absolute quality of the class-average, not a local quality relative to other similar sized classes.")
parser.add_argument("--altedgemask",action="store_true",default=False, dest="altedgemask", help="If set, assumes tomographic data with a thin specimen, and masks out the +-X edges based on the alt Euler angle.")
parser.add_argument("--no_wt", action="store_true", dest="no_wt", default=False, help="This argument turns automatic weighting off causing all images to be weighted by 1. If this argument is not specified images inserted into the reconstructed volume are weighted by the number of particles that contributed to them (i.e. as in class averages), which is extracted from the image header (as the ptcl_repr attribute).")
parser.add_argument("--sqrt_wt", action="store_true", default=False, help="Normally class-averages are weighted into the reconstruction based on the number of particles in the average. This option causes the sqrt of the number of particles to be used instead.")
parser.add_argument("--iterative", action="store_true", default=False, help="Uses iterative interpolation in Fourier space instead of single step gridding or interpolation. --mode and --usessnr are ignored with this option.")
parser.add_argument("--itermask",type=str, default = None, help="Used in iterative mode to define a mask to apply between iterations. The resulting reconstruction won't be masked per-se, but will be biased towards lower densities outside the mask.")
parser.add_argument("--usessnr", action="store_true", default=False, help="Makes use of the class_ssnr header data to weight each slice during insertion, instead of the default behavior of just using the number of particles in the average as a global weight.")
parser.add_argument("--mode", type=str, default="gauss_var", help="Fourier reconstruction 'mode' to use. The default should not normally be changed. default='gauss_var'")
parser.add_argument("--noradcor", action="store_true",default=False, help="Normally a radial correction will be applied based on the --mode used. This option disables that correction.")
parser.add_argument("--seedmap",type=str, default = None, help="If specified this volume will be used as a starting point for the reconstruction, filling any missing values in Fourier space. experimental.")
parser.add_argument("--seedweight", type=float, default=1.0, help="If seedmap specified, this is how strongly the seedmap will bias existing values. 1 is default, and is equivalent to a one particle weight.")
parser.add_argument("--seedweightmap", type=str, default=None, help="Specify a full map of weights for the seed. This must be in the same format as the --savenorm output map.")
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--threads", default=4,type=int,help="Number of threads to run in parallel on a single computer. This is the only parallelism supported by e2make3dpar", guitype='intbox', row=24, col=2, rowspan=1, colspan=1, mode="refinement")
parser.add_argument("--preprocess", metavar="processor_name(param1=value1:param2=value2)", type=str, action="append", help="preprocessor to be applied to the projections prior to 3D insertion. There can be more than one preprocessor and they are applied in the order in which they are specifed. Applied before padding occurs. See e2help.py processors for a complete list of available processors.")
parser.add_argument("--setsf",type=str,help="Force the structure factor to match a 'known' curve prior to postprocessing (<filename>, auto or none). default=none",default="none")
parser.add_argument("--postprocess", metavar="processor_name(param1=value1:param2=value2)", type=str, action="append", help="postprocessor to be applied to the 3D volume once the reconstruction is completed. There can be more than one postprocessor, and they are applied in the order in which they are specified. See e2help.py processors for a complete list of available processors.")
parser.add_argument("--apix",metavar="A/pix",type=float,help="A/pix value for output, overrides automatic values",default=None)
parser.add_argument("--parallel", type=str,help="Thread/mpi parallelism to use", default=None)
# Database Metadata storage
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
(options, args) = parser.parse_args()
# options.fillangle=options.fillangle*pi/180.0
# make sure that the user has atleast specified an input file
# if len(args) <1:
# parser.error("No input file specified")
# options.input = args[0]
if (not options.keep and not options.keepsig):
print("Warning, neither the keep nor the keepsig argument was specified. Setting keep=1 (keeping 100% of inserted slices)")
options.keep=1
if options.input_model!=None : options.input_model=int(options.input_model)
print("e2make3dpar.py")
logger=E2init(sys.argv,options.ppid)
# update the no_wt parameter
if options.sqrt_wt : options.no_wt=2
elif options.no_wt : options.no_wt=1
else : options.no_wt=0
# get basic image parameters
tmp=EMData()
tmp.read_image(options.input,0,True)
if options.no_wt!=1 :
try:
n=1
while tmp["ptcl_repr"]==0 :
tmp.read_image(options.input,n,True)
n+=1
except: pass
nx=tmp["nx"]
ny=tmp["ny"]
nslice=tmp["nz"]
if options.apix!=None : apix=options.apix
else : apix=tmp["apix_x"]
if options.verbose>0: print("Image dimensions %d x %d"%(nx,ny))
# parse the padding options, to make sure we have a 2 or 3 tuple for each
try :
if options.pad==None or options.pad==0: options.pad=(max(nx,ny),max(nx,ny))
elif options.pad[0]=='-':
sz=max(nx,ny)
sz=good_size(sz*1.25)
options.pad=(sz,sz)
print("padding to {}x{}".format(sz,sz))
elif "," in options.pad :
s=options.pad.split(",")
options.pad=(int(s[0]),int(s[1]))
else : options.pad=(int(options.pad),int(options.pad))
if options.verbose>0 : print("Pad to %d x %d"%(options.pad[0],options.pad[1]))
except:
print("Couldn't parse pad option :",options.pad)
exit(1)
try :
if options.padvol==None : padvol=(max(options.pad),max(options.pad),max(options.pad))
elif "," in options.padvol :
s=options.padvol.split(",")
padvol=(int(s[0]),int(s[1]),int(s[2]))
else : padvol=(int(options.padvol),int(options.padvol),int(options.padvol))
if options.verbose>0 : print("Padded volume to reconstruct %d x %d x %d"%(padvol[0],padvol[1],padvol[2]))
except:
print("Couldn't parse padvol option :",options.padvol)
exit(1)
try :
if options.outsize==None : outsize=(max(nx,ny),max(nx,ny),max(nx,ny))
elif "," in options.outsize :
s=options.outsize.split(",")
outsize=(int(s[0]),int(s[1]),int(s[2]))
else : outsize=(int(options.outsize),int(options.outsize),int(options.outsize))
if options.verbose>0 : print("Final output volume %d x %d x %d"%(outsize[0],outsize[1],outsize[2]))
except:
print("Couldn't parse outsize option :",options.outsize)
exit(1)
data=initialize_data(options.input,options.input_model,options.tlt,options.pad,options.no_wt,options.preprocess)
# Filter out averages/images which aren't good enough
if options.verbose: print("Filtering data, %d images ->"%len(data), end=' ')
# we have an absolute threshold
if options.keepabs: thr=options.keep
else :
quals=np.array(sorted([i["quality"] for i in data]))
if np.std(quals)==0 : thr=max(quals)+1.0
else :
if options.keepsig:
thr=np.mean(quals)+options.keep*np.std(quals)
else:
try:
thr=quals[int(floor(len(quals)*options.keep))]
except: thr=max(quals)+1.0
excluded=[max(i["fileslice"],i["filenum"]) for i in data if i["quality"]>thr]
included=[max(i["fileslice"],i["filenum"]) for i in data if i["quality"]<=thr]
data=[i for i in data if i["quality"]<=thr]
ptclcount=sum([i["weight"] for i in data])
if options.verbose: print("After filter, %d images"%len(data))
if options.itermask:
itmask=EMData(options.itermask)
itmask=itmask.get_clip(Region(old_div((itmask["nx"]-options.pad[1]),2),old_div((itmask["ny"]-options.pad[1]),2),old_div((itmask["nz"]-options.pad[1]),2),options.pad[1],options.pad[1],options.pad[1]))
# Get the reconstructor and initialize it correctly
if options.iterative :
a = {"size":padvol,"sym":options.sym,"verbose":options.verbose-1}
if options.savenorm!=None : a["savenorm"]=options.savenorm
recon=Reconstructors.get("fourier_iter", a)
niter=4
else :
a = {"size":padvol,"sym":options.sym,"mode":options.mode,"usessnr":options.usessnr,"verbose":options.verbose-1}
if options.savenorm!=None : a["savenorm"]=options.savenorm
recon=Reconstructors.get("fourier", a)
niter=1
#########################################################
# The actual reconstruction
options.padvol3=padvol
if options.parallel!=None:
par=options.parallel.split(':')
if par[0].startswith("thread"):
#options.parallel=None
options.threads=int(par[1])
elif par[0]=="mpi":
nthr=int(par[1])
ppt=len(data)/nthr
if ppt>16:
print("Inserting {:.1f} images per thread...".format(ppt))
### prepare some options
else:
print("Too few images in input ({:.1f} images per thread). Switching back to threading".format(ppt))
options.parallel=None
options.threads=nthr
else:
options.parallel=None
if options.seedmap!=None :
seed=EMData(options.seedmap)
seed.clip_inplace(Region(old_div((nx-padvol[0]),2),old_div((ny-padvol[1]),2),old_div((nslice-padvol[2]),2),padvol[0],padvol[1],padvol[2]))
seed.do_fft_inplace()
else:
seed=None
for it in range(niter):
if options.parallel:
print("running in mpi mode. This is experimental, so please switch back to threading if anything goes wrong...")
if it>0:
seed=output
from EMAN2PAR import EMTaskCustomer
etc=EMTaskCustomer(options.parallel, module="e2make3dpar.Make3dTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
tasks=[data[i::num_cpus] for i in range(num_cpus)]
print("{} jobs".format(len(tasks)))
tids=[]
for t in tasks:
task = Make3dTask(t, seed, options)
tid=etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
#print("{:.1f}/{} finished".format(np.mean(st_vals), 100))
#print(tids)
if np.min(st_vals) == 100: break
time.sleep(5)
#dics=[0]*nptcl
output=EMData(padvol[0], padvol[1], padvol[2])
normvol=EMData(padvol[0]//2+1, padvol[1], padvol[2])
output.to_zero()
output.do_fft_inplace()
normvol.to_zero()
for i in tids:
threed, norm=etc.get_results(i)[1]
threed.process_inplace("math.multamplitude", {"amp":norm})
output.add(threed)
normvol.add(norm)
normvol.process_inplace("math.reciprocal")
output.process_inplace("math.multamplitude", {"amp":normvol})
output.do_ift_inplace()
output.depad()
output.process_inplace("xform.phaseorigin.tocenter")
del etc
else:
threads=[threading.Thread(target=reconstruct,args=(data[i::options.threads],recon,options.preprocess,options.pad,
options.fillangle,options.altedgemask,max(options.verbose-1,0),options.input.endswith(".lst"))) for i in range(options.threads)]
if it==0:
if options.seedmap!=None :
#seed=EMData(options.seedmap)
# seed.process_inplace("normalize.edgemean")
#seed.clip_inplace(Region(old_div((nx-padvol[0]),2),old_div((ny-padvol[1]),2),old_div((nslice-padvol[2]),2),padvol[0],padvol[1],padvol[2]))
#seed.do_fft_inplace()
if options.seedweightmap==None:
recon.setup_seed(seed,options.seedweight)
else:
seedweightmap=EMData(seedweightmap,0)
recon.setup_seedandweights(seed,seedweightmap)
else : recon.setup()
else:
recon.setup_seed(output,1.0)
for i,t in enumerate(threads):
if options.verbose>1: print("started thread ",i)
t.start()
for t in threads: t.join()
# output = recon.finish(it==niter-1) # only return real-space on the final pass
output = recon.finish(True)
if options.verbose:
print("Iteration ",it)
if options.verbose>2: output.write_image("it{}.hdf".format(it))
#if options.verbose>2 and it!=niter-1:
#output.process("xform.phaseorigin.tocenter").do_ift().write_image("it{}.hdf".format(it))
if it<niter-1 :
if options.itermask:
output.mult(itmask)
else:
output.process_inplace("threshold.compress",{"range":output["sigma"],"value":output["mean"],"clamponly":True})
if options.verbose:
print("Iteration ",it)
output.write_image("it{}.hdf".format(it))
#if options.verbose>2 and it!=niter-1:
#output.process("xform.phaseorigin.tocenter").do_ift().write_image("it{}.hdf".format(it))
if options.verbose>0 : print("Finished Reconstruction")
try:
output.set_attr("ptcl_repr",ptclcount)
if len(included)>0 : output.set_attr("threed_ptcl_idxs",included)
if len(excluded)>0 : output.set_attr("threed_excl_ptcl_idxs",excluded)
output.set_attr("threed_ptcl_src",data[0]["filename"])
except:
print("Warning, error setting reconstruction attributes")
# progress += 10
# E2progress(logid,float(progress)/total_progress)
#
########################################################3
# Correct for Gaussian falloff
if not options.noradcor :
cor={"gauss_2":4.0,"gauss_3":6.4,"gauss_4":8.8,"gauss_5":10.4,"gauss_5_slow":10.4} # Gaussian widths for different reconstruction modes
if options.mode in cor :
output.process_inplace("math.gausskernelfix",{"gauss_width":cor[options.mode]})
elif options.mode in ["gridding_5","gridding_7"]:
output.process_inplace("math.gridkernelfix",{"mode":options.mode})
else:
print("Warning: no radial correction applied for this mode")
# clip to the requested final dimensions
if output["nx"]!=outsize[0] or output["ny"]!=outsize[1] or output["nz"]!=outsize[2] :
output.clip_inplace(Region(old_div((output["nx"]-outsize[0]),2),old_div((output["ny"]-outsize[1]),2),old_div((output["nz"]-outsize[2]),2), outsize[0],outsize[1],outsize[2]))
#if options.clipz != None:
# output.clip_inplace(Region(0,0,(output["nz"]-options.clipz)//2, output["nx"], output["ny"], options.clipz))
if options.apix!=None : apix=options.apix
output["apix_x"]=apix
output["apix_y"]=apix
output["apix_z"]=apix
# Structure factor setting
if options.setsf.lower() != "none" :
if options.setsf.lower()!="auto" :
try:
sfcurve=XYData()
sfcurve.read_file(options.setsf)
for i in range(sfcurve.get_size()):
v=sfcurve.get_y(i)
if v<=0 :
print("Warning values <=0 found in structure factor file. Please remove.")
sfcurve.update()
except:
print("ERROR: Specified structure factor ({}) not found.".format(options.setsf))
sys.exit(1)
else:
try:
sfcurve=XYData()
sfcurve.read_file("strucfac.txt")
for i in range(sfcurve.get_size()):
v=sfcurve.get_y(i)
if v<=0 :
print("Warning values <=0 found in structure factor file. Please remove.")
sfcurve.update()
except : sfcurve=None
if sfcurve==None:
print("ERROR : Structure factor read failed. Not applying structure factor")
else:
# need to be really careful about the corners
# for i in range(sfcurve.get_size()):
# if sfcurve.get_x(i)>1.0/(2.0*apix) : sfcurve.set_y(i,0.0)
output.process_inplace("filter.setstrucfac",{"apix":apix,"strucfac":sfcurve})
# output.process_inplace("filter.lowpass.tophat",{"apix":apix,"cutoff_abs":0.49})
if options.postprocess != None:
for p in options.postprocess:
try:
(processorname, param_dict) = parsemodopt(p)
if not param_dict : param_dict={}
output.process_inplace(str(processorname), param_dict)
except:
print("warning - application of the post processor",p," failed. Continuing anyway")
# output.process_inplace("normalize.circlemean")
# just remove the output file
if file_exists(options.output):
remove_file(options.output)
# write the reconstruction to disk
output.write_image(options.output,0)
if options.verbose>0:
print("Output File: "+options.output)
E2end(logger)
print("Exiting")
def main():
progname = os.path.basename(sys.argv[0])
usage = """
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument(
"--path",
type=str,
help=
"Path to a folder where results should be stored, following standard naming conventions",
default="subtlt_00")
parser.add_argument("--ref", type=str, help="reference map", default=None)
parser.add_argument(
"--iter",
type=int,
help="Iteration number within path. Default = start a new iteration",
default=0)
parser.add_argument(
"--goldcontinue",
action="store_true",
help=
"Will use even/odd refs corresponding to specified reference to continue refining without phase randomizing again",
default=False)
parser.add_argument(
"--maxres",
type=float,
help="Maximum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument(
"--minres",
type=float,
help="Minimum resolution to consider in alignment (in A, not 1/A)",
default=0)
parser.add_argument(
"--smooth",
type=float,
help=
"smooth local motion by this factor. smoother local motion with larger numbers. default 100",
default=100)
parser.add_argument(
"--smoothN",
type=int,
help="number of neighboring particles used for smoothing. default 15",
default=15)
parser.add_argument("--maxshift",
type=float,
help="max shift in pixel. default default box size/6",
default=-1)
parser.add_argument("--refine_trans",
action="store_true",
help="do translational alignment.",
default=False)
parser.add_argument("--refine_trans_ccf",
action="store_true",
help="do translational alignment using simple ccf.",
default=False)
parser.add_argument(
"--refine_rot",
action="store_true",
help=
"do translational-rotational alignment. better to start from an existing translational alignment.",
default=False)
parser.add_argument(
"--refine_defocus",
action="store_true",
help=
"do defocus refinement. need aliptcls input. doesn't work with refine_trans or rot yet..",
default=False)
parser.add_argument(
"--use3d",
action="store_true",
help="use projection of 3d particles instead of 2d ones..",
default=False)
parser.add_argument(
"--preprocess",
metavar="processor_name:param1=value1:param2=value2",
type=str,
default=None,
help="Preprocess each 2-D subtilt while loading (alignment only)")
parser.add_argument(
"--aliptcls2d",
type=str,
help=
"optional aliptcls input. the program can start search from the position from last run.",
default="")
parser.add_argument("--aliptcls3d",
type=str,
help="optional aliptcls input.",
default="")
parser.add_argument("--parallel",
type=str,
help="Thread/mpi parallelism to use",
default="thread:4")
parser.add_argument("--debug",
action="store_true",
help="for testing.",
default=False)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv, options.ppid)
options.info2dname = "{}/particle_info_2d.lst".format(options.path)
options.info3dname = "{}/particle_info_3d.lst".format(options.path)
n = EMUtil.get_image_count(options.info2dname)
tasks = list(range(n))
if options.preprocess != None:
options.preprocess = parsemodopt(options.preprocess)
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(options.parallel,
module="e2spt_subtlt_local.SptAlignTask")
num_cpus = etc.cpu_est()
if options.debug:
tasks = tasks[:num_cpus * 4]
print("{} jobs on {} CPUs".format(len(tasks), num_cpus))
njob = num_cpus
tids = []
for i in range(njob):
t = tasks[i::njob]
task = SptAlignTask(t, options)
if options.debug:
ret = task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
output = [None] * len(tasks)
for i in tids:
rets = etc.get_results(i)[1]
for r in rets:
output[r[0]] = r[1]
del etc
fm = "{}/aliptcls2d_{:02d}.lst".format(options.path, options.iter)
save_lst_params(output, fm)
E2end(logid)
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--ptclin",
type=str,
help="particle input",
default=None)
parser.add_argument("--output",
type=str,
help="score output",
default=None)
parser.add_argument(
"--parallel",
type=str,
help="Thread/mpi parallelism to use. Default is thread:12",
default="thread:12")
parser.add_argument(
"--debug",
action="store_true",
default=False,
help=
"Turn on debug mode. This will only process a small subset of the data (threads * 8 particles)"
)
parser.add_argument("--maxres",
type=float,
default=15,
help="max resolution for cmp")
parser.add_argument("--minres",
type=float,
default=300,
help="min resolution for cmp")
parser.add_argument("--sym", type=str, help="symmetry. ", default="c1")
parser.add_argument("--mask", type=str, help="mask. ", default=None)
parser.add_argument("--maxshift", type=int, help="max shift.", default=0)
parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
threedname = args
lst = LSXFile(options.ptclin, True)
pinfo = []
nptcl = lst.n
for i in range(nptcl):
pinfo.append(lst.read(i))
lst = None
e = EMData(options.ptclin, 0, True)
nx = e["nx"]
apix = e["apix_x"]
b = good_size(nx * apix / options.maxres * 2)
options.shrink = max(1, nx / b)
print("Initializing parallelism...")
etc = EMTaskCustomer(options.parallel,
module="e2spa_classify.SpaClassifyTask")
num_cpus = etc.cpu_est()
print("{} total CPUs available".format(num_cpus))
print("{} jobs".format(nptcl))
infos = [[] for i in range(num_cpus)]
for i, info in enumerate(pinfo):
infos[i % num_cpus].append([i, info])
tids = []
for info in infos:
task = SpaClassifyTask(info, threedname, options)
if options.debug:
task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
dics = np.zeros((nptcl, len(threedname)))
for i in tids:
ret = etc.get_results(i)[1]
for r in ret:
#print(r)
ii = r.pop("idx")
dics[ii] = r["score"]
del etc
np.savetxt(options.output, dics)
E2end(logid)
def main():
usage = " "
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--ptclin",
type=str,
help="particle input",
default=None)
parser.add_argument("--ptclout",
type=str,
help="particle output",
default=None)
parser.add_argument("--ref",
type=str,
help="reference input",
default=None)
parser.add_argument("--keep",
type=float,
help="propotion of tilts to keep. default is 0.8",
default=0.8)
parser.add_argument(
"--parallel",
type=str,
help="Thread/mpi parallelism to use. Default is thread:12",
default="thread:12")
parser.add_argument(
"--debug",
action="store_true",
default=False,
help=
"Turn on debug mode. This will only process a small subset of the data"
)
parser.add_argument("--maxshift",
type=int,
help="maximum shift allowed",
default=-1)
parser.add_argument("--localrefine",
action="store_true",
default=False,
help="local refinement")
#parser.add_argument("--ctfweight", action="store_true", default=False ,help="weight by ctf. not used yet...")
parser.add_argument("--slow",
action="store_true",
default=False,
help="slow but finer search")
parser.add_argument("--maxres",
type=float,
default=-1,
help="max resolution for cmp")
parser.add_argument("--minrespx",
type=int,
default=4,
help="skip the first x pixel in fourier space")
parser.add_argument("--sym", type=str, help="symmetry. ", default="c1")
parser.add_argument("--ppid", type=int, help="ppid...", default=-1)
#parser.add_argument("--nkeep", type=int,help="", default=1)
parser.add_argument("--verbose", "-v", type=int, help="Verbose", default=0)
(options, args) = parser.parse_args()
logid = E2init(sys.argv)
lstname = options.ptclin
threedname = options.ref
lname = options.ptclout
lst = LSXFile(lstname, True)
m = EMData(threedname)
bxsz = m["nx"]
apix = m["apix_x"]
options.shrink = 1
pinfo = []
nptcl = lst.n
if options.maxshift < 0:
options.maxshift = bxsz // 2
print("Initializing parallelism...")
etc = EMTaskCustomer(options.parallel, module="e2spa_align.SpaAlignTask")
num_cpus = etc.cpu_est()
print("{} particles".format(nptcl))
print("{} total CPUs available".format(num_cpus))
if options.debug:
nptcl = min(4 * num_cpus, nptcl)
print("Debugging mode. running on one thread with 8 particles")
for i in range(nptcl):
pinfo.append(lst.read(i))
lst = None
nbatch = min(nptcl // 4, num_cpus)
infos = [[] for i in range(nbatch)]
for i, info in enumerate(pinfo):
infos[i % nbatch].append([i, info])
print("{} jobs, each with {:.1f} particles".format(
len(infos), np.mean([len(i) for i in infos])))
tids = []
for info in infos:
task = SpaAlignTask(info, threedname, options)
if options.debug:
task.execute(print)
return
tid = etc.send_task(task)
tids.append(tid)
while 1:
st_vals = etc.check_task(tids)
if -100 in st_vals:
print("Error occurs in parallelism. Exit")
return
E2progress(logid, np.mean(st_vals) / 100.)
if np.min(st_vals) == 100: break
time.sleep(5)
dics = [0] * nptcl
for i in tids:
ret = etc.get_results(i)[1]
for r in ret:
ii = r.pop("idx")
dics[ii] = r
del etc
allscr = [d["score"] for d in dics]
maxl = np.max([len(s) for s in allscr])
maxv = np.max(np.concatenate(allscr))
for s in allscr:
s.extend([maxv] * (maxl - len(s)))
allscr = np.array(allscr)
try:
os.remove(lname)
except:
pass
lout = LSXFile(lname, False)
for i, dc in enumerate(dics):
lc = ""
for j, xf in enumerate(dc["xform.align3d"]):
d = xf.get_params("eman")
d["score"] = float(allscr[i, j])
lc = lc + str(d) + ';'
l = pinfo[i]
lout.write(-1, l[0], l[1], lc[:-1])
lout = None
E2end(logid)