def execute(self, alignment_jobs, files, caller):
'''
The main function
@param alignment_jobs a list of alignment pair indices like this [[0,1],[2,1],[2,3],[0,5],...] etc the indices pair represent images to be aligned and correspond to the order of the files argument
@param files a list of filenames - used to read image based on the indices present in alignment_jobs
@param caller - the calling object - it needs to have a function called process_output that takes a dictionary as the argument
'''
options = self.options
align_data = EMAN2.parsemodopt(options.align)
align_cmp_data = EMAN2.parsemodopt(options.aligncmp)
cmp_data = EMAN2.parsemodopt(options.cmp)
ralign_data = None
if options.ralign != None:
ralign_data = EMAN2.parsemodopt(options.ralign)
ralign_cmp_data = EMAN2.parsemodopt(options.raligncmp)
data = {}
data["align"] = align_data
data["aligncmp"] = align_cmp_data
data["cmp"] = cmp_data
if ralign_data:
data["ralign"] = ralign_data
data["raligncmp"] = ralign_cmp_data
data["using_cuda"] = self.using_cuda
data["nsoln"] = self.nsoln
if self.options.parallel:
task_customers = []
tids = []
if options.shrink:
scratch_name_1 = numbered_bdb(
"bdb:tomo_scratch#scratch_shrink")
scratch_name_2 = numbered_bdb(
"bdb:tomo_scratch##scratch_shrink")
else:
print("no shrink")
for i, j in alignment_jobs:
if options.shrink or options.filter:
a = EMData(files[i], 0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0], filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",
{"n": options.shrink})
a.set_attr("src_image", files[i])
a.write_image(scratch_name_1, 0)
a = EMData(files[j], 0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0], filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",
{"n": options.shrink})
a.set_attr("src_image", files[j])
a.write_image(scratch_name_2, 0)
data["probe"] = ("cache", scratch_name_1, 0)
data["target"] = ("cache", scratch_name_2, 0)
else:
data["probe"] = ("cache", files[i], 0)
data["target"] = ("cache", files[j], 0)
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTaskDC(data=data)
from EMAN2PAR import EMTaskCustomer
etc = EMTaskCustomer(self.options.parallel)
#print "Est %d CPUs"%etc.cpu_est()
tid = etc.send_task(task)
#print "Task submitted tid=",tid
task_customers.append(etc)
tids.append(tid)
self.dc_monitor(task_customers, tids, caller)
else:
n = len(alignment_jobs)
p = 0.0
for i, j in alignment_jobs:
probe = EMData(files[i], 0)
target = EMData(files[j], 0)
if options.filter:
print("filtered")
filter_params = EMAN2.parsemodopt(options.filter)
probe.process_inplace(filter_params[0], filter_params[1])
target.process_inplace(filter_params[0], filter_params[1])
if options.shrink:
probe.process_inplace("math.meanshrink",
{"n": options.shrink})
target.process_inplace("math.meanshrink",
{"n": options.shrink})
else:
print("no shrink")
data["target"] = target
data["probe"] = probe
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTask(data=data)
rslts = task.execute(self.progress_callback)
if options.shrink:
self.correction_translation(rslts, options.shrink)
caller.process_output(rslts)
p += 1.0
def execute(self,alignment_jobs,files,caller):
'''
The main function
@param alignment_jobs a list of alignment pair indices like this [[0,1],[2,1],[2,3],[0,5],...] etc the indices pair represent images to be aligned and correspond to the order of the files argument
@param files a list of filenames - used to read image based on the indices present in alignment_jobs
@param caller - the calling object - it needs to have a function called process_output that takes a dictionary as the argument
'''
options = self.options
align_data = EMAN2.parsemodopt(options.align)
align_cmp_data = EMAN2.parsemodopt(options.aligncmp)
cmp_data = EMAN2.parsemodopt(options.cmp)
ralign_data = None
if options.ralign != None:
ralign_data = EMAN2.parsemodopt(options.ralign)
ralign_cmp_data = EMAN2.parsemodopt(options.raligncmp)
data = {}
data["align"] = align_data
data["aligncmp"] = align_cmp_data
data["cmp"] = cmp_data
if ralign_data:
data["ralign"] = ralign_data
data["raligncmp"] = ralign_cmp_data
data["using_cuda"] = self.using_cuda
data["nsoln"] = self.nsoln
if self.options.parallel :
task_customers = []
tids = []
if options.shrink:
scratch_name_1 = numbered_bdb("bdb:tomo_scratch#scratch_shrink")
scratch_name_2 = numbered_bdb("bdb:tomo_scratch##scratch_shrink")
else: print "no shrink"
for i,j in alignment_jobs:
if options.shrink or options.filter:
a = EMData(files[i],0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",{"n":options.shrink})
a.set_attr("src_image",files[i])
a.write_image(scratch_name_1,0)
a = EMData(files[j],0)
if options.filter:
filter_params = EMAN2.parsemodopt(options.filter)
a.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
a.process_inplace("math.meanshrink",{"n":options.shrink})
a.set_attr("src_image",files[j])
a.write_image(scratch_name_2,0)
data["probe"] = ("cache",scratch_name_1,0)
data["target"] = ("cache",scratch_name_2,0)
else:
data["probe"] = ("cache",files[i],0)
data["target"] = ("cache",files[j],0)
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTaskDC(data=data)
from EMAN2PAR import EMTaskCustomer
etc=EMTaskCustomer(self.options.parallel)
#print "Est %d CPUs"%etc.cpu_est()
tid=etc.send_task(task)
#print "Task submitted tid=",tid
task_customers.append(etc)
tids.append(tid)
self.dc_monitor(task_customers,tids,caller)
else:
n = len(alignment_jobs)
p = 0.0
for i,j in alignment_jobs:
probe = EMData(files[i],0)
target = EMData(files[j],0)
if options.filter:
print "filtered"
filter_params = EMAN2.parsemodopt(options.filter)
probe.process_inplace(filter_params[0],filter_params[1])
target.process_inplace(filter_params[0],filter_params[1])
if options.shrink:
probe.process_inplace("math.meanshrink",{"n":options.shrink})
target.process_inplace("math.meanshrink",{"n":options.shrink})
else:
print "no shrink"
data["target"] = target
data["probe"] = probe
data["target_idx"] = j
data["probe_idx"] = i
task = EMTomoAlignTask(data=data)
rslts = task.execute(self.progress_callback)
if options.shrink:
self.correction_translation(rslts,options.shrink)
caller.process_output(rslts)
p += 1.0
def execute(self):
'''
The main function - executes the job of performing all v all boot strapped probe generation
'''
if self.logger: E2progress(self.logger, 0.0)
# all_v_all_cmd = self.get_all_v_all_cmd()
# all_v_all_output = self.get_all_v_all_output()
#
# # NOTE: calling the allvall program is probably not strictly necessary, seeing
# # as there is a generic framework for generating and executing alignment jobs
# # implemented below that would be easily adaptable to this - however I left it
# # because doing it this way is absolutely equivalent and has the same cost.
# all_v_all_cmd += " --output="+all_v_all_output
# print "executing",all_v_all_cmd
# if self.logger: E2progress(self.logger,0.01)
# if ( launch_childprocess(all_v_all_cmd) != 0 ):
# print "Failed to execute %s" %all_v_all_cmd
# sys.exit(1)
# if self.logger: E2progress(self.logger,0.02)
#
# images = []
# images.append(EMData(all_v_all_output,0))
# images.append(EMData(all_v_all_output,1))
# images.append(EMData(all_v_all_output,2))
# images.append(EMData(all_v_all_output,3))
# images.append(EMData(all_v_all_output,4))
# images.append(EMData(all_v_all_output,5))
# images.append(EMData(all_v_all_output,6))
#
start_n = len(self.files) # the number of averages produced
images = []
e = EMData(start_n, start_n)
e.to_zero()
images.append(e)
for j in range(6):
images.append(e.copy())
# keep tracks of the names of the new files
big_n = images[0].get_xsize() * (images[0].get_xsize() - 1) / 2.0
iter = 1
current_files = self.files
alignment_jobs = [] # a list of comparisons to be performed
for i in range(len(current_files)):
for j in range(i + 1, len(current_files)):
alignment_jobs.append([i, j])
self.register_current_images(images)
self.register_current_files(self.files)
alignments_manager = EMTomoAlignments(self.options)
alignments_manager.execute(alignment_jobs, self.files, self)
self.write_current_images(self.files)
# this loop
while True:
couples = self.get_couples(images[0])
taken = list(range(images[0].get_xsize()))
done = False
if len(couples) == 1 and len(taken) == 2: done = True
#print len(couples),len(taken)
new_files = []
# write the averages of the couples to disk, store the new names
for i, j in couples:
image_1 = EMData(current_files[j], 0)
image_2 = EMData(current_files[i], 0)
image_1_weight = 1
if image_1.has_attr("total_inc"):
image_1_weight = image_1["total_inc"]
image_2_weight = 1
if image_2.has_attr("total_inc"):
image_2_weight = image_2["total_inc"]
total_weight = image_1_weight + image_2_weight
image_1.mult(old_div(float(image_1_weight), total_weight))
image_2.mult(old_div(float(image_2_weight), total_weight))
d = {}
d["type"] = "eman"
d["tx"] = images[1].get(i, j)
d["ty"] = images[2].get(i, j)
d["tz"] = images[3].get(i, j)
d["az"] = images[4].get(i, j)
d["alt"] = images[5].get(i, j)
d["phi"] = images[6].get(i, j)
t = Transform(d)
image_1.process_inplace("xform", {"transform": t})
image_2 += image_1
image_2.set_attr(
"src_image",
current_files[j]) # so we can recollect how it was created
image_2.set_attr(
"added_src_image",
current_files[i]) # so we can recollect how it was created
image_2.set_attr("added_src_transform",
t) # so we can recollect how it was created
image_2.set_attr("added_src_cmp", images[0](
i, j)) # so we can recollect how it was created
image_2.set_attr(
"total_inc",
total_weight) # so we can recollect how it was created
output_name = numbered_bdb("bdb:" + self.options.path +
"#tomo_ave_0" + str(iter - 1))
image_2.write_image(output_name, 0)
if self.options.dbls: self.save_to_workflow_db(output_name)
new_files.append(output_name)
taken.remove(i)
taken.remove(j)
if done: break
num_new = len(new_files) # the number of averages produced
new_n = len(new_files) + len(taken)
new_images = []
e = EMData(new_n, new_n)
e.to_zero()
new_images.append(e)
for j in range(6):
new_images.append(e.copy())
for i, idxi in enumerate(taken):
new_files.append(current_files[idxi])
for j, idxj in enumerate(taken):
if i == j: continue
else:
new_images[0].set(num_new + i, num_new + j,
images[0].get(idxi, idxj))
new_images[1].set(num_new + i, num_new + j,
images[1].get(idxi, idxj))
new_images[2].set(num_new + i, num_new + j,
images[2].get(idxi, idxj))
new_images[3].set(num_new + i, num_new + j,
images[3].get(idxi, idxj))
new_images[4].set(num_new + i, num_new + j,
images[4].get(idxi, idxj))
new_images[5].set(num_new + i, num_new + j,
images[5].get(idxi, idxj))
new_images[6].set(num_new + i, num_new + j,
images[6].get(idxi, idxj))
alignment_jobs = [] # a list of comparisons to be performed
for i in range(num_new):
for j in range(i + 1, len(new_files)):
alignment_jobs.append([i, j])
if self.logger:
E2progress(self.logger,
1.0 - old_div(len(alignment_jobs), big_n))
self.register_current_images(new_images)
self.register_current_files(new_files)
alignments_manager = EMTomoAlignments(self.options)
alignments_manager.execute(alignment_jobs, new_files, self)
self.write_current_images(new_files)
current_files = new_files
images = new_images
iter += 1
print(couples, taken)
if self.logger: E2progress(self.logger, 1.0)
def execute(self):
'''
The main function - executes the job of performing all v all boot strapped probe generation
'''
if self.logger: E2progress(self.logger,0.0)
# all_v_all_cmd = self.get_all_v_all_cmd()
# all_v_all_output = self.get_all_v_all_output()
#
# # NOTE: calling the allvall program is probably not strictly necessary, seeing
# # as there is a generic framework for generating and executing alignment jobs
# # implemented below that would be easily adaptable to this - however I left it
# # because doing it this way is absolutely equivalent and has the same cost.
# all_v_all_cmd += " --output="+all_v_all_output
# print "executing",all_v_all_cmd
# if self.logger: E2progress(self.logger,0.01)
# if ( launch_childprocess(all_v_all_cmd) != 0 ):
# print "Failed to execute %s" %all_v_all_cmd
# sys.exit(1)
# if self.logger: E2progress(self.logger,0.02)
#
# images = []
# images.append(EMData(all_v_all_output,0))
# images.append(EMData(all_v_all_output,1))
# images.append(EMData(all_v_all_output,2))
# images.append(EMData(all_v_all_output,3))
# images.append(EMData(all_v_all_output,4))
# images.append(EMData(all_v_all_output,5))
# images.append(EMData(all_v_all_output,6))
#
start_n = len(self.files) # the number of averages produced
images = []
e = EMData(start_n,start_n)
e.to_zero()
images.append(e)
for j in range(6): images.append(e.copy())
# keep tracks of the names of the new files
big_n = images[0].get_xsize()*(images[0].get_xsize()-1)/2.0
iter = 1
current_files = self.files
alignment_jobs = []# a list of comparisons to be performed
for i in range(len(current_files)):
for j in range(i+1,len(current_files)):
alignment_jobs.append([i,j])
self.register_current_images(images)
self.register_current_files(self.files)
alignments_manager = EMTomoAlignments(self.options)
alignments_manager.execute(alignment_jobs, self.files,self)
self.write_current_images(self.files)
# this loop
while True:
couples = self.get_couples(images[0])
taken = range(images[0].get_xsize())
done = False
if len(couples) == 1 and len(taken) == 2: done = True
#print len(couples),len(taken)
new_files = []
# write the averages of the couples to disk, store the new names
for i,j in couples:
image_1 = EMData(current_files[j],0)
image_2 = EMData(current_files[i],0)
image_1_weight = 1
if image_1.has_attr("total_inc"): image_1_weight = image_1["total_inc"]
image_2_weight = 1
if image_2.has_attr("total_inc"): image_2_weight = image_2["total_inc"]
total_weight = image_1_weight+image_2_weight
image_1.mult(float(image_1_weight)/total_weight)
image_2.mult(float(image_2_weight)/total_weight)
d = {}
d["type"] = "eman"
d["tx"] = images[1].get(i,j)
d["ty"] = images[2].get(i,j)
d["tz"] = images[3].get(i,j)
d["az"] = images[4].get(i,j)
d["alt"] = images[5].get(i,j)
d["phi"] = images[6].get(i,j)
t = Transform(d)
image_1.process_inplace("xform",{"transform":t})
image_2 += image_1
image_2.set_attr("src_image",current_files[j]) # so we can recollect how it was created
image_2.set_attr("added_src_image",current_files[i]) # so we can recollect how it was created
image_2.set_attr("added_src_transform",t) # so we can recollect how it was created
image_2.set_attr("added_src_cmp",images[0](i,j)) # so we can recollect how it was created
image_2.set_attr("total_inc",total_weight) # so we can recollect how it was created
output_name = numbered_bdb("bdb:"+self.options.path+"#tomo_ave_0"+str(iter-1))
image_2.write_image(output_name,0)
if self.options.dbls: self.save_to_workflow_db(output_name)
new_files.append(output_name)
taken.remove(i)
taken.remove(j)
if done: break
num_new = len(new_files) # the number of averages produced
new_n = len(new_files) + len(taken)
new_images = []
e = EMData(new_n,new_n)
e.to_zero()
new_images.append(e)
for j in range(6): new_images.append(e.copy())
for i,idxi in enumerate(taken):
new_files.append(current_files[idxi])
for j,idxj in enumerate(taken):
if i == j: continue
else:
new_images[0].set(num_new+i,num_new+j,images[0].get(idxi,idxj))
new_images[1].set(num_new+i,num_new+j,images[1].get(idxi,idxj))
new_images[2].set(num_new+i,num_new+j,images[2].get(idxi,idxj))
new_images[3].set(num_new+i,num_new+j,images[3].get(idxi,idxj))
new_images[4].set(num_new+i,num_new+j,images[4].get(idxi,idxj))
new_images[5].set(num_new+i,num_new+j,images[5].get(idxi,idxj))
new_images[6].set(num_new+i,num_new+j,images[6].get(idxi,idxj))
alignment_jobs = []# a list of comparisons to be performed
for i in range(num_new):
for j in range(i+1,len(new_files)):
alignment_jobs.append([i,j])
if self.logger:
E2progress(self.logger,1.0-len(alignment_jobs)/big_n)
self.register_current_images(new_images)
self.register_current_files(new_files)
alignments_manager = EMTomoAlignments(self.options)
alignments_manager.execute(alignment_jobs, new_files,self)
self.write_current_images(new_files)
current_files = new_files
images = new_images
iter += 1
print couples,taken
if self.logger: E2progress(self.logger,1.0)