def run_summovie( opt ):
# Lists to write the text files later
micrograph_list = []
# Get the number of files
nr_files = len(opt['mic_list'])
# Timeing stuff
time_start = time.time()
time_list = []
# Loop over all files
for index, inputfile in enumerate(sorted(opt['mic_list'])):
# Check, if there is an prefix and suffix.
# If there is more then one entry: the suffix is the last one.
# Otherwhise its just the one after the dot.
input_suffix = inputfile.split('/')[-1].split('.')[-1]
# First output to introduce the programm
if opt['verbose'] and index == 0:
sxprint(
'Progress: 0.0%; Time: --h:--m:--s/--h:--m:--s; Summovie started!'
)
# Time begin
t1 = time.time()
# Get the output names
file_name = inputfile[len(opt['input_dir']):-len(opt['mic_suffix'])]
file_wildcard = file_name[
len(opt['mic_prefix']):
]
micrograph_name = '{0}/{1}{2}.mrc'.format(
opt['output_path'], file_name, opt['sum_suffix']
)
frc_name = '{0}/{1}{2}.txt'.format(
opt['frc_path'], file_name, opt['frc_suffix']
)
shift_name = '{0}{1}{2}'.format(
opt['shift_prefix'], file_wildcard, opt['shift_suffix']
)
if not opt['summovie_ready']:
temp_name = '{0}/{1}{2}.mrc'.format(
opt['temp_path'], file_name, opt['sum_suffix']
)
else:
temp_name = inputfile
log_name = '{0}/{1}.log'.format(
opt['log_path'], file_name
)
error_name = '{0}/{1}.err'.format(
opt['log_path'], file_name
)
# Append the names to the lists
micrograph_list.append('{0}{1}.mrc'.format(
file_name, opt['sum_suffix'])
)
# First build the summovie command
summovie_command = create_summovie_command(
temp_name,
micrograph_name,
shift_name,
frc_name,
opt
)
# Export the number of threads
export_threads_command = []
# Export
export_threads_command.append('export')
# Nr of threads
export_threads_command.append('OMP_NUM_THREADS={0}'.format(
opt['nr_threads']
))
if not opt['summovie_ready']:
# Do a e2proc3d.py
e2proc3d_command = []
# e2proc3d
e2proc3d_command.append('e2proc3d.py')
# inputfile
e2proc3d_command.append('{0}'.format(inputfile))
# outputfile
e2proc3d_command.append('{0}'.format(temp_name))
# Translate the command to single strings
if not opt['summovie_ready']:
e2proc3d_command = r' '.join(e2proc3d_command)
export_threads_command = r' '.join(export_threads_command)
summovie_command = '\n'.join(summovie_command)
# Build full command
if not opt['summovie_ready']:
full_command = r'{0}; {1}; echo "{2}" | {3}'.format(
export_threads_command,
e2proc3d_command,
summovie_command,
opt['summovie_path']
)
else:
full_command = r'{0}; echo "{1}" | {2}'.format(
export_threads_command,
summovie_command,
opt['summovie_path']
)
# Remove temp summovie files
temp_summovie_files = glob('.SumMovie*')
for entry in temp_summovie_files:
remove(entry)
with open(log_name, 'w') as f:
with open(error_name, 'w') as e:
# Execute Command
subprocess.Popen(
[full_command], shell=True,
stdout=f,
stderr=e
).wait()
# Remove temp summovie files
temp_summovie_files = glob('.SumMovie*')
for entry in temp_summovie_files:
remove(entry)
if not opt['summovie_ready']:
if path.exists(temp_name):
# Remove temp file
remove(temp_name)
else:
ERROR('e2proc2d.py error. File was not created:\n{0}'.format(inputfile), action=0)
time_list.append(time.time() - t1)
# Check if SumMovie finished cleanly
with open(log_name, 'r') as r:
clean = False
for line in r:
if 'SumMovie finished cleanly.' in line:
clean = True
break
if clean:
sxprint('SumMovie finished cleanly.')
else:
ERROR( 'sum movie error. check the logfile for more information: {0}'.format(log_name), action=0 )
# Do progress output
if opt['verbose']:
percent = round(100 * (index + 1) / float(nr_files), 2)
estimated_time = \
nr_files * sum(time_list) / float(len(time_list))
estimated_time_h = estimated_time // 3600
estimated_time_m = (estimated_time - estimated_time_h*3600) // 60
estimated_time_s = (
estimated_time -
estimated_time_h*3600 -
estimated_time_m*60
)
current_time = time.time() - time_start
current_time_h = current_time // 3600
current_time_m = (current_time - current_time_h*3600) // 60
current_time_s = (
current_time -
current_time_h*3600 -
current_time_m*60
)
sxprint((
'Progress: {0:.2f}%; Time: {1:.0f}h:{2:.0f}m:{3:.0f}s/{4:.0f}h:{5:.0f}m:{6:.0f}s; Micrograph done:{7}'.format(
percent,
current_time_h,
current_time_m,
current_time_s,
estimated_time_h,
estimated_time_m,
estimated_time_s,
file_name
)
))
# Write micrograph list
with open('{0}/summovie_micrographs.txt'.format(opt['output_dir']), 'w') as f:
for entry in sorted(micrograph_list):
f.write('{0}\n'.format(entry))
def run_unblur(
unblur_path,
input_image,
input_dir,
output_dir,
corrected_path,
uncorrected_path,
shift_path,
frc_path,
temp_path,
log_path,
file_list,
options,
):
# Lists to write the text files later
micrograph_list = []
shift_list = []
if options.save_frames:
frames_list = []
# If micrograph list is provided just process the images in the list
mic_list = options.selection_list
if mic_list:
# Import list file
try:
set_selection = numpy.genfromtxt(mic_list, dtype=None)
except TypeError:
sp_global_def.ERROR("no entrys in list file {0}".format(mic_list))
# List of files which are in pattern and list
file_list = [
entry for entry in file_list if
entry[len(input_dir):] in set_selection and os.path.exists(entry)
]
# If no match is there abort
if len(file_list) == 0:
sp_global_def.ERROR(
"no files in {0} matched the file pattern:\n".format(mic_list),
1)
# Get the number of files
nr_files = len(file_list)
# Timeing stuff
time_start = time.time()
time_list = []
# Loop over all files
for index, inputfile in enumerate(sorted(file_list)):
# Check, if there is an prefix and suffix.
# If there is more then one entry: the suffix is the last one.
# Otherwhise its just the one after the dot.
input_suffix = inputfile.split("/")[-1].split(".")[-1]
# First output to introduce the programm
if index == 0:
sp_global_def.sxprint(
"Progress: 0.0%; Time: --h:--m:--s/--h:--m:--s; Unblur started!"
)
# Time begin
t1 = time.time()
# Get the output names
file_name = inputfile[len(input_dir):-len(input_suffix) - 1]
if options.skip_dose_filter:
micrograph_name = "{0}/{1}{2}.mrc".format(uncorrected_path,
file_name,
options.sum_suffix)
frames_name = "{0}/{1}{2}.mrc".format(uncorrected_path, file_name,
options.frames_suffix)
frc_name = "{0}/{1}{2}.txt".format(frc_path, file_name,
options.frc_suffix)
else:
micrograph_name = "{0}/{1}{2}.mrc".format(corrected_path,
file_name,
options.sum_suffix)
frames_name = "{0}/{1}{2}.mrc".format(corrected_path, file_name,
options.frames_suffix)
micrograph_name_skip = "{0}/{1}{2}.mrc".format(
uncorrected_path, file_name, options.sum_suffix)
frames_name_skip = "{0}/{1}{2}.mrc".format(uncorrected_path,
file_name,
options.frames_suffix)
frc_name = "{0}/{1}{2}.txt".format(frc_path, file_name,
options.frc_suffix)
frc_summovie_name = "{0}/{1}_summovie{2}.txt".format(
frc_path, file_name, options.frc_suffix)
shift_name = "{0}/{1}{2}.txt".format(shift_path, file_name,
options.shift_suffix)
if not options.unblur_ready:
temp_name = "{0}/{1}{2}.mrc".format(temp_path, file_name,
options.sum_suffix)
else:
temp_name = inputfile
log_name = "{0}/{1}.log".format(log_path, file_name)
error_name = "{0}/{1}.err".format(log_path, file_name)
# Append the names to the lists
micrograph_list.append("{0}{1}.mrc".format(file_name,
options.sum_suffix))
shift_list.append(shift_name)
if options.save_frames:
frames_list.append("{0}{1}.mrc".format(file_name,
options.frames_suffix))
# First build the unblur/summovie command
if not options.skip_dose_filter:
unblur_command = create_unblur_command(temp_name, micrograph_name,
shift_name, frames_name,
frc_name, options)
# Options for the summovie command
options_summovie = {
"first": 1,
"last": -1,
"nr_frames": options.nr_frames,
"pixel_size": options.pixel_size,
"exposure_per_frame": options.exposure_per_frame,
"voltage": options.voltage,
"pre_exposure": options.pre_exposure,
"dont_restore_noise": options.dont_restore_noise,
"apply_dose_filter": False,
}
summovie_command = sp_utilities.create_summovie_command(
temp_name,
micrograph_name_skip,
shift_name,
frc_summovie_name,
options_summovie,
)
else:
unblur_command = create_unblur_command(temp_name, micrograph_name,
shift_name, frc_name,
frames_name, options)
# Export the number of threads
export_threads_command = []
# Export
export_threads_command.append("export")
# Nr of threads
export_threads_command.append("OMP_NUM_THREADS={0}".format(
options.nr_threads))
if not options.unblur_ready:
# Do a e2proc3d.py
e2proc3d_command = []
# e2proc3d
e2proc3d_command.append("e2proc3d.py")
# inputfile
e2proc3d_command.append("{0}".format(inputfile))
# outputfile
e2proc3d_command.append("{0}".format(temp_name))
# Translate the command to single strings
if not options.unblur_ready:
e2proc3d_command = r" ".join(e2proc3d_command)
export_threads_command = r" ".join(export_threads_command)
unblur_command = "\n".join(unblur_command)
if not options.skip_dose_filter:
summovie_command = "\n".join(summovie_command)
# Build full command
if not options.unblur_ready:
if not options.skip_dose_filter:
full_command = r'{0}; {1}; echo "{2}" | {3}'.format(
export_threads_command,
e2proc3d_command,
unblur_command,
unblur_path,
)
full_command_summovie = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, summovie_command,
options.summovie_path)
else:
full_command = r'{0}; {1}; echo "{2}" | {3}'.format(
export_threads_command,
e2proc3d_command,
unblur_command,
unblur_path,
)
else:
if not options.skip_dose_filter:
full_command = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, unblur_command, unblur_path)
full_command_summovie = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, summovie_command,
options.summovie_path)
else:
full_command = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, unblur_command, unblur_path)
# Remove temp unblur files
temp_unblur_files = glob.glob(".UnBlur*")
for entry in temp_unblur_files:
os.remove(entry)
# Remove temp summovie files
temp_summovie_files = glob.glob(".SumMovie*")
for entry in temp_summovie_files:
os.remove(entry)
with open(log_name, "w") as f:
with open(error_name, "w") as e:
# Execute Command
if not options.skip_dose_filter:
subprocess.Popen([full_command],
shell=True,
stdout=f,
stderr=e).wait()
# Remove temp unblur files
temp_unblur_files = glob.glob(".UnBlur*")
for entry in temp_unblur_files:
os.remove(entry)
# Remove temp summovie files
temp_summovie_files = glob.glob(".SumMovie*")
for entry in temp_summovie_files:
os.remove(entry)
subprocess.Popen([full_command_summovie],
shell=True,
stdout=f,
stderr=e).wait()
else:
subprocess.Popen([full_command],
shell=True,
stdout=f,
stderr=e).wait()
# Remove temp unblur files
temp_unblur_files = glob.glob(".UnBlur*")
for entry in temp_unblur_files:
os.remove(entry)
# Remove temp summovie files
temp_summovie_files = glob.glob(".SumMovie*")
for entry in temp_summovie_files:
os.remove(entry)
if not options.unblur_ready:
if os.path.exists(temp_name):
# Remove temp file
os.remove(temp_name)
else:
sp_global_def.sxprint(
("Error with file:\n{0}".format(inputfile)))
# Check if SumMovie and UnBlur finished cleanly
with open(log_name, "r") as r:
clean_summovie = False
clean_unblur = False
for line in r:
if "SumMovie finished cleanly." in line:
clean_summovie = True
if "UnBlur finished cleanly." in line:
clean_unblur = True
if clean_unblur:
sp_global_def.sxprint("UnBlur finished cleanly.")
else:
sp_global_def.ERROR(
"unblur error. check the logfile for more information: {0}".
format(log_name),
action=0,
)
if clean_summovie:
sp_global_def.sxprint("SumMovie finished cleanly.")
else:
sp_global_def.ERROR(
"summovie error. check the logfile for more information: {0}".
format(log_name),
action=0,
)
time_list.append(time.time() - t1)
# Do progress output
percent = round(100 * (index + 1) / float(nr_files), 2)
estimated_time = nr_files * sum(time_list) / float(len(time_list))
estimated_time_h = estimated_time // 3600
estimated_time_m = (estimated_time - estimated_time_h * 3600) // 60
estimated_time_s = (estimated_time - estimated_time_h * 3600 -
estimated_time_m * 60)
current_time = time.time() - time_start
current_time_h = current_time // 3600
current_time_m = (current_time - current_time_h * 3600) // 60
current_time_s = current_time - current_time_h * 3600 - current_time_m * 60
sp_global_def.sxprint((
"Progress: {0:.2f}%; Time: {1:.0f}h:{2:.0f}m:{3:.0f}s/{4:.0f}h:{5:.0f}m:{6:.0f}s; Micrograph done:{7}"
.format(
percent,
current_time_h,
current_time_m,
current_time_s,
estimated_time_h,
estimated_time_m,
estimated_time_s,
file_name,
)))
# Write micrograph and shift list
with open("{0}/unblur_micrographs.txt".format(output_dir), "w") as f:
for entry in sorted(micrograph_list):
f.write("{0}\n".format(entry))
with open("{0}/unblur_shiftfiles.txt".format(output_dir), "w") as f:
for entry in sorted(shift_list):
f.write("{0}\n".format(entry))
if options.save_frames:
with open("{0}/unblur_frames.txt".format(output_dir), "w") as f:
for entry in sorted(frames_list):
f.write("{0}\n".format(entry))
def run_summovie(opt):
# Lists to write the text files later
micrograph_list = []
# Get the number of files
nr_files = len(opt["mic_list"])
# Timeing stuff
time_start = time.time()
time_list = []
# Loop over all files
for index, inputfile in enumerate(sorted(opt["mic_list"])):
# Check, if there is an prefix and suffix.
# If there is more then one entry: the suffix is the last one.
# Otherwhise its just the one after the dot.
input_suffix = inputfile.split("/")[-1].split(".")[-1]
# First output to introduce the programm
if opt["verbose"] and index == 0:
sp_global_def.sxprint(
"Progress: 0.0%; Time: --h:--m:--s/--h:--m:--s; Summovie started!"
)
# Time begin
t1 = time.time()
# Get the output names
file_name = inputfile[len(opt["input_dir"]) : -len(opt["mic_suffix"])]
file_wildcard = file_name[len(opt["mic_prefix"]) :]
micrograph_name = "{0}/{1}{2}.mrc".format(
opt["output_path"], file_name, opt["sum_suffix"]
)
frc_name = "{0}/{1}{2}.txt".format(
opt["frc_path"], file_name, opt["frc_suffix"]
)
shift_name = "{0}{1}{2}".format(
opt["shift_prefix"], file_wildcard, opt["shift_suffix"]
)
if not opt["summovie_ready"]:
temp_name = "{0}/{1}{2}.mrc".format(
opt["temp_path"], file_name, opt["sum_suffix"]
)
else:
temp_name = inputfile
log_name = "{0}/{1}.log".format(opt["log_path"], file_name)
error_name = "{0}/{1}.err".format(opt["log_path"], file_name)
# Append the names to the lists
micrograph_list.append("{0}{1}.mrc".format(file_name, opt["sum_suffix"]))
# First build the summovie command
summovie_command = sp_utilities.create_summovie_command(
temp_name, micrograph_name, shift_name, frc_name, opt
)
# Export the number of threads
export_threads_command = []
# Export
export_threads_command.append("export")
# Nr of threads
export_threads_command.append("OMP_NUM_THREADS={0}".format(opt["nr_threads"]))
if not opt["summovie_ready"]:
# Do a e2proc3d.py
e2proc3d_command = []
# e2proc3d
e2proc3d_command.append("e2proc3d.py")
# inputfile
e2proc3d_command.append("{0}".format(inputfile))
# outputfile
e2proc3d_command.append("{0}".format(temp_name))
# Translate the command to single strings
if not opt["summovie_ready"]:
e2proc3d_command = r" ".join(e2proc3d_command)
export_threads_command = r" ".join(export_threads_command)
summovie_command = "\n".join(summovie_command)
# Build full command
if not opt["summovie_ready"]:
full_command = r'{0}; {1}; echo "{2}" | {3}'.format(
export_threads_command,
e2proc3d_command,
summovie_command,
opt["summovie_path"],
)
else:
full_command = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, summovie_command, opt["summovie_path"]
)
# Remove temp summovie files
temp_summovie_files = glob.glob(".SumMovie*")
for entry in temp_summovie_files:
os.remove(entry)
with open(log_name, "w") as f:
with open(error_name, "w") as e:
# Execute Command
subprocess.Popen([full_command], shell=True, stdout=f, stderr=e).wait()
# Remove temp summovie files
temp_summovie_files = glob.glob(".SumMovie*")
for entry in temp_summovie_files:
os.remove(entry)
if not opt["summovie_ready"]:
if os.path.exists(temp_name):
# Remove temp file
os.remove(temp_name)
else:
sp_global_def.ERROR(
"e2proc2d.py error. File was not created:\n{0}".format(inputfile),
action=0,
)
time_list.append(time.time() - t1)
# Check if SumMovie finished cleanly
with open(log_name, "r") as r:
clean = False
for line in r:
if "SumMovie finished cleanly." in line:
clean = True
break
if clean:
sp_global_def.sxprint("SumMovie finished cleanly.")
else:
sp_global_def.ERROR(
"sum movie error. check the logfile for more information: {0}".format(
log_name
),
action=0,
)
# Do progress output
if opt["verbose"]:
percent = round(100 * (index + 1) / float(nr_files), 2)
estimated_time = nr_files * sum(time_list) / float(len(time_list))
estimated_time_h = estimated_time // 3600
estimated_time_m = (estimated_time - estimated_time_h * 3600) // 60
estimated_time_s = (
estimated_time - estimated_time_h * 3600 - estimated_time_m * 60
)
current_time = time.time() - time_start
current_time_h = current_time // 3600
current_time_m = (current_time - current_time_h * 3600) // 60
current_time_s = current_time - current_time_h * 3600 - current_time_m * 60
sp_global_def.sxprint(
(
"Progress: {0:.2f}%; Time: {1:.0f}h:{2:.0f}m:{3:.0f}s/{4:.0f}h:{5:.0f}m:{6:.0f}s; Micrograph done:{7}".format(
percent,
current_time_h,
current_time_m,
current_time_s,
estimated_time_h,
estimated_time_m,
estimated_time_s,
file_name,
)
)
)
# Write micrograph list
with open("{0}/summovie_micrographs.txt".format(opt["output_dir"]), "w") as f:
for entry in sorted(micrograph_list):
f.write("{0}\n".format(entry))