def main():
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from logger import Logger, BaseLogger_Files
import sys, os, time
global Tracker, Blockdata
from global_def import ERROR
progname = os.path.basename(sys.argv[0])
usage = progname + " --output_dir=output_dir --isac_dir=output_dir_of_isac "
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--adjust_to_analytic_model",
action="store_true",
default=False,
help="adjust power spectrum of 2-D averages to an analytic model ")
parser.add_option(
"--adjust_to_given_pw2",
action="store_true",
default=False,
help="adjust power spectrum to 2-D averages to given 1D power spectrum"
)
parser.add_option("--B_enhance",
action="store_true",
default=False,
help="using B-factor to enhance 2-D averages")
parser.add_option("--no_adjustment",
action="store_true",
default=False,
help="No power spectrum adjustment")
options_list = []
adjust_to_analytic_model = False
for q in sys.argv[1:]:
if (q[:26] == "--adjust_to_analytic_model"):
adjust_to_analytic_model = True
options_list.append(q)
break
adjust_to_given_pw2 = False
for q in sys.argv[1:]:
if (q[:21] == "--adjust_to_given_pw2"):
adjust_to_given_pw2 = True
options_list.append(q)
break
B_enhance = False
for q in sys.argv[1:]:
if (q[:11] == "--B_enhance"):
B_enhance = True
options_list.append(q)
break
no_adjustment = False
for q in sys.argv[1:]:
if (q[:15] == "--no_adjustment"):
no_adjustment = True
options_list.append(q)
break
if len(options_list) == 0:
if (Blockdata["myid"] == Blockdata["main_node"]):
print(
"specify one of the following options to start: 1. adjust_to_analytic_model; 2. adjust_to_given_pw2; 3. B_enhance; 4. no_adjustment"
)
if len(options_list) > 1:
ERROR(
"The specified options are exclusive. Use only one of them to start",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
# options in common
parser.add_option(
"--isac_dir",
type="string",
default='',
help="ISAC run output directory, input directory for this command")
parser.add_option(
"--output_dir",
type="string",
default='',
help="output directory where computed averages are saved")
parser.add_option("--pixel_size",
type="float",
default=-1.0,
help="pixel_size of raw images")
parser.add_option(
"--fl",
type="float",
default=-1.0,
help=
"low pass filter, =-1, not applied; =1, using FH1 (initial resolution), =2 using FH2 (resolution after local alignment), or user provided value"
)
parser.add_option("--stack",
type="string",
default="",
help="data stack used in ISAC")
parser.add_option("--radius", type="int", default=-1, help="radius")
parser.add_option("--xr",
type="float",
default=-1.0,
help="local alignment search range")
parser.add_option("--ts",
type="float",
default=1.0,
help="local alignment search step")
parser.add_option("--fh",
type="float",
default=-1.,
help="local alignment high frequencies limit")
parser.add_option("--maxit",
type="int",
default=5,
help="local alignment iterations")
parser.add_option("--navg",
type="int",
default=-1,
help="number of aveages")
parser.add_option("--skip_local_alignment",
action="store_true",
default=False,
help="skip local alignment")
parser.add_option(
"--noctf",
action="store_true",
default=False,
help=
"no ctf correction, useful for negative stained data. always ctf for cryo data"
)
if B_enhance:
parser.add_option(
"--B_start",
type="float",
default=10.0,
help=
"start frequency (1./Angstrom) of power spectrum for B_factor estimation"
)
parser.add_option(
"--Bfactor",
type="float",
default=-1.0,
help=
"User defined bactors (e.g. 45.0[A^2]). By default, the program automatically estimates B-factor. "
)
if adjust_to_given_pw2:
parser.add_option("--modelpw",
type="string",
default='',
help="1-D reference power spectrum")
checking_flag = 0
if (Blockdata["myid"] == Blockdata["main_node"]):
if not os.path.exists(options.modelpw): checking_flag = 1
checking_flag = bcast_number_to_all(checking_flag,
Blockdata["main_node"],
MPI_COMM_WORLD)
if checking_flag == 1:
ERROR("User provided power spectrum does not exist",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
(options, args) = parser.parse_args(sys.argv[1:])
Tracker = {}
Constants = {}
Constants["isac_dir"] = options.isac_dir
Constants["masterdir"] = options.output_dir
Constants["pixel_size"] = options.pixel_size
Constants["orgstack"] = options.stack
Constants["radius"] = options.radius
Constants["xrange"] = options.xr
Constants["xstep"] = options.ts
Constants["FH"] = options.fh
Constants["maxit"] = options.maxit
Constants["navg"] = options.navg
Constants["low_pass_filter"] = options.fl
if B_enhance:
Constants["B_start"] = options.B_start
Constants["Bfactor"] = options.Bfactor
if adjust_to_given_pw2: Constants["modelpw"] = options.modelpw
Tracker["constants"] = Constants
# -------------------------------------------------------------
#
# Create and initialize Tracker dictionary with input options # State Variables
#<<<---------------------->>>imported functions<<<---------------------------------------------
from utilities import get_im, bcast_number_to_all, write_text_file, read_text_file, wrap_mpi_bcast, write_text_row
from utilities import cmdexecute
from filter import filt_tanl
from time import sleep
from logger import Logger, BaseLogger_Files
import user_functions
import string
from string import split, atoi, atof
import json
#x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1)
#y_range = x_range
####-----------------------------------------------------------
# Create Master directory
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]:
masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen")
else:
masterdir = Tracker["constants"]["masterdir"]
if (Blockdata["myid"] == Blockdata["main_node"]):
msg = "Postprocessing ISAC 2D averages starts"
print(line, "Postprocessing ISAC 2D averages starts")
if not masterdir:
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "sharpen_" + Tracker["constants"]["isac_dir"]
os.mkdir(masterdir)
else:
if os.path.exists(masterdir):
print("%s already exists" % masterdir)
else:
os.mkdir(masterdir)
li = len(masterdir)
else:
li = 0
li = mpi_bcast(li, 1, MPI_INT, Blockdata["main_node"], MPI_COMM_WORLD)[0]
masterdir = mpi_bcast(masterdir, li, MPI_CHAR, Blockdata["main_node"],
MPI_COMM_WORLD)
masterdir = string.join(masterdir, "")
Tracker["constants"]["masterdir"] = masterdir
log_main = Logger(BaseLogger_Files())
log_main.prefix = Tracker["constants"]["masterdir"] + "/"
while not os.path.exists(Tracker["constants"]["masterdir"]):
print("Node ", Blockdata["myid"], " waiting...",
Tracker["constants"]["masterdir"])
sleep(1)
mpi_barrier(MPI_COMM_WORLD)
if (Blockdata["myid"] == Blockdata["main_node"]):
init_dict = {}
print(Tracker["constants"]["isac_dir"])
Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"],
"2dalignment")
core = read_text_row(
os.path.join(Tracker["directory"], "initial2Dparams.txt"))
for im in xrange(len(core)):
init_dict[im] = core[im]
del core
else:
init_dict = 0
init_dict = wrap_mpi_bcast(init_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
###
if (Blockdata["myid"] == Blockdata["main_node"]):
#Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack")
image = get_im(Tracker["constants"]["orgstack"], 0)
Tracker["constants"]["nnxo"] = image.get_xsize()
try:
ctf_params = image.get_attr("ctf")
if Tracker["constants"]["pixel_size"] == -1.:
Tracker["constants"]["pixel_size"] = ctf_params.apix
except:
print("pixel size value is not given.")
Tracker["ini_shrink"] = float(
get_im(os.path.join(Tracker["directory"], "aqfinal.hdf"),
0).get_xsize()) / Tracker["constants"]["nnxo"]
else:
Tracker["ini_shrink"] = 0
Tracker = wrap_mpi_bcast(Tracker,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
#print(Tracker["constants"]["pixel_size"], "pixel_size")
x_range = max(Tracker["constants"]["xrange"],
int(1. / Tracker["ini_shrink"]) + 1)
y_range = x_range
if (Blockdata["myid"] == Blockdata["main_node"]):
parameters = read_text_row(
os.path.join(Tracker["constants"]["isac_dir"],
"all_parameters.txt"))
else:
parameters = 0
parameters = wrap_mpi_bcast(parameters,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
params_dict = {}
list_dict = {}
#parepare params_dict
if Tracker["constants"]["navg"] < 0:
navg = EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"))
else:
navg = min(
Tracker["constants"]["navg"],
EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf")))
global_dict = {}
ptl_list = []
memlist = []
if (Blockdata["myid"] == Blockdata["main_node"]):
for iavg in xrange(navg):
params_of_this_average = []
image = get_im(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"), iavg)
members = image.get_attr("members")
memlist.append(members)
for im in xrange(len(members)):
abs_id = members[im]
global_dict[abs_id] = [iavg, im]
P = combine_params2( init_dict[abs_id][0], init_dict[abs_id][1], init_dict[abs_id][2], init_dict[abs_id][3], \
parameters[abs_id][0], parameters[abs_id][1]/Tracker["ini_shrink"], parameters[abs_id][2]/Tracker["ini_shrink"], parameters[abs_id][3])
if parameters[abs_id][3] == -1: print("wrong one")
params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0])
ptl_list.append(abs_id)
params_dict[iavg] = params_of_this_average
list_dict[iavg] = members
write_text_row(
params_of_this_average,
os.path.join(Tracker["constants"]["masterdir"],
"params_avg_%03d.txt" % iavg))
ptl_list.sort()
init_params = [None for im in xrange(len(ptl_list))]
for im in xrange(len(ptl_list)):
init_params[im] = [ptl_list[im]] + params_dict[global_dict[
ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
init_params,
os.path.join(Tracker["constants"]["masterdir"],
"init_isac_params.txt"))
else:
params_dict = 0
list_dict = 0
memlist = 0
params_dict = wrap_mpi_bcast(params_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
list_dict = wrap_mpi_bcast(list_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
memlist = wrap_mpi_bcast(memlist,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
# Now computing!
del init_dict
tag_sharpen_avg = 1000
## always apply low pass filter to B_enhanced images to suppress noise in high frequencies
enforced_to_H1 = False
if options.B_enhance:
if Tracker["constants"]["low_pass_filter"] == -1:
print("User does not provide low pass filter")
enforced_to_H1 = True
if navg < Blockdata["nproc"]: # Each CPU do one average
FH_list = [None for im in xrange(navg)]
for iavg in xrange(navg):
if Blockdata["myid"] == iavg:
mlist = [None for i in xrange(len(list_dict[iavg]))]
for im in xrange(len(mlist)):
mlist[im] = get_im(Tracker["constants"]["orgstack"],
list_dict[iavg][im])
set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
FH1 = get_optimistic_res(frc)
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d_before_ali.txt"%iavg))
if not options.skip_local_alignment:
new_average1 = within_group_refinement([mlist[kik] for kik in xrange(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1)
new_average2 = within_group_refinement([mlist[kik] for kik in xrange(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1)
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
FH2 = get_optimistic_res(frc)
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg))
#if Tracker["constants"]["nopwadj"]: # pw adjustment, 1. analytic model 2. PDB model 3. B-facttor enhancement
else:
FH2 = 0.0
FH_list[iavg] = [FH1, FH2]
if options.B_enhance:
new_avg, gb = apply_enhancement(
new_avg, Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"])
print("Process avg %d %f %f %f" %
(iavg, gb, FH1, FH2))
elif options.adjust_to_given_pw2:
roo = read_text_file(Tracker["constants"]["modelpw"], -1)
roo = roo[0] # always put pw in the first column
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], roo)
elif options.adjust_to_analytic_model:
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], None)
elif options.no_adjustment:
pass
print("Process avg %d %f %f" % (iavg, FH1, FH2))
if Tracker["constants"]["low_pass_filter"] != -1.:
if Tracker["constants"]["low_pass_filter"] == 1.:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 2.:
low_pass_filter = FH2
if options.skip_local_alignment: low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"][
"low_pass_filter"]
if low_pass_filter >= 0.45: low_pass_filter = 0.45
new_avg = filt_tanl(new_avg, low_pass_filter, 0.1)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
mpi_barrier(MPI_COMM_WORLD)
for im in xrange(navg): # avg
if im == Blockdata[
"myid"] and Blockdata["myid"] != Blockdata["main_node"]:
send_EMData(new_avg, Blockdata["main_node"], tag_sharpen_avg)
elif Blockdata["myid"] == Blockdata["main_node"]:
if im != Blockdata["main_node"]:
new_avg_other_cpu = recv_EMData(im, tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
else:
new_avg.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
if not options.skip_local_alignment:
if im == Blockdata["myid"]:
write_text_row(
plist,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params_avg_%03d.txt" % im))
if Blockdata["myid"] == im and Blockdata["myid"] != Blockdata[
"main_node"]:
wrap_mpi_send(plist_dict[im], Blockdata["main_node"],
MPI_COMM_WORLD)
elif im != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(im, MPI_COMM_WORLD)
plist_dict[im] = dummy
if im == Blockdata["myid"] and im != Blockdata["main_node"]:
wrap_mpi_send(FH_list[im], Blockdata["main_node"],
MPI_COMM_WORLD)
elif im != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(im, MPI_COMM_WORLD)
FH_list[im] = dummy
else:
if im == Blockdata["myid"] and im != Blockdata["main_node"]:
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif im != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(im, MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi_barrier(MPI_COMM_WORLD)
else:
FH_list = [[0, 0.0, 0.0] for im in xrange(navg)]
image_start, image_end = MPI_start_end(navg, Blockdata["nproc"],
Blockdata["myid"])
if Blockdata["myid"] == Blockdata["main_node"]:
cpu_dict = {}
for iproc in xrange(Blockdata["nproc"]):
local_image_start, local_image_end = MPI_start_end(
navg, Blockdata["nproc"], iproc)
for im in xrange(local_image_start, local_image_end):
cpu_dict[im] = iproc
else:
cpu_dict = 0
cpu_dict = wrap_mpi_bcast(cpu_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
slist = [None for im in xrange(navg)]
ini_list = [None for im in xrange(navg)]
avg1_list = [None for im in xrange(navg)]
avg2_list = [None for im in xrange(navg)]
plist_dict = {}
data_list = [None for im in xrange(navg)]
if Blockdata["myid"] == Blockdata["main_node"]: print("read data")
for iavg in xrange(image_start, image_end):
mlist = [None for i in xrange(len(list_dict[iavg]))]
for im in xrange(len(mlist)):
mlist[im] = get_im(Tracker["constants"]["orgstack"],
list_dict[iavg][im])
set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
data_list[iavg] = mlist
print("read data done %d" % Blockdata["myid"])
#if Blockdata["myid"] == Blockdata["main_node"]: print("start to compute averages")
for iavg in xrange(image_start, image_end):
mlist = data_list[iavg]
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
FH1 = get_optimistic_res(frc)
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d_before_ali.txt"%iavg))
if not options.skip_local_alignment:
new_average1 = within_group_refinement([mlist[kik] for kik in xrange(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH=max(Tracker["constants"]["FH"], FH1), FF=0.1)
new_average2 = within_group_refinement([mlist[kik] for kik in xrange(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou= Tracker["constants"]["radius"], rs=1.0, xrng=[ x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1)
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
plist_dict[iavg] = plist
FH2 = get_optimistic_res(frc)
else:
FH2 = 0.0
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg))
FH_list[iavg] = [iavg, FH1, FH2]
if options.B_enhance:
new_avg, gb = apply_enhancement(
new_avg, Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"])
print("Process avg %d %f %f %f" % (iavg, gb, FH1, FH2))
elif options.adjust_to_given_pw2:
roo = read_text_file(Tracker["constants"]["modelpw"], -1)
roo = roo[0] # always on the first column
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], roo)
print("Process avg %d %f %f" % (iavg, FH1, FH2))
elif adjust_to_analytic_model:
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], None)
print("Process avg %d %f %f" % (iavg, FH1, FH2))
elif options.no_adjustment:
pass
if Tracker["constants"]["low_pass_filter"] != -1.:
new_avg = filt_tanl(new_avg,
Tracker["constants"]["low_pass_filter"],
0.1)
if Tracker["constants"]["low_pass_filter"] != -1.:
if Tracker["constants"]["low_pass_filter"] == 1.:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 2.:
low_pass_filter = FH2
if options.skip_local_alignment: low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"]["low_pass_filter"]
if low_pass_filter >= 0.45: low_pass_filter = 0.45
new_avg = filt_tanl(new_avg, low_pass_filter, 0.1)
else:
if enforced_to_H1: new_avg = filt_tanl(new_avg, FH1, 0.1)
if options.B_enhance: new_avg = fft(new_avg)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
slist[iavg] = new_avg
## send to main node to write
mpi_barrier(MPI_COMM_WORLD)
for im in xrange(navg):
# avg
if cpu_dict[im] == Blockdata[
"myid"] and Blockdata["myid"] != Blockdata["main_node"]:
send_EMData(slist[im], Blockdata["main_node"], tag_sharpen_avg)
elif cpu_dict[im] == Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
slist[im].set_attr("members", memlist[im])
slist[im].write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
elif cpu_dict[im] != Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
new_avg_other_cpu = recv_EMData(cpu_dict[im], tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
if not options.skip_local_alignment:
if cpu_dict[im] == Blockdata["myid"]:
write_text_row(
plist_dict[im],
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params_avg_%03d.txt" % im))
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(plist_dict[im], Blockdata["main_node"],
MPI_COMM_WORLD)
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
plist_dict[im] = dummy
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
else:
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi_barrier(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if not options.skip_local_alignment:
if Blockdata["myid"] == Blockdata["main_node"]:
ali3d_local_params = [None for im in xrange(len(ptl_list))]
for im in xrange(len(ptl_list)):
ali3d_local_params[im] = [ptl_list[im]] + plist_dict[
global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
ali3d_local_params,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params.txt"))
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
else:
if Blockdata["myid"] == Blockdata["main_node"]:
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
mpi_barrier(MPI_COMM_WORLD)
target_xr = 3
target_yr = 3
if (Blockdata["myid"] == 0):
cmd = "{} {} {} {} {} {} {} {} {} {}".format("sxchains.py", os.path.join(Tracker["constants"]["masterdir"],"class_averages.hdf"),\
os.path.join(Tracker["constants"]["masterdir"],"junk.hdf"),os.path.join(Tracker["constants"]["masterdir"],"ordered_class_averages.hdf"),\
"--circular","--radius=%d"%Tracker["constants"]["radius"] , "--xr=%d"%(target_xr+1),"--yr=%d"%(target_yr+1),"--align", ">/dev/null")
junk = cmdexecute(cmd)
cmd = "{} {}".format(
"rm -rf",
os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"))
junk = cmdexecute(cmd)
from mpi import mpi_finalize
mpi_finalize()
exit()
def main():
# Parse the Options
progname = path.basename(argv[0])
usage = progname + """ unblur_path input_micrograph_pattern output_directory
--summovie_path
--selection_list
--nr_frames=nr_frames
--pixel_size=pixel_size
--voltage=voltage
--exposure_per_frame=exposure_per_frame
--pre_exposure=pre_exposure
--nr_threads
--save_frames
--skip_dose_filter
--expert_mode
--shift_initial=shift_initial
--shift_radius=shift_radius
--b_factor=b_factor
--fourier_vertical=fourier_vertical
--fourier_horizontal=fourier_horizontal
--shift_threshold=shift_threshold
--iterations=iterations
--dont_restore_noise
--verbose
sxunblur exists only in non-MPI version.
Perform unblur and with dose filtering and summovie without dose filtering.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --nr_threads=1
Perform unblur with dose filtering and summovie without dose filtering with selection list.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--selection_list=selected_micrograph_file
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --nr_threads=1
Perform unblur without dose filtering.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--nr_frames=25 --pixel_size=1.19 --skip_dose_filter --nr_threads=1
Perform unblur without dose filtering and save the frames.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--nr_frames=25 --pixel_size=1.19 --skip_dose_filter --save_frames --nr_threads=1
Perform unblur with dose filtering and summovie without dose filtering with all options.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --save_frames --expert_mode
--shift_initial=2.0 --shift_radius=200.0 --b_factor=1500.0
--fourier_vertical=1 --fourier_horizontal=1 --shift_threshold=0.1
--iterations=10 --verbose --nr_threads=1
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
'--summovie_path',
type='str',
default='',
help=
'summovie executable path (SPHIRE specific): Specify the file path of summovie executable. (default none)'
)
parser.add_option(
'--selection_list',
type='str',
default='',
help=
'Micrograph selecting list (SPHIRE specific): Specify a name of micrograph selection list text file. The file extension must be \'.txt\'. If this is not provided, all files matched with the micrograph name pattern will be processed. (default none)'
)
parser.add_option(
'--nr_frames',
type='int',
default=3,
help=
'Number of movie frames: The number of movie frames in each input micrograph. (default 3)'
)
parser.add_option('--sum_suffix',
type='str',
default='_sum',
help=SUPPRESS_HELP)
parser.add_option('--shift_suffix',
type='str',
default='_shift',
help=SUPPRESS_HELP)
parser.add_option(
'--pixel_size',
type='float',
default=-1.0,
help=
'Pixel size [A]: The pixel size of input micrographs. (default required float)'
)
parser.add_option(
'--voltage',
type='float',
default=300.0,
help=
'Microscope voltage [kV]: The acceleration voltage of microscope used for imaging. (default 300.0)'
)
parser.add_option(
'--exposure_per_frame',
type='float',
default=2.0,
help=
'Per frame exposure [e/A^2]: The electron dose per frame in e/A^2. (default 2.0)'
)
parser.add_option(
'--pre_exposure',
type='float',
default=0.0,
help=
'Pre-exposure [e/A^2]: The electron does in e/A^2 used for exposure prior to imaging .(default 0.0)'
)
parser.add_option(
'--nr_threads',
type='int',
default=1,
help=
'Number of threads: The number of threads unblur can use. The higher the faster, but it requires larger memory. (default 1)'
)
parser.add_option(
'--save_frames',
action='store_true',
default=False,
help=
'Save aligned movie frames: Save aligned movie frames. This option slows down the process. (default False)'
)
parser.add_option('--frames_suffix',
type='string',
default='_frames',
help=SUPPRESS_HELP)
parser.add_option(
'--skip_dose_filter',
action='store_true',
default=False,
help=
'Skip dose filter step: With this option, voltage, exposure per frame, and pre exposure will be ignored. (default False)'
)
parser.add_option(
'--expert_mode',
action='store_true',
default=False,
help=
'Use expert mode: Requires initial shift, shift radius, b-factor, fourier_vertical, fourier_horizontal, shift threshold, iterations, restore noise, and verbosity options. (default False)'
)
parser.add_option('--frc_suffix',
type='string',
default='_frc',
help=SUPPRESS_HELP)
parser.add_option(
'--shift_initial',
type='float',
default=2.0,
help=
'Minimum shift for initial search [A] (expert mode): Effective with unblur expert mode. (default 2.0)'
)
parser.add_option(
'--shift_radius',
type='float',
default=200.0,
help=
'Outer radius shift limit [A] (expert mode): Effective with unblur expert mode. (default 200.0)'
)
parser.add_option(
'--b_factor',
type='float',
default=1500.0,
help=
'Apply B-factor to images [A^2] (expert mode): Effective with unblur expert mode. (default 1500.0)'
)
parser.add_option(
'--fourier_vertical',
type='int',
default=1,
help=
'Vertical Fourier central mask size (expert mode): The half-width of central vertical line of Fourier mask. Effective with unblur expert mode. (default 1)'
)
parser.add_option(
'--fourier_horizontal',
type='int',
default=1,
help=
'Horizontal Fourier central mask size (expert mode): The half-width of central horizontal line of Fourier mask. Effective with unblur expert mode. (default 1)'
)
parser.add_option(
'--shift_threshold',
type='float',
default=0.1,
help=
'Termination shift threshold (expert mode): Effective with unblur expert mode. (default 0.1)'
)
parser.add_option(
'--iterations',
type='int',
default=10,
help=
'Maximum iterations (expert mode): Effective with unblur expert mode. (default 10)'
)
parser.add_option(
'--dont_restore_noise',
action='store_true',
default=False,
help=
'Do not restore noise power (expert mode): Effective with unblur expert mode. (default False)'
)
parser.add_option(
'--verbose',
action='store_true',
default=False,
help=
'Verbose (expert mode): Effective with unblur expert mode. (default False)'
)
parser.add_option('--unblur_ready',
action='store_true',
default=False,
help=SUPPRESS_HELP)
# list of the options and the arguments
(options, args) = parser.parse_args(argv[1:])
global_def.BATCH = True
# If there arent enough arguments, stop the script
if len(args) != 3:
ERROR("see usage " + usage, 1)
# Convert the realtive parts to absolute ones
unblur_path = path.realpath(args[0]) # unblur_path
input_image = path.realpath(args[1]) # input_micrograph_pattern
output_dir = path.realpath(args[2]) # output_directory
# If the unblur executable file does not exists, stop the script
if not path.exists(unblur_path):
ERROR(
'Unblur directory does not exist, please change' +
' the name and restart the program.', 'sxunblur.py', 1)
# If the output directory exists, stop the script
if path.exists(output_dir):
ERROR(
'Output directory exists, please change' +
' the name and restart the program.', 'sxunblur.py', 1)
# If the input file does not exists, stop the script
file_list = glob(input_image)
if not file_list:
ERROR(
'Input file does not exist, please change' +
' the name and restart the program.', 'sxunblur.py', 1)
# If the skip_dose_filter option is false, the summovie path is necessary
if not options.skip_dose_filter and not path.exists(options.summovie_path):
ERROR(
'Path to the SumMovie executable is necessary when dose weighting is performed.',
'sxunblur.py', 1)
# Output paths
corrected_path = '{:s}/corrsum_dose_filtered'.format(output_dir)
uncorrected_path = '{:s}/corrsum'.format(output_dir)
shift_path = '{:s}/shift'.format(output_dir)
frc_path = '{:s}/frc'.format(output_dir)
log_path = '{:s}/logfiles'.format(output_dir)
temp_path = '{0}/temp'.format(output_dir)
# Split the path of the image name at the "/" Characters.
# The last entry contains the micrograph name.
# Split the micrograph name at the wildcard character for the
# prefix and suffix.
input_split = input_image.split('/')
input_name = input_split[-1].split('*')
# Get the input directory
if len(input_split) != 1:
input_dir = input_image[:-len(input_split[-1])]
else:
input_dir = ''
# Create output directorys
if not path.exists(output_dir):
mkdir(output_dir)
if not path.exists(uncorrected_path):
mkdir(uncorrected_path)
if not path.exists(shift_path):
mkdir(shift_path)
if not path.exists(corrected_path):
if not options.skip_dose_filter:
mkdir(corrected_path)
if not path.exists(frc_path):
if options.expert_mode or not options.skip_dose_filter:
mkdir(frc_path)
if not path.exists(temp_path) and not options.unblur_ready:
mkdir(temp_path)
if not path.exists(log_path):
mkdir(log_path)
# Run unblur
run_unblur(unblur_path=unblur_path,
input_image=input_image,
input_dir=input_dir,
output_dir=output_dir,
corrected_path=corrected_path,
uncorrected_path=uncorrected_path,
shift_path=shift_path,
frc_path=frc_path,
temp_path=temp_path,
log_path=log_path,
file_list=file_list,
options=options)
if not options.unblur_ready:
# Remove temp folder
for entry in glob('{0}/*'.format(temp_path)):
remove(entry)
rmdir(temp_path)
print('All Done!')
global_def.BATCH = False
def main():
import os
import sys
from optparse import OptionParser
from global_def import SPARXVERSION, ERROR
import global_def
arglist = []
for arg in sys.argv:
arglist.append( arg )
progname = os.path.basename(arglist[0])
usage = progname + " stack ref_vol outdir <maskfile> --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --ynumber=y_numbers --txs=translational_search_stepx --delta=angular_step --an=angular_neighborhood --maxit=max_iter --CTF --snr=1.0 --sym=c1 --datasym=symdoc"
parser = OptionParser(usage,version=SPARXVERSION)
#parser.add_option("--ir", type="float", default= -1, help="Inner radius for psi angle search > 0 (set to 1) (Angstroms)")
parser.add_option("--ou", type="float", default= -1, help="Outer radius for psi angle search < int(nx*pixel_size/2)-1 (Angstroms)")
parser.add_option("--rs", type="int", default= 1, help="Step between rings in rotational correlation >0 (set to 1)" )
parser.add_option("--xr", type="string", default= " 4 2 1 1 1", help="Range for translation search in x direction, search within +/-xr (Angstroms) ")
parser.add_option("--txs", type="string", default= "1 1 1 0.5 0.25", help="Step size of the translation search in x directions, search is -xr, -xr+ts, 0, xr-ts, xr (Angstroms)")
parser.add_option("--y_restrict", type="string", default= "-1 -1 -1 -1 -1", help="Range for translational search in y-direction, search is +/-y_restrict in Angstroms. This only applies to local search, i.e., when an is not -1. If y_restrict < 0, then the y search range is set such that it is the same ratio to dp as angular search range is to dphi. For regular ihrsr, y search range is the full range when y_restrict< 0. Default is -1.")
parser.add_option("--ynumber", type="string", default= "4 8 16 32 32", help="Even number of the steps for the search in y direction, search is (-dpp/2,-dpp/2+dpp/ny,,..,0,..,dpp/2-dpp/ny dpp/2]")
parser.add_option("--delta", type="string", default= "10 6 4 3 2", help="Angular step of reference projections")
parser.add_option("--an", type="string", default= "-1", help="Angular neighborhood for local searches")
parser.add_option("--maxit", type="int", default= 30, help="Maximum number of iterations performed for each angular step (set to 30) ")
parser.add_option("--searchit", type="int", default= 1, help="Number of iterations to predict/search before doing reconstruction and updating of reference volume. Default is 1. If maxit=3 and searchit=2, then for each of the 3 inner iterations, 2 iterations of prediction/search will be performed before generating reconstruction.")
parser.add_option("--CTF", action="store_true", default=False, help="CTF correction")
parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data")
parser.add_option("--slowIO", action="store_true", default=False, help="sequential reading data for each processor in MPI mode")
#parser.add_option("--fourvar", action="store_true", default=False, help="compute Fourier variance")
parser.add_option("--apix", type="float", default= -1.0, help="Pixel size in Angstroms")
parser.add_option("--dp", type="float", default= -1.0, help="Helical symmetry axial rise (Angstroms)")
parser.add_option("--dphi", type="float", default= -1.0, help="Helical symmetry azimuthal angle")
#parser.add_option("--MA", action="store_true", default=False, help="predict consistent parameters based on moving average")
parser.add_option("--psi_max", type="float", default= 10.0, help="Maximum psi - how far rotation in plane can can deviate from 90 or 270 degrees")
parser.add_option("--rmin", type="float", default= 0.0, help="Min radius for application of helical symmetry (Angstroms)")
parser.add_option("--rmax", type="float", default= 80.0, help="Max radius for application of helical symmetry (Angstroms)")
parser.add_option("--fract", type="float", default= 0.7, help="Fraction of volume used for application of helical symmetry")
parser.add_option("--sym", type="string", default= "c1", help="Point-group symmetry of the filament")
parser.add_option("--function", type="string", default="helical", help="Name of the reference preparation function (Default: helical)")
parser.add_option("--npad", type="int", default= 2, help="Padding size for 3D reconstruction (default=2)")
parser.add_option("--debug", action="store_true", default=False, help="debug")
parser.add_option("--initial_theta", type="float", default=90.0, help="Intial theta for out-of-plane tilt search, the range will be (initial theta to 90.0 in steps of delta) (default = 90, no out-of-plane tilt)")
parser.add_option("--delta_theta", type="float", default=1.0, help="Delta theta for out-of-plane tilt search (default = 1)")
#parser.add_option("--boundaryavg", action="store_true", default=False, help="boundaryavg")
#parser.add_option("--MA_WRAP", type="int", default= 0, help="do wrapping in MA if MA_WRAP=1, else no wrapping in MA. Default is 0.")
parser.add_option("--seg_ny", type="int", default= 256, help="y dimension of desired segment size, should be related to fract in that fract ~ seg_ny/ny, where ny is dimension of input projections. (pixels)")
parser.add_option("--new", action="store_true", default=False, help="use new version")
parser.add_option("--snake", action="store_true", default=False, help="use snake method")
parser.add_option("--snakeknots", type="int", default= -1, help="maximal number of knots for each filament snake. If take default value -1, it will take nseg//2+1, where nseg is the number of segments in the filament")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 3 or len(args) > 4:
print("usage: " + usage + "\n")
print("Please run '" + progname + " -h' for detailed options")
else:
global_def.BATCH = True
# Convert input arguments in the units/format as expected by ihrsr_MPI in applications.
if options.apix < 0:
ERROR("Please specify pixel size apix","sxheliconlocal",1)
if options.dp < 0 or options.dphi < 0:
ERROR("Please specify helical symmetry parameters dp and dphi","sxheliconlocal",1)
if options.an <= 0 :
ERROR("Angular search range (an) has to be given. Only local searches are permitted.","sxheliconlocal",1)
print(" This code is under development, some instabilities are possible 12/28/2014")
rminp = int((float(options.rmin)/options.apix) + 0.5)
rmaxp = int((float(options.rmax)/options.apix) + 0.5)
from utilities import get_input_from_string, get_im
xr = get_input_from_string(options.xr)
txs = get_input_from_string(options.txs)
y_restrict = get_input_from_string(options.y_restrict)
irp = 1
if options.ou < 0: oup = -1
else: oup = int( (options.ou/options.apix) + 0.5)
xrp = ""
txsp = ""
y_restrict2 = ""
for i in xrange(len(xr)): xrp += str(float(xr[i])/options.apix)+" "
xrp = xrp[:-1]
for i in xrange(len(txs)): txsp += str(float(txs[i])/options.apix)+" "
txsp = txsp[:-1]
# now y_restrict has the same format as x search range .... has to change ihrsr accordingly
for i in xrange(len(y_restrict)): y_restrict2 += str(float(y_restrict[i])/options.apix)+" "
y_restrict2 = y_restrict2[:-1]
from mpi import mpi_init, mpi_finalize
sys.argv = mpi_init(len(sys.argv), sys.argv)
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from applications import localhelicon_MPI, localhelicon_MPInew, localhelicon_MPIming
if len(args) < 4: mask = None
else: mask = args[3]
if options.new: localhelicon_MPInew(args[0], args[1], args[2], options.seg_ny, mask, irp, oup, options.rs, xrp, options.ynumber, \
txsp, options.delta, options.initial_theta, options.delta_theta, options.an, options.maxit, options.CTF, options.snr, \
options.dp, options.dphi, options.psi_max, \
rminp, rmaxp, options.fract, options.npad,options.sym, options.function,\
options.apix, options.debug, y_restrict2, options.searchit, options.slowIO)
elif options.snake: localhelicon_MPIming(args[0], args[1], args[2], options.seg_ny, mask, irp, oup, options.rs, xrp, options.ynumber, \
txsp, options.delta, options.initial_theta, options.delta_theta, options.an, options.maxit, options.CTF, options.snr, \
options.dp, options.dphi, options.psi_max, \
rminp, rmaxp, options.fract, options.npad,options.sym, options.function,\
options.apix, options.debug, y_restrict2, options.searchit, options.snakeknots, options.slowIO)
else: localhelicon_MPI(args[0], args[1], args[2], options.seg_ny, mask, irp, oup, options.rs, xrp, options.ynumber, \
txsp, options.delta, options.initial_theta, options.delta_theta, options.an, options.maxit, options.CTF, options.snr, \
options.dp, options.dphi, options.psi_max, \
rminp, rmaxp, options.fract, options.npad,options.sym, options.function,\
options.apix, options.debug, y_restrict2, options.searchit, options.slowIO)
global_def.BATCH = False
from mpi import mpi_finalize
mpi_finalize()
def main():
# Parse the Options
progname = path.basename(argv[0])
usage = progname + """ summovie_path input_micrograph_pattern input_shift_pattern output_directory
--selection_list
--nr_frames=nr_frames
--first
--last
--pixel_size=pixel_size
--nr_threads
--apply_dose_filter
--voltage=voltage
--exposure_per_frame=exposure_per_frame
--pre_exposure=pre_exposure
--dont_restore_noise
sxsummovie exists only in non-MPI version.
Perform summovie without dose filtering.
sxsummovie.py ~/my_app/summovie 'outdir_unblur/corrsum/micrograph_*_frames_sum.mrc' 'outdir_unblur/shift/micrograph_*_frames_shift.txt'
outdir_summovie --nr_frames=24 --pixel_size=1.19 --nr_threads=1
Perform summovie without dose filtering and with less frames.
sxsummovie.py ~/my_app/summovie 'outdir_unblur/corrsum/micrograph_*_frames_sum.mrc' 'outdir_unblur/shift/micrograph_*_frames_shift.txt'
outdir_summovie --nr_frames=24 --first=3 --last=15 --pixel_size=1.19 --nr_threads=1
Perform summovie with dose filtering and with less frames.
sxsummovie.py ~/my_app/summovie 'outdir_unblur/corrsum/micrograph_*_frames_sum.mrc' 'outdir_unblur/shift/micrograph_*_frames_shift.txt'
outdir_summovie --nr_frames=24 --first=3 --last=15 --pixel_size=1.19 --nr_threads=1 --apply_dose_filter --voltage=300 --exposure_per_frame=2 --pre_exposure=0
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
'--selection_list',
type='str',
default='',
help=
'Micrograph selecting list (SPHIRE specific): Specify a name of micrograph selection list text file. The file extension must be \'.txt\'. If this is not provided, all files matched with the micrograph name pattern will be processed. (default none)'
)
parser.add_option(
'--nr_frames',
type='int',
default=3,
help=
'Number of movie frames: The number of movie frames in each input micrograph. (default 3)'
)
parser.add_option(
'--first',
type='int',
default=1,
help='First movie frame: First movie frame for summing. (default 1)')
parser.add_option(
'--last',
type='int',
default=-1,
help='Last movie frame: Last movie frame for summing. (default -1)')
parser.add_option('--sum_suffix',
type='str',
default='_sum',
help=SUPPRESS_HELP)
parser.add_option(
'--pixel_size',
type='float',
default=-1.0,
help=
'Pixel size [A]: The pixel size of input micrographs. (default required float)'
)
parser.add_option(
'--nr_threads',
type='int',
default=1,
help=
'Number of threads: The number of threads summovie can use. The higher the faster, but it requires larger memory. (default 1)'
)
parser.add_option(
'--apply_dose_filter',
action='store_true',
default=False,
help=
'Apply dose filter step: Requires voltage, exposure per frame, and pre exposure options. (default False)'
)
parser.add_option(
'--voltage',
type='float',
default=300.0,
help=
'Microscope voltage (dose filter) [kV]: The acceleration voltage of microscope used for imaging. (default 300.0)'
)
parser.add_option(
'--exposure_per_frame',
type='float',
default=2.0,
help=
'Per frame exposure (dose filter) [e/A^2]: The electron dose per frame in e/A^2. (default 2.0)'
)
parser.add_option(
'--pre_exposure',
type='float',
default=0.0,
help=
'Pre-exposure (dose filter) [e/A^2]: The electron does in e/A^2 used for exposure prior to imaging .(default 0.0)'
)
parser.add_option('--frc_suffix',
type='string',
default='_frc',
help=SUPPRESS_HELP)
parser.add_option(
'--dont_restore_noise',
action='store_true',
default=False,
help=
'Do not restore noise power: Do not restore noise power. (default False)'
)
parser.add_option('--summovie_ready',
action='store_true',
default=False,
help=SUPPRESS_HELP)
# list of the options and the arguments
(options, args) = parser.parse_args(argv[1:])
global_def.BATCH = True
# If there arent enough arguments, stop the script
if len(args) != 4:
ERROR("see usage " + usage, 1)
# Convert the realtive parts to absolute ones
summovie_path = path.realpath(args[0]) # summovie_path
input_image = path.realpath(args[1]) # input_micrograph_pattern
input_shift = path.realpath(args[2]) # input_shift_pattern
output_dir = path.realpath(args[3]) # output_directory
# If the summovie executable file does not exists, stop the script
if not path.exists(summovie_path):
ERROR(
'Summovie directory does not exist, please change' +
' the name and restart the program.', 'sxsummovie.py', 1)
# If the output directory exists, stop the script
if path.exists(output_dir):
ERROR(
'Output directory exists, please change' +
' the name and restart the program.', 'sxsummovie.py', 1)
# If the input file does not exists, stop the script
file_list = glob(input_image)
shift_list = glob(input_shift)
if not file_list:
ERROR(
'Input micrograph file(s) does not exist, please change' +
' the name and restart the program.', 'sxsummovie.py', 1)
if not shift_list:
ERROR(
'Input shift file(s) does not exist, please change' +
' the name and restart the program.', 'sxsummovie.py', 1)
# Output paths
if options.apply_dose_filter:
output_path = '{:s}/corrsum_dose_filtered'.format(output_dir)
else:
output_path = '{:s}/corrsum'.format(output_dir)
frc_path = '{:s}/frc'.format(output_dir)
log_path = '{:s}/logfiles'.format(output_dir)
temp_path = '{0}/temp'.format(output_dir)
# Split the path of the image name at the "/" Characters.
# The last entry contains the micrograph name.
# Split the micrograph name at the wildcard character for the
# prefix and suffix.
input_mic_split = input_image.split('/')
input_mic_name = input_mic_split[-1].split('*')
input_shift_name = input_shift.split('*')
if len(input_mic_name) != 2 or len(input_shift_name) != 2:
ERROR(
'Too many wildcard arguments.' +
'Please use exactly one * in the pattern.', 'sxsummovie.py', 1)
# Get the input directory
if len(input_mic_split) != 1:
input_dir = input_image[:-len(input_mic_split[-1])]
else:
input_dir = ''
# Create output directorys
if not path.exists(output_dir):
mkdir(output_dir)
if not path.exists(output_path):
mkdir(output_path)
if not path.exists(frc_path):
mkdir(frc_path)
if not path.exists(temp_path) and not options.summovie_ready:
mkdir(temp_path)
if not path.exists(log_path):
mkdir(log_path)
# shift wildcard list
shift_wildcard = [entry[len(input_shift_name[0]):-len(input_shift_name[-1])] \
for entry in shift_list]
# Just use shifts that have a micrograph and vise versa
mic_list = [
entry for entry in file_list \
if entry[len(input_dir) + len(input_mic_name[0]):-len(input_mic_name[-1])]\
in shift_wildcard]
# If micrograph list is provided just process the images in the list
selection_file = options.selection_list
if selection_file:
# Import list file
try:
selection = genfromtxt(selection_file, dtype=None)
except TypeError:
ERROR(
'no entrys in micrograph list file {0}'.format(selection_file),
'sxsummovie.py', 1)
# List of files which are in pattern and list
mic_list = [
entry for entry in mic_list \
if entry[len(input_dir):] in selection and \
path.exists(entry)
]
# If no match is there abort
if len(mic_list) == 0:
ERROR(
'no files in {0} matched the micrograph file pattern:\n'.
format(selection_file), 'sxsummovie.py', 1)
option_dict = {
'summovie_path': summovie_path,
'mic_list': mic_list,
'mic_prefix': input_mic_name[0],
'mic_suffix': input_mic_name[1],
'shift_prefix': input_shift_name[0],
'shift_suffix': input_shift_name[1],
'input_dir': input_dir,
'output_dir': output_dir,
'output_path': output_path,
'frc_path': frc_path,
'log_path': log_path,
'temp_path': temp_path,
'nr_frames': options.nr_frames,
'sum_suffix': options.sum_suffix,
'first': options.first,
'last': options.last,
'pixel_size': options.pixel_size,
'apply_dose_filter': options.apply_dose_filter,
'exposure_per_frame': options.exposure_per_frame,
'voltage': options.voltage,
'pre_exposure': options.pre_exposure,
'frc_suffix': options.frc_suffix,
'dont_restore_noise': options.dont_restore_noise,
'nr_threads': options.nr_threads,
'summovie_ready': options.summovie_ready,
'verbose': True
}
# Run summovie
run_summovie(opt=option_dict)
if not options.summovie_ready:
# Remove temp folder
for entry in glob('{0}/*'.format(temp_path)):
remove(entry)
rmdir(temp_path)
print('All Done!')
global_def.BATCH = False
def run_unblur(unblur_path, input_image, input_dir, output_dir, corrected_path,
uncorrected_path, shift_path, frc_path, temp_path, log_path,
file_list, options):
# Lists to write the text files later
micrograph_list = []
shift_list = []
if options.save_frames:
frames_list = []
# If micrograph list is provided just process the images in the list
mic_list = options.selection_list
if mic_list:
# Import list file
try:
set_selection = genfromtxt(mic_list, dtype=None)
except TypeError:
ERROR('no entrys in list file {0}'.format(mic_list), 'sxunblur.py',
1)
# List of files which are in pattern and list
file_list = [
entry for entry in file_list \
if entry[len(input_dir):] in set_selection and \
path.exists(entry)
]
# If no match is there abort
if len(file_list) == 0:
ERROR(
'no files in {0} matched the file pattern:\n'.format(mic_list),
'sxunblur.py', 1)
# Get the number of files
nr_files = len(file_list)
# Timeing stuff
time_start = time.time()
time_list = []
# Loop over all files
for index, inputfile in enumerate(sorted(file_list)):
# Check, if there is an prefix and suffix.
# If there is more then one entry: the suffix is the last one.
# Otherwhise its just the one after the dot.
input_suffix = inputfile.split('/')[-1].split('.')[-1]
# First output to introduce the programm
if index == 0:
print(
'Progress: 0.0%; Time: --h:--m:--s/--h:--m:--s; Unblur started!'
)
# Time begin
t1 = time.time()
# Get the output names
file_name = inputfile[len(input_dir):-len(input_suffix) - 1]
if options.skip_dose_filter:
micrograph_name = '{0}/{1}{2}.mrc'.format(uncorrected_path,
file_name,
options.sum_suffix)
frames_name = '{0}/{1}{2}.mrc'.format(uncorrected_path, file_name,
options.frames_suffix)
frc_name = '{0}/{1}{2}.txt'.format(frc_path, file_name,
options.frc_suffix)
else:
micrograph_name = '{0}/{1}{2}.mrc'.format(corrected_path,
file_name,
options.sum_suffix)
frames_name = '{0}/{1}{2}.mrc'.format(corrected_path, file_name,
options.frames_suffix)
micrograph_name_skip = '{0}/{1}{2}.mrc'.format(
uncorrected_path, file_name, options.sum_suffix)
frames_name_skip = '{0}/{1}{2}.mrc'.format(uncorrected_path,
file_name,
options.frames_suffix)
frc_name = '{0}/{1}{2}.txt'.format(frc_path, file_name,
options.frc_suffix)
frc_summovie_name = '{0}/{1}_summovie{2}.txt'.format(
frc_path, file_name, options.frc_suffix)
shift_name = '{0}/{1}{2}.txt'.format(shift_path, file_name,
options.shift_suffix)
if not options.unblur_ready:
temp_name = '{0}/{1}{2}.mrc'.format(temp_path, file_name,
options.sum_suffix)
else:
temp_name = inputfile
log_name = '{0}/{1}.log'.format(log_path, file_name)
error_name = '{0}/{1}.err'.format(log_path, file_name)
# Append the names to the lists
micrograph_list.append('{0}{1}.mrc'.format(file_name,
options.sum_suffix))
shift_list.append(shift_name)
if options.save_frames:
frames_list.append('{0}{1}.mrc'.format(file_name,
options.frames_suffix))
# First build the unblur/summovie command
if not options.skip_dose_filter:
unblur_command = create_unblur_command(temp_name, micrograph_name,
shift_name, frames_name,
frc_name, options)
# Options for the summovie command
options_summovie = {
'first': 1,
'last': -1,
'nr_frames': options.nr_frames,
'pixel_size': options.pixel_size,
'exposure_per_frame': options.exposure_per_frame,
'voltage': options.voltage,
'pre_exposure': options.pre_exposure,
'dont_restore_noise': options.dont_restore_noise,
'apply_dose_filter': False
}
summovie_command = create_summovie_command(temp_name,
micrograph_name_skip,
shift_name,
frc_summovie_name,
options_summovie)
else:
unblur_command = create_unblur_command(temp_name, micrograph_name,
shift_name, frc_name,
frames_name, options)
# Export the number of threads
export_threads_command = []
# Export
export_threads_command.append('export')
# Nr of threads
export_threads_command.append('OMP_NUM_THREADS={0}'.format(
options.nr_threads))
if not options.unblur_ready:
# Do a e2proc3d.py
e2proc3d_command = []
# e2proc3d
e2proc3d_command.append('e2proc3d.py')
# inputfile
e2proc3d_command.append('{0}'.format(inputfile))
# outputfile
e2proc3d_command.append('{0}'.format(temp_name))
# Translate the command to single strings
if not options.unblur_ready:
e2proc3d_command = r' '.join(e2proc3d_command)
export_threads_command = r' '.join(export_threads_command)
unblur_command = '\n'.join(unblur_command)
if not options.skip_dose_filter:
summovie_command = '\n'.join(summovie_command)
# Build full command
if not options.unblur_ready:
if not options.skip_dose_filter:
full_command = r'{0}; {1}; echo "{2}" | {3}'.format(
export_threads_command, e2proc3d_command, unblur_command,
unblur_path)
full_command_summovie = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, summovie_command,
options.summovie_path)
else:
full_command = r'{0}; {1}; echo "{2}" | {3}'.format(
export_threads_command, e2proc3d_command, unblur_command,
unblur_path)
else:
if not options.skip_dose_filter:
full_command = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, unblur_command, unblur_path)
full_command_summovie = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, summovie_command,
options.summovie_path)
else:
full_command = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, unblur_command, unblur_path)
# Remove temp unblur files
temp_unblur_files = glob('.UnBlur*')
for entry in temp_unblur_files:
remove(entry)
# Remove temp summovie files
temp_summovie_files = glob('.SumMovie*')
for entry in temp_summovie_files:
remove(entry)
with open(log_name, 'w') as f:
with open(error_name, 'w') as e:
# Execute Command
if not options.skip_dose_filter:
subprocess.Popen([full_command],
shell=True,
stdout=f,
stderr=e).wait()
# Remove temp unblur files
temp_unblur_files = glob('.UnBlur*')
for entry in temp_unblur_files:
remove(entry)
# Remove temp summovie files
temp_summovie_files = glob('.SumMovie*')
for entry in temp_summovie_files:
remove(entry)
subprocess.Popen([full_command_summovie],
shell=True,
stdout=f,
stderr=e).wait()
else:
subprocess.Popen([full_command],
shell=True,
stdout=f,
stderr=e).wait()
# Remove temp unblur files
temp_unblur_files = glob('.UnBlur*')
for entry in temp_unblur_files:
remove(entry)
# Remove temp summovie files
temp_summovie_files = glob('.SumMovie*')
for entry in temp_summovie_files:
remove(entry)
if not options.unblur_ready:
if path.exists(temp_name):
# Remove temp file
remove(temp_name)
else:
print(('Error with file:\n{0}'.format(inputfile)))
# Check if SumMovie and UnBlur finished cleanly
with open(log_name, 'r') as r:
clean_summovie = False
clean_unblur = False
for line in r:
if 'SumMovie finished cleanly.' in line:
clean_summovie = True
if 'UnBlur finished cleanly.' in line:
clean_unblur = True
if clean_unblur:
print('UnBlur finished cleanly.')
else:
ERROR(
'unblur error. check the logfile for more information: {0}'.
format(log_name), 'sxunblur.py', 0)
if clean_summovie:
print('SumMovie finished cleanly.')
else:
ERROR(
'summovie error. check the logfile for more information: {0}'.
format(log_name), 'sxunblur.py', 0)
time_list.append(time.time() - t1)
# Do progress output
percent = round(100 * (index + 1) / float(nr_files), 2)
estimated_time = \
nr_files * sum(time_list) / float(len(time_list))
estimated_time_h = estimated_time // 3600
estimated_time_m = (estimated_time - estimated_time_h * 3600) // 60
estimated_time_s = (estimated_time - estimated_time_h * 3600 -
estimated_time_m * 60)
current_time = time.time() - time_start
current_time_h = current_time // 3600
current_time_m = (current_time - current_time_h * 3600) // 60
current_time_s = (current_time - current_time_h * 3600 -
current_time_m * 60)
print((
'Progress: {0:.2f}%; Time: {1:.0f}h:{2:.0f}m:{3:.0f}s/{4:.0f}h:{5:.0f}m:{6:.0f}s; Micrograph done:{7}'
.format(percent, current_time_h, current_time_m, current_time_s,
estimated_time_h, estimated_time_m, estimated_time_s,
file_name)))
# Write micrograph and shift list
with open('{0}/unblur_micrographs.txt'.format(output_dir), 'w') as f:
for entry in sorted(micrograph_list):
f.write('{0}\n'.format(entry))
with open('{0}/unblur_shiftfiles.txt'.format(output_dir), 'w') as f:
for entry in sorted(shift_list):
f.write('{0}\n'.format(entry))
if options.save_frames:
with open('{0}/unblur_frames.txt'.format(output_dir), 'w') as f:
for entry in sorted(frames_list):
f.write('{0}\n'.format(entry))
def run_summovie(opt):
# Lists to write the text files later
micrograph_list = []
# Get the number of files
nr_files = len(opt['mic_list'])
# Timeing stuff
time_start = time.time()
time_list = []
# Loop over all files
for index, inputfile in enumerate(sorted(opt['mic_list'])):
# Check, if there is an prefix and suffix.
# If there is more then one entry: the suffix is the last one.
# Otherwhise its just the one after the dot.
input_suffix = inputfile.split('/')[-1].split('.')[-1]
# First output to introduce the programm
if opt['verbose'] and index == 0:
print(
'Progress: 0.0%; Time: --h:--m:--s/--h:--m:--s; Summovie started!'
)
# Time begin
t1 = time.time()
# Get the output names
file_name = inputfile[len(opt['input_dir']):-len(opt['mic_suffix'])]
file_wildcard = file_name[len(opt['mic_prefix']):]
micrograph_name = '{0}/{1}{2}.mrc'.format(opt['output_path'],
file_name, opt['sum_suffix'])
frc_name = '{0}/{1}{2}.txt'.format(opt['frc_path'], file_name,
opt['frc_suffix'])
shift_name = '{0}{1}{2}'.format(opt['shift_prefix'], file_wildcard,
opt['shift_suffix'])
if not opt['summovie_ready']:
temp_name = '{0}/{1}{2}.mrc'.format(opt['temp_path'], file_name,
opt['sum_suffix'])
else:
temp_name = inputfile
log_name = '{0}/{1}.log'.format(opt['log_path'], file_name)
error_name = '{0}/{1}.err'.format(opt['log_path'], file_name)
# Append the names to the lists
micrograph_list.append('{0}{1}.mrc'.format(file_name,
opt['sum_suffix']))
# First build the summovie command
summovie_command = create_summovie_command(temp_name, micrograph_name,
shift_name, frc_name, opt)
# Export the number of threads
export_threads_command = []
# Export
export_threads_command.append('export')
# Nr of threads
export_threads_command.append('OMP_NUM_THREADS={0}'.format(
opt['nr_threads']))
if not opt['summovie_ready']:
# Do a e2proc3d.py
e2proc3d_command = []
# e2proc3d
e2proc3d_command.append('e2proc3d.py')
# inputfile
e2proc3d_command.append('{0}'.format(inputfile))
# outputfile
e2proc3d_command.append('{0}'.format(temp_name))
# Translate the command to single strings
if not opt['summovie_ready']:
e2proc3d_command = r' '.join(e2proc3d_command)
export_threads_command = r' '.join(export_threads_command)
summovie_command = '\n'.join(summovie_command)
# Build full command
if not opt['summovie_ready']:
full_command = r'{0}; {1}; echo "{2}" | {3}'.format(
export_threads_command, e2proc3d_command, summovie_command,
opt['summovie_path'])
else:
full_command = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, summovie_command, opt['summovie_path'])
# Remove temp summovie files
temp_summovie_files = glob('.SumMovie*')
for entry in temp_summovie_files:
remove(entry)
with open(log_name, 'w') as f:
with open(error_name, 'w') as e:
# Execute Command
subprocess.Popen([full_command],
shell=True,
stdout=f,
stderr=e).wait()
# Remove temp summovie files
temp_summovie_files = glob('.SumMovie*')
for entry in temp_summovie_files:
remove(entry)
if not opt['summovie_ready']:
if path.exists(temp_name):
# Remove temp file
remove(temp_name)
else:
ERROR(
'e2proc2d.py error. File was not created:\n{0}'.format(
inputfile), 'sxsummovie.py', 0)
time_list.append(time.time() - t1)
# Check if SumMovie finished cleanly
with open(log_name, 'r') as r:
clean = False
for line in r:
if 'SumMovie finished cleanly.' in line:
clean = True
break
if clean:
print('SumMovie finished cleanly.')
else:
ERROR(
'sum movie error. check the logfile for more information: {0}'.
format(log_name), 'sxsummovie.py', 0)
# Do progress output
if opt['verbose']:
percent = round(100 * (index + 1) / float(nr_files), 2)
estimated_time = \
nr_files * sum(time_list) / float(len(time_list))
estimated_time_h = estimated_time // 3600
estimated_time_m = (estimated_time - estimated_time_h * 3600) // 60
estimated_time_s = (estimated_time - estimated_time_h * 3600 -
estimated_time_m * 60)
current_time = time.time() - time_start
current_time_h = current_time // 3600
current_time_m = (current_time - current_time_h * 3600) // 60
current_time_s = (current_time - current_time_h * 3600 -
current_time_m * 60)
print((
'Progress: {0:.2f}%; Time: {1:.0f}h:{2:.0f}m:{3:.0f}s/{4:.0f}h:{5:.0f}m:{6:.0f}s; Micrograph done:{7}'
.format(percent, current_time_h, current_time_m,
current_time_s, estimated_time_h, estimated_time_m,
estimated_time_s, file_name)))
# Write micrograph list
with open('{0}/summovie_micrographs.txt'.format(opt['output_dir']),
'w') as f:
for entry in sorted(micrograph_list):
f.write('{0}\n'.format(entry))
def main():
# Parse the Options
progname = path.basename(argv[0])
usage = progname + """ unblur input_image output
--nr_frames=nr_frames
--pixel_size=pixel_size
--dose_filter
--exposure_per_frame=exposure_per_frame
--voltage=voltage
--pre_exposure=pre_exposure
--save_frames --expert_mode
--shift_initial=shift_initial
--shift_radius=shift_radius
--b_factor=b_factor
--fourier_vertical=fourier_vertical
--fourier_horizontal=fourier_horizontal
--shift_threshold=shift_threshold
--iterations=iterations
--restore_noise
--verbose
--filter_sum
--lowpass=lowpass
--highpass=highpass
--remove_sum
--nr_threads'
sxunblur exists in non-MPI version.
Just shift data.
sxunblur.py directory_to_unblur directory/prefix*suffix.mrc output_directory
--nr_frames=25 --pixel_size=1.19 --remove_sum
Shift data with aligned sum files, filtered sum files and aligned frames.
sxunblur.py directory_to_unblur directory/prefix*suffix.mrc output_directory
--nr_frames=25 --pixel_size=1.19 --save_frames --filter_sum
--lowpass=0.033 --highpass=0.00033 --nr_threads=2
Dose filter and Expert Options
sxunblur.py directory_to_unblur directory/prefix*suffix.mrc output_directory
--nr_frames=25 --pixel_size=1.19 --dose_filter --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --save_frames --expert_mode
--shift_initial=2.0 --shift_radius=200.0 --b_factor=1500.0
--fourier_vertical=1 --fourier_horizontal=1 --shift_threshold=0.1
--iterations=10 --restore_noise --verbose --filter_sum --lowpass=0.033
--highpass=0.00033 --nr_threads=2
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option('--nr_frames',
type='int',
default=3,
help='number of frames in the set of micrographs')
parser.add_option('--sum_suffix',
type='str',
default='_sum',
help=SUPPRESS_HELP)
parser.add_option('--shift_suffix',
type='str',
default='_shift',
help=SUPPRESS_HELP)
parser.add_option('--pixel_size',
type='float',
default=-1.0,
help='pixel size [A]')
parser.add_option('--dose_filter',
action='store_true',
default=False,
help='apply dose filter options')
parser.add_option('--exposure_per_frame',
type='float',
default=2.0,
help='exposure per frame [e/A^2]')
parser.add_option('--voltage',
type='float',
default=300.0,
help='accelerate voltage [kV]')
parser.add_option('--pre_exposure',
type='float',
default=0.0,
help='pre exposure amount [e/A^2]')
parser.add_option('--save_frames',
action='store_true',
default=False,
help='save aligned frames')
parser.add_option('--frames_suffix',
type='string',
default='_frames',
help=SUPPRESS_HELP)
parser.add_option('--expert_mode',
action='store_true',
default=False,
help='set expert mode settings')
parser.add_option('--frc_suffix',
type='string',
default='_frc',
help=SUPPRESS_HELP)
parser.add_option('--shift_initial',
type='float',
default=2.0,
help='minimum shift for inital search [A]')
parser.add_option('--shift_radius',
type='float',
default=200.0,
help='outer radius shift limit [A]')
parser.add_option('--b_factor',
type='float',
default=1500.0,
help='b-factor to appy to image [A^2]')
parser.add_option(
'--fourier_vertical',
type='int',
default=1,
help='half-width of central vertical line of fourier mask')
parser.add_option(
'--fourier_horizontal',
type='int',
default=1,
help='half-width of central horizontal line of fourier mask')
parser.add_option('--shift_threshold',
type='float',
default=0.1,
help='termination shift threshold')
parser.add_option('--iterations',
type='int',
default=10,
help='maximum number of iterations')
parser.add_option('--restore_noise',
action='store_true',
default=False,
help='restore noise power')
parser.add_option('--verbose',
action='store_true',
default=False,
help='verbose output')
parser.add_option('--filter_sum',
action='store_true',
default=False,
help='filter the output images')
parser.add_option('--lowpass',
type='float',
default=0.033,
help='apply a lowpass filter: abolute frequency')
parser.add_option('--highpass',
type='float',
default=0.00033,
help='apply a highpass filter: abolute frequency')
parser.add_option('--remove_sum',
action='store_true',
default=False,
help='remove the calculated sum files')
parser.add_option('--nr_threads',
type='int',
default=2,
help='Number of threads')
# list of the options and the arguments
(options, args) = parser.parse_args(argv[1:])
global_def.BATCH = True
# If there arent enough arguments, stop the script
if len(args) != 3:
ERROR("see usage " + usage, 1)
# Convert the realtive parts to absolute ones
unblur_path = path.realpath(args[0])
input_image = path.realpath(args[1])
output_dir = path.realpath(args[2])
# If the unblur executable file does not exists, stop the script
if not path.exists(unblur_path):
ERROR(
'Unblur directory does not exist, please change' +
' the name and restart the program.', 1)
# If the output directory exists, stop the script
if path.exists(output_dir):
ERROR(
'Output directory exists, please change' +
' the name and restart the program.', 1)
# If the input file does not exists, stop the script
fileList = glob(input_image)
if not fileList:
ERROR(
'Input file does not exist, please change' +
' the name and restart the program.', 1)
# Split the path of the image name at the "/" Characters.
# The last entry contains the micrograph name.
# Split the micrograph name at the wildcard character for the
# prefix and suffix.
input_split = input_image.split('/')
input_name = input_split[-1].split('*')
# Check, if there is an prefix and suffix.
if len(input_name) == 2:
input_suffix = input_name[1]
else:
if '*' in input_split[-1]:
input_suffix = input_name[0]
else:
input_suffix = '.mrc'
if len(input_split) != 1:
input_dir = input_image[:-len(input_split[-1])]
else:
input_dir = ''
# Create output directorys
if not path.exists('{:s}'.format(output_dir)):
mkdir('{:s}'.format(output_dir))
if not path.exists('{:s}/Doseuncorrected'.format(output_dir)):
mkdir('{:s}/Doseuncorrected'.format(output_dir))
if not path.exists('{:s}/Shift'.format(output_dir)):
mkdir('{:s}/Shift'.format(output_dir))
if not path.exists('{:s}/Filtered'.format(output_dir)) \
and options.filter_sum:
mkdir('{:s}/Filtered'.format(output_dir))
if not path.exists('{:s}/Dosecorrected'.format(output_dir)) \
and options.dose_filter:
mkdir('{:s}/Dosecorrected'.format(output_dir))
if not path.exists('{:s}/FRC'.format(output_dir)) \
and options.expert_mode:
mkdir('{:s}/FRC'.format(output_dir))
# Create sh script
create_sh_script(unblur_path=unblur_path,
input_image=input_image,
input_dir=input_dir,
output_dir=output_dir,
input_suffix=input_suffix,
options=options)
# Start sh script
system('sh {:s}/scriptUnblur.sh'.format(output_dir))
global_def.BATCH = False
def main():
import sys
import os
import math
import random
import pyemtbx.options
import time
from random import random, seed, randint
from optparse import OptionParser
from global_def import ERROR
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Forms chains of 2D images based on their similarities.
Functionality:
Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent).
Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers.
The ways to use the program:
1. Use option initial to specify which image will be used as an initial seed to form the chain.
sxchains.py input_stack.hdf output_stack.hdf --initial=23 --radius=25
2. If options initial is omitted, the program will determine which image best serves as initial seed to form the chain
sxchains.py input_stack.hdf output_stack.hdf --radius=25
3. Use option circular to form a circular chain.
sxchains.py input_stack.hdf output_stack.hdf --circular--radius=25
4. New circular code based on pairwise alignments
sxchains.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt
5. Circular ordering based on pairwise alignments
sxchains.py vols.hdf chain.hdf mask.hdf --dd --radius=25
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--dd",
action="store_true",
help="Circular ordering without adjustment of orientations",
default=False)
parser.add_option(
"--circular",
action="store_true",
help=
"Select circular ordering (first image has to be similar to the last)",
default=False)
parser.add_option(
"--align",
action="store_true",
help=
"Compute all pairwise alignments and from the table of image similarities find the best chain",
default=False)
parser.add_option(
"--initial",
type="int",
default=-1,
help=
"Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)"
)
parser.add_option(
"--radius",
type="int",
default=-1,
help="Radius of a circular mask for similarity based ordering")
# import params for 2D alignment
parser.add_option(
"--ou",
type="int",
default=-1,
help=
"outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)"
)
parser.add_option(
"--xr",
type="int",
default=0,
help="range for translation search in x direction, search is +/xr (0)")
parser.add_option(
"--yr",
type="int",
default=0,
help="range for translation search in y direction, search is +/yr (0)")
#parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)")
parser.add_option("--pairwiseccc",
type="string",
default=" ",
help="Input/output pairwise ccc file")
(options, args) = parser.parse_args()
global_def.BATCH = True
if options.dd:
nargs = len(args)
if nargs != 3:
print("must provide name of input and two output files!")
return
stack = args[0]
new_stack = args[1]
from utilities import model_circle
from statistics import ccc
from statistics import mono
lend = EMUtil.get_image_count(stack)
lccc = [None] * (lend * (lend - 1) / 2)
for i in range(lend - 1):
v1 = get_im(stack, i)
if (i == 0 and nargs == 2):
nx = v1.get_xsize()
ny = v1.get_ysize()
nz = v1.get_ysize()
if options.ou < 1: radius = nx // 2 - 2
else: radius = options.ou
mask = model_circle(radius, nx, ny, nz)
else:
mask = get_im(args[2])
for j in range(i + 1, lend):
lccc[mono(i, j)] = [ccc(v1, get_im(stack, j), mask), 0]
order = tsp(lccc)
if (len(order) != lend):
print(" problem with data length")
from sys import exit
exit()
print("Total sum of cccs :", TotalDistance(order, lccc))
print("ordering :", order)
for i in range(lend):
get_im(stack, order[i]).write_image(new_stack, i)
elif options.align:
nargs = len(args)
if nargs != 3:
print("must provide name of input and two output files!")
return
from utilities import get_params2D, model_circle
from fundamentals import rot_shift2D
from statistics import ccc
from time import time
from alignment import align2d, align2d_scf
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
if (len(d) < 6):
ERROR(
"Chains requires at least six images in the input stack to be executed",
"sxchains.py", 1)
"""
# will align anyway
try:
ttt = d[0].get_attr('xform.params2d')
for i in xrange(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
"""
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.ou < 1: radius = nx // 2 - 2
else: radius = options.ou
mask = model_circle(radius, nx, ny)
if (options.xr < 0): xrng = 0
else: xrng = options.xr
if (options.yr < 0): yrng = xrng
else: yrng = options.yr
initial = max(options.initial, 0)
from statistics import mono
lend = len(d)
lccc = [None] * (lend * (lend - 1) / 2)
from utilities import read_text_row
if options.pairwiseccc == " " or not os.path.exists(
options.pairwiseccc):
st = time()
for i in range(lend - 1):
for j in range(i + 1, lend):
# j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger)
#alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F")
alpha, sx, sy, mir, peak = align2d_scf(d[i],
d[j],
xrng,
yrng,
ou=radius)
lccc[mono(i, j)] = [
ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0),
mask), alpha, sx, sy, mir
]
#print " %4d %10.1f"%(i,time()-st)
if ((not os.path.exists(options.pairwiseccc))
and (options.pairwiseccc != " ")):
from utilities import write_text_row
write_text_row([[initial, 0, 0, 0, 0]] + lccc,
options.pairwiseccc)
elif (os.path.exists(options.pairwiseccc)):
lccc = read_text_row(options.pairwiseccc)
initial = int(lccc[0][0] + 0.1)
del lccc[0]
for i in range(len(lccc)):
T = Transform({
"type": "2D",
"alpha": lccc[i][1],
"tx": lccc[i][2],
"ty": lccc[i][3],
"mirror": int(lccc[i][4] + 0.1)
})
lccc[i] = [lccc[i][0], T]
tdummy = Transform({"type": "2D"})
maxsum = -1.023
for m in range(0, lend): #initial, initial+1):
indc = list(range(lend))
lsnake = [[m, tdummy, 0.0]]
del indc[m]
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in range(len(indc)):
cuc = lccc[mono(indc[i], lsnake[-1][0])][0]
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
# Here we need transformation from the current to the previous,
# meaning indc[i] -> lsnake[-1][0]
T = lccc[mono(indc[i], lsnake[-1][0])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if (indc[i] > lsnake[-1][0]): T = T.inverse()
lsnake.append([qi, T, maxcit])
lsum += maxcit
del indc[indc.index(qi)]
T = lccc[mono(indc[-1], lsnake[-1][0])][1]
if (indc[-1] > lsnake[-1][0]): T = T.inverse()
lsnake.append(
[indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]])
print(" initial image and lsum ", m, lsum)
#print lsnake
if (lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in range(lend)]
print(" Initial image selected : ", init, maxsum, " ",
TotalDistance([snake[m][0] for m in range(lend)], lccc))
#for q in snake: print q
from copy import deepcopy
trans = deepcopy([snake[i][1] for i in range(len(snake))])
print([snake[i][0] for i in range(len(snake))])
"""
for m in xrange(lend):
prms = trans[m].get_params("2D")
print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
"""
for k in range(lend - 2, 0, -1):
T = snake[k][1]
for i in range(k + 1, lend):
trans[i] = T * trans[i]
# To add - apply all transformations and do the overall centering.
for m in range(lend):
prms = trans[m].get_params("2D")
#print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
#rot_shift2D(d[snake[m][0]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image(new_stack, m)
rot_shift2D(d[snake[m][0]], prms["alpha"], 0.0, 0.0,
prms["mirror"]).write_image(new_stack, m)
order = tsp(lccc)
if (len(order) != lend):
print(" problem with data length")
from sys import exit
exit()
print(TotalDistance(order, lccc))
print(order)
ibeg = order.index(init)
order = [order[(i + ibeg) % lend] for i in range(lend)]
print(TotalDistance(order, lccc))
print(order)
snake = [tdummy]
for i in range(1, lend):
# Here we need transformation from the current to the previous,
# meaning order[i] -> order[i-1]]
T = lccc[mono(order[i], order[i - 1])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if (order[i] > order[i - 1]): T = T.inverse()
snake.append(T)
assert (len(snake) == lend)
from copy import deepcopy
trans = deepcopy(snake)
for k in range(lend - 2, 0, -1):
T = snake[k]
for i in range(k + 1, lend):
trans[i] = T * trans[i]
# Try to smooth the angles - complicated, I am afraid one would have to use angles forward and backwards
# and find their average??
# In addition, one would have to recenter them
"""
trms = []
for m in xrange(lend):
prms = trans[m].get_params("2D")
trms.append([prms["alpha"], prms["mirror"]])
for i in xrange(3):
for m in xrange(lend):
mb = (m-1)%lend
me = (m+1)%lend
# angles order mb,m,me
# calculate predicted angles mb->m
"""
for m in range(lend):
prms = trans[m].get_params("2D")
#rot_shift2D(d[order[m]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image("metro.hdf", m)
rot_shift2D(d[order[m]], prms["alpha"], 0.0, 0.0,
prms["mirror"]).write_image(args[2], m)
"""
# This was an effort to get number of loops, inconclusive, to say the least
from numpy import outer, zeros, float32, sqrt
lend = len(d)
cor = zeros(lend,float32)
cor = outer(cor, cor)
for i in xrange(lend): cor[i][i] = 1.0
for i in xrange(lend-1):
for j in xrange(i+1, lend):
cor[i,j] = lccc[mono(i,j)][0]
cor[j,i] = cor[i,j]
lmbd, eigvec = pca(cor)
from utilities import write_text_file
nvec=20
print [lmbd[j] for j in xrange(nvec)]
print " G"
mm = [-1]*lend
for i in xrange(lend): # row
mi = -1.0e23
for j in xrange(nvec):
qt = eigvec[j][i]
if(abs(qt)>mi):
mi = abs(qt)
mm[i] = j
for j in xrange(nvec):
qt = eigvec[j][i]
print round(qt,3), # eigenvector
print mm[i]
print
for j in xrange(nvec):
qt = []
for i in xrange(lend):
if(mm[i] == j): qt.append(i)
if(len(qt)>0): write_text_file(qt,"loop%02d.txt"%j)
"""
"""
print [lmbd[j] for j in xrange(nvec)]
print " B"
mm = [-1]*lend
for i in xrange(lend): # row
mi = -1.0e23
for j in xrange(nvec):
qt = eigvec[j][i]/sqrt(lmbd[j])
if(abs(qt)>mi):
mi = abs(qt)
mm[i] = j
for j in xrange(nvec):
qt = eigvec[j][i]/sqrt(lmbd[j])
print round(qt,3), # eigenvector
print mm[i]
print
"""
"""
lend=3
cor = zeros(lend,float32)
cor = outer(cor, cor)
cor[0][0] =136.77
cor[0][1] = 79.15
cor[0][2] = 37.13
cor[1][0] = 79.15
cor[2][0] = 37.13
cor[1][1] = 50.04
cor[1][2] = 21.65
cor[2][1] = 21.65
cor[2][2] = 13.26
lmbd, eigvec = pca(cor)
print lmbd
print eigvec
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i], # eigenvector
print
print " B"
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i]/sqrt(lmbd[j]), # eigenvector
print
print " G"
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i]*sqrt(lmbd[j]), # eigenvector
print
"""
else:
nargs = len(args)
if nargs != 2:
print("must provide name of input and output file!")
return
from utilities import get_params2D, model_circle
from fundamentals import rot_shift2D
from statistics import ccc
from time import time
from alignment import align2d
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
try:
print("Using 2D alignment parameters from header.")
ttt = d[0].get_attr('xform.params2d')
for i in range(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.radius < 1: radius = nx // 2 - 2
else: radius = options.radius
mask = model_circle(radius, nx, ny)
init = options.initial
if init > -1:
print(" initial image: %d" % init)
temp = d[init].copy()
temp.write_image(new_stack, 0)
del d[init]
k = 1
lsum = 0.0
while len(d) > 1:
maxcit = -111.
for i in range(len(d)):
cuc = ccc(d[i], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = i
# print k, maxcit
lsum += maxcit
temp = d[qi].copy()
del d[qi]
temp.write_image(new_stack, k)
k += 1
print(lsum)
d[0].write_image(new_stack, k)
else:
if options.circular:
print("Using options.circular, no alignment")
# figure the "best circular" starting image
maxsum = -1.023
for m in range(len(d)):
indc = list(range(len(d)))
lsnake = [-1] * (len(d) + 1)
lsnake[0] = m
lsnake[-1] = m
del indc[m]
temp = d[m].copy()
lsum = 0.0
direction = +1
k = 1
while len(indc) > 1:
maxcit = -111.
for i in range(len(indc)):
cuc = ccc(d[indc[i]], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
lsnake[k] = qi
lsum += maxcit
del indc[indc.index(qi)]
direction = -direction
for i in range(1, len(d)):
if (direction > 0):
if (lsnake[i] == -1):
temp = d[lsnake[i - 1]].copy()
#print " forw ",lsnake[i-1]
k = i
break
else:
if (lsnake[len(d) - i] == -1):
temp = d[lsnake[len(d) - i + 1]].copy()
#print " back ",lsnake[len(d) - i +1]
k = len(d) - i
break
lsnake[lsnake.index(-1)] = indc[-1]
#print " initial image and lsum ",m,lsum
#print lsnake
if (lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in range(len(d))]
print(" Initial image selected : ", init, maxsum)
print(lsnake)
for m in range(len(d)):
d[snake[m]].write_image(new_stack, m)
else:
# figure the "best" starting image
print("Straight chain, no alignment")
maxsum = -1.023
for m in range(len(d)):
indc = list(range(len(d)))
lsnake = [m]
del indc[m]
temp = d[m].copy()
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in range(len(indc)):
cuc = ccc(d[indc[i]], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
lsnake.append(qi)
lsum += maxcit
temp = d[qi].copy()
del indc[indc.index(qi)]
lsnake.append(indc[-1])
#print " initial image and lsum ",m,lsum
#print lsnake
if (lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in range(len(d))]
print(" Initial image selected : ", init, maxsum)
print(lsnake)
for m in range(len(d)):
d[snake[m]].write_image(new_stack, m)
def create_summovie_command(temp_name, micrograph_name, shift_name, frc_name,
opt):
# Handle first and last case events
if opt['first'] == 0:
ERROR(
'SumMovie indexing starts with 1.\n' +
'0 is not a valid entry for --first', 'sxsummovie.py', 1)
elif opt['first'] < 0:
first = opt['nr_frames'] + opt['first'] + 1
else:
first = opt['first']
if opt['last'] == 0:
ERROR(
'SumMovie indexing starts with 1.\n' +
'0 is not a valid entry for --last', 'sxsummovie.py', 1)
elif opt['last'] < 0:
last = opt['nr_frames'] + opt['last'] + 1
else:
last = opt['last']
if first > last:
ERROR(
'First option musst be smaller equals last option!\n' +
'first: {0}; last: {1}'.format(first, last), 'sxsummovie.py', 1)
if opt['nr_frames'] < last or last <= 0:
ERROR(
'--last option {0} is out of range:\n'.format(last) +
'min: 1; max {0}'.format(opt['nr_frames']), 'sxsummovie.py', 1)
if opt['nr_frames'] < first or first <= 0:
ERROR(
'--first option {0} is out of range:\n'.format(first) +
'min: 1; max {0}'.format(opt['nr_frames']), 'sxsummovie.py', 1)
# Command list
summovie_command = []
# Input file
summovie_command.append('{0}'.format(temp_name))
# Number of frames
summovie_command.append('{0}'.format(opt['nr_frames']))
# Sum file
summovie_command.append(micrograph_name)
# Shift file
summovie_command.append(shift_name)
# FRC file
summovie_command.append(frc_name),
# First frame
summovie_command.append('{0}'.format(first))
# Last frame
summovie_command.append('{0}'.format(last))
# Pixel size
summovie_command.append('{0}'.format(opt['pixel_size']))
# Dose correction
if not opt['apply_dose_filter']:
summovie_command.append('NO')
else:
summovie_command.append('YES')
# Exposure per frame
summovie_command.append('{0}'.format(opt['exposure_per_frame']))
# Acceleration voltage
summovie_command.append('{0}'.format(opt['voltage']))
# Pre exposure
summovie_command.append('{0}'.format(opt['pre_exposure']))
# Restore noise power
if opt['dont_restore_noise']:
summovie_command.append('NO')
else:
summovie_command.append('YES')
return summovie_command
def main():
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from logger import Logger, BaseLogger_Files
import sys, os, time
global Tracker, Blockdata
from global_def import ERROR
progname = os.path.basename(sys.argv[0])
usage = progname + " --output_dir=output_dir --isac_dir=output_dir_of_isac "
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--pw_adjustment", type ="string", default ='analytical_model', \
help="adjust power spectrum of 2-D averages to an analytic model. Other opions: no_adjustment; bfactor; a text file of 1D rotationally averaged PW")
#### Four options for --pw_adjustment:
# 1> analytical_model(default);
# 2> no_adjustment;
# 3> bfactor;
# 4> adjust_to_given_pw2(user has to provide a text file that contains 1D rotationally averaged PW)
# options in common
parser.add_option(
"--isac_dir",
type="string",
default='',
help="ISAC run output directory, input directory for this command")
parser.add_option(
"--output_dir",
type="string",
default='',
help="output directory where computed averages are saved")
parser.add_option(
"--pixel_size",
type="float",
default=-1.0,
help=
"pixel_size of raw images. one can put 1.0 in case of negative stain data"
)
parser.add_option(
"--fl",
type="float",
default=-1.0,
help=
"low pass filter, = -1.0, not applied; =0.0, using FH1 (initial resolution), = 1.0 using FH2 (resolution after local alignment), or user provided value in absolute freqency [0.0:0.5]"
)
parser.add_option("--stack",
type="string",
default="",
help="data stack used in ISAC")
parser.add_option("--radius", type="int", default=-1, help="radius")
parser.add_option("--xr",
type="float",
default=-1.0,
help="local alignment search range")
#parser.add_option("--ts", type ="float", default =1.0, help= "local alignment search step")
parser.add_option("--fh",
type="float",
default=-1.0,
help="local alignment high frequencies limit")
#parser.add_option("--maxit", type ="int", default =5, help= "local alignment iterations")
parser.add_option("--navg",
type="int",
default=1000000,
help="number of aveages")
parser.add_option("--local_alignment",
action="store_true",
default=False,
help="do local alignment")
parser.add_option(
"--noctf",
action="store_true",
default=False,
help=
"no ctf correction, useful for negative stained data. always ctf for cryo data"
)
parser.add_option(
"--B_start",
type="float",
default=45.0,
help=
"start frequency (Angstrom) of power spectrum for B_factor estimation")
parser.add_option(
"--Bfactor",
type="float",
default=-1.0,
help=
"User defined bactors (e.g. 25.0[A^2]). By default, the program automatically estimates B-factor. "
)
(options, args) = parser.parse_args(sys.argv[1:])
adjust_to_analytic_model = False
adjust_to_given_pw2 = False
B_enhance = False
no_adjustment = False
if options.pw_adjustment == 'analytical_model':
adjust_to_analytic_model = True
elif options.pw_adjustment == 'no_adjustment':
no_adjustment = True
elif options.pw_adjustment == 'bfactor':
B_enhance = True
else:
adjust_to_given_pw2 = True
from utilities import get_im, bcast_number_to_all, write_text_file, read_text_file, wrap_mpi_bcast, write_text_row
from utilities import cmdexecute
from filter import filt_tanl
from logger import Logger, BaseLogger_Files
import user_functions
import string
from string import split, atoi, atof
import json
mpi_init(0, [])
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
Blockdata = {}
# MPI stuff
Blockdata["nproc"] = nproc
Blockdata["myid"] = myid
Blockdata["main_node"] = 0
Blockdata["shared_comm"] = mpi_comm_split_type(MPI_COMM_WORLD,
MPI_COMM_TYPE_SHARED, 0,
MPI_INFO_NULL)
Blockdata["myid_on_node"] = mpi_comm_rank(Blockdata["shared_comm"])
Blockdata["no_of_processes_per_group"] = mpi_comm_size(
Blockdata["shared_comm"])
masters_from_groups_vs_everything_else_comm = mpi_comm_split(
MPI_COMM_WORLD, Blockdata["main_node"] == Blockdata["myid_on_node"],
Blockdata["myid_on_node"])
Blockdata["color"], Blockdata["no_of_groups"], balanced_processor_load_on_nodes = get_colors_and_subsets(Blockdata["main_node"], MPI_COMM_WORLD, Blockdata["myid"], \
Blockdata["shared_comm"], Blockdata["myid_on_node"], masters_from_groups_vs_everything_else_comm)
# We need two nodes for processing of volumes
Blockdata["node_volume"] = [
Blockdata["no_of_groups"] - 3, Blockdata["no_of_groups"] - 2,
Blockdata["no_of_groups"] - 1
] # For 3D stuff take three last nodes
# We need two CPUs for processing of volumes, they are taken to be main CPUs on each volume
# We have to send the two myids to all nodes so we can identify main nodes on two selected groups.
Blockdata["nodes"] = [Blockdata["node_volume"][0]*Blockdata["no_of_processes_per_group"],Blockdata["node_volume"][1]*Blockdata["no_of_processes_per_group"], \
Blockdata["node_volume"][2]*Blockdata["no_of_processes_per_group"]]
# End of Blockdata: sorting requires at least three nodes, and the used number of nodes be integer times of three
global_def.BATCH = True
global_def.MPI = True
if adjust_to_given_pw2:
checking_flag = 0
if (Blockdata["myid"] == Blockdata["main_node"]):
if not os.path.exists(options.pw_adjustment): checking_flag = 1
checking_flag = bcast_number_to_all(checking_flag,
Blockdata["main_node"],
MPI_COMM_WORLD)
if checking_flag == 1:
ERROR("User provided power spectrum does not exist",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
Tracker = {}
Constants = {}
Constants["isac_dir"] = options.isac_dir
Constants["masterdir"] = options.output_dir
Constants["pixel_size"] = options.pixel_size
Constants["orgstack"] = options.stack
Constants["radius"] = options.radius
Constants["xrange"] = options.xr
Constants["FH"] = options.fh
Constants["low_pass_filter"] = options.fl
#Constants["maxit"] = options.maxit
Constants["navg"] = options.navg
Constants["B_start"] = options.B_start
Constants["Bfactor"] = options.Bfactor
if adjust_to_given_pw2: Constants["modelpw"] = options.pw_adjustment
Tracker["constants"] = Constants
# -------------------------------------------------------------
#
# Create and initialize Tracker dictionary with input options # State Variables
#<<<---------------------->>>imported functions<<<---------------------------------------------
#x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1)
#y_range = x_range
####-----------------------------------------------------------
# Create Master directory and associated subdirectories
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]:
masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen")
else:
masterdir = Tracker["constants"]["masterdir"]
if (Blockdata["myid"] == Blockdata["main_node"]):
msg = "Postprocessing ISAC 2D averages starts"
print(line, "Postprocessing ISAC 2D averages starts")
if not masterdir:
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "sharpen_" + Tracker["constants"]["isac_dir"]
os.mkdir(masterdir)
else:
if os.path.exists(masterdir):
print("%s already exists" % masterdir)
else:
os.mkdir(masterdir)
subdir_path = os.path.join(masterdir, "ali2d_local_params_avg")
if not os.path.exists(subdir_path): os.mkdir(subdir_path)
subdir_path = os.path.join(masterdir, "params_avg")
if not os.path.exists(subdir_path): os.mkdir(subdir_path)
li = len(masterdir)
else:
li = 0
li = mpi_bcast(li, 1, MPI_INT, Blockdata["main_node"], MPI_COMM_WORLD)[0]
masterdir = mpi_bcast(masterdir, li, MPI_CHAR, Blockdata["main_node"],
MPI_COMM_WORLD)
masterdir = string.join(masterdir, "")
Tracker["constants"]["masterdir"] = masterdir
log_main = Logger(BaseLogger_Files())
log_main.prefix = Tracker["constants"]["masterdir"] + "/"
while not os.path.exists(Tracker["constants"]["masterdir"]):
print("Node ", Blockdata["myid"], " waiting...",
Tracker["constants"]["masterdir"])
sleep(1)
mpi_barrier(MPI_COMM_WORLD)
if (Blockdata["myid"] == Blockdata["main_node"]):
init_dict = {}
print(Tracker["constants"]["isac_dir"])
Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"],
"2dalignment")
core = read_text_row(
os.path.join(Tracker["directory"], "initial2Dparams.txt"))
for im in range(len(core)):
init_dict[im] = core[im]
del core
else:
init_dict = 0
init_dict = wrap_mpi_bcast(init_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
###
do_ctf = True
if options.noctf: do_ctf = False
if (Blockdata["myid"] == Blockdata["main_node"]):
if do_ctf: print("CTF correction is on")
else: print("CTF correction is off")
if options.local_alignment: print("local refinement is on")
else: print("local refinement is off")
if B_enhance: print("Bfactor is to be applied on averages")
elif adjust_to_given_pw2:
print("PW of averages is adjusted to a given 1D PW curve")
elif adjust_to_analytic_model:
print("PW of averages is adjusted to analytical model")
else:
print("PW of averages is not adjusted")
#Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack")
image = get_im(Tracker["constants"]["orgstack"], 0)
Tracker["constants"]["nnxo"] = image.get_xsize()
if Tracker["constants"]["pixel_size"] == -1.0:
print(
"Pixel size value is not provided by user. extracting it from ctf header entry of the original stack."
)
try:
ctf_params = image.get_attr("ctf")
Tracker["constants"]["pixel_size"] = ctf_params.apix
except:
ERROR(
"Pixel size could not be extracted from the original stack.",
"sxcompute_isac_avg.py", 1,
Blockdata["myid"]) # action=1 - fatal error, exit
## Now fill in low-pass filter
isac_shrink_path = os.path.join(Tracker["constants"]["isac_dir"],
"README_shrink_ratio.txt")
if not os.path.exists(isac_shrink_path):
ERROR(
"%s does not exist in the specified ISAC run output directory"
% (isac_shrink_path), "sxcompute_isac_avg.py", 1,
Blockdata["myid"]) # action=1 - fatal error, exit
isac_shrink_file = open(isac_shrink_path, "r")
isac_shrink_lines = isac_shrink_file.readlines()
isac_shrink_ratio = float(
isac_shrink_lines[5]
) # 6th line: shrink ratio (= [target particle radius]/[particle radius]) used in the ISAC run
isac_radius = float(
isac_shrink_lines[6]
) # 7th line: particle radius at original pixel size used in the ISAC run
isac_shrink_file.close()
print("Extracted parameter values")
print("ISAC shrink ratio : {0}".format(isac_shrink_ratio))
print("ISAC particle radius : {0}".format(isac_radius))
Tracker["ini_shrink"] = isac_shrink_ratio
else:
Tracker["ini_shrink"] = 0.0
Tracker = wrap_mpi_bcast(Tracker,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
#print(Tracker["constants"]["pixel_size"], "pixel_size")
x_range = max(Tracker["constants"]["xrange"],
int(1. / Tracker["ini_shrink"] + 0.99999))
a_range = y_range = x_range
if (Blockdata["myid"] == Blockdata["main_node"]):
parameters = read_text_row(
os.path.join(Tracker["constants"]["isac_dir"],
"all_parameters.txt"))
else:
parameters = 0
parameters = wrap_mpi_bcast(parameters,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
params_dict = {}
list_dict = {}
#parepare params_dict
#navg = min(Tracker["constants"]["navg"]*Blockdata["nproc"], EMUtil.get_image_count(os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf")))
navg = min(
Tracker["constants"]["navg"],
EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf")))
global_dict = {}
ptl_list = []
memlist = []
if (Blockdata["myid"] == Blockdata["main_node"]):
print("Number of averages computed in this run is %d" % navg)
for iavg in range(navg):
params_of_this_average = []
image = get_im(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"), iavg)
members = sorted(image.get_attr("members"))
memlist.append(members)
for im in range(len(members)):
abs_id = members[im]
global_dict[abs_id] = [iavg, im]
P = combine_params2( init_dict[abs_id][0], init_dict[abs_id][1], init_dict[abs_id][2], init_dict[abs_id][3], \
parameters[abs_id][0], parameters[abs_id][1]/Tracker["ini_shrink"], parameters[abs_id][2]/Tracker["ini_shrink"], parameters[abs_id][3])
if parameters[abs_id][3] == -1:
print(
"WARNING: Image #{0} is an unaccounted particle with invalid 2D alignment parameters and should not be the member of any classes. Please check the consitency of input dataset."
.format(abs_id)
) # How to check what is wrong about mirror = -1 (Toshio 2018/01/11)
params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0])
ptl_list.append(abs_id)
params_dict[iavg] = params_of_this_average
list_dict[iavg] = members
write_text_row(
params_of_this_average,
os.path.join(Tracker["constants"]["masterdir"], "params_avg",
"params_avg_%03d.txt" % iavg))
ptl_list.sort()
init_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
init_params[im] = [ptl_list[im]] + params_dict[global_dict[
ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
init_params,
os.path.join(Tracker["constants"]["masterdir"],
"init_isac_params.txt"))
else:
params_dict = 0
list_dict = 0
memlist = 0
params_dict = wrap_mpi_bcast(params_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
list_dict = wrap_mpi_bcast(list_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
memlist = wrap_mpi_bcast(memlist,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
# Now computing!
del init_dict
tag_sharpen_avg = 1000
## always apply low pass filter to B_enhanced images to suppress noise in high frequencies
enforced_to_H1 = False
if B_enhance:
if Tracker["constants"]["low_pass_filter"] == -1.0:
enforced_to_H1 = True
if navg < Blockdata["nproc"]: # Each CPU do one average
ERROR("number of nproc is larger than number of averages",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
else:
FH_list = [[0, 0.0, 0.0] for im in range(navg)]
image_start, image_end = MPI_start_end(navg, Blockdata["nproc"],
Blockdata["myid"])
if Blockdata["myid"] == Blockdata["main_node"]:
cpu_dict = {}
for iproc in range(Blockdata["nproc"]):
local_image_start, local_image_end = MPI_start_end(
navg, Blockdata["nproc"], iproc)
for im in range(local_image_start, local_image_end):
cpu_dict[im] = iproc
else:
cpu_dict = 0
cpu_dict = wrap_mpi_bcast(cpu_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
slist = [None for im in range(navg)]
ini_list = [None for im in range(navg)]
avg1_list = [None for im in range(navg)]
avg2_list = [None for im in range(navg)]
plist_dict = {}
data_list = [None for im in range(navg)]
if Blockdata["myid"] == Blockdata["main_node"]:
if B_enhance:
print(
"Avg ID B-factor FH1(Res before ali) FH2(Res after ali)"
)
else:
print("Avg ID FH1(Res before ali) FH2(Res after ali)")
for iavg in range(image_start, image_end):
mlist = EMData.read_images(Tracker["constants"]["orgstack"],
list_dict[iavg])
for im in range(len(mlist)):
#mlist[im]= get_im(Tracker["constants"]["orgstack"], list_dict[iavg][im])
set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
if options.local_alignment:
"""
new_average1 = within_group_refinement([mlist[kik] for kik in range(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH=max(Tracker["constants"]["FH"], FH1), FF=0.02, method="")
new_average2 = within_group_refinement([mlist[kik] for kik in range(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou= Tracker["constants"]["radius"], rs=1.0, xrng=[ x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.02, method="")
new_avg, frc, plist = compute_average(mlist, Tracker["constants"]["radius"], do_ctf)
"""
new_avg, plist, FH2 = refinement_2d_local(
mlist,
Tracker["constants"]["radius"],
a_range,
x_range,
y_range,
CTF=do_ctf,
SNR=1.0e10)
plist_dict[iavg] = plist
FH1 = -1.0
else:
new_avg, frc, plist = compute_average(
mlist, Tracker["constants"]["radius"], do_ctf)
FH1 = get_optimistic_res(frc)
FH2 = -1.0
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg))
FH_list[iavg] = [iavg, FH1, FH2]
if B_enhance:
new_avg, gb = apply_enhancement(
new_avg, Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"])
print(" %6d %6.3f %4.3f %4.3f" % (iavg, gb, FH1, FH2))
elif adjust_to_given_pw2:
roo = read_text_file(Tracker["constants"]["modelpw"], -1)
roo = roo[0] # always on the first column
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], roo)
print(" %6d %4.3f %4.3f " % (iavg, FH1, FH2))
elif adjust_to_analytic_model:
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], None)
print(" %6d %4.3f %4.3f " % (iavg, FH1, FH2))
elif no_adjustment:
pass
if Tracker["constants"]["low_pass_filter"] != -1.0:
if Tracker["constants"]["low_pass_filter"] == 0.0:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 1.0:
low_pass_filter = FH2
if not options.local_alignment: low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"]["low_pass_filter"]
if low_pass_filter >= 0.45: low_pass_filter = 0.45
new_avg = filt_tanl(new_avg, low_pass_filter, 0.02)
else: # No low pass filter but if enforced
if enforced_to_H1: new_avg = filt_tanl(new_avg, FH1, 0.02)
if B_enhance: new_avg = fft(new_avg)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
slist[iavg] = new_avg
print(
strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>",
"Refined average %7d" % iavg)
## send to main node to write
mpi_barrier(MPI_COMM_WORLD)
for im in range(navg):
# avg
if cpu_dict[im] == Blockdata[
"myid"] and Blockdata["myid"] != Blockdata["main_node"]:
send_EMData(slist[im], Blockdata["main_node"], tag_sharpen_avg)
elif cpu_dict[im] == Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
slist[im].set_attr("members", memlist[im])
slist[im].set_attr("n_objects", len(memlist[im]))
slist[im].write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
elif cpu_dict[im] != Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
new_avg_other_cpu = recv_EMData(cpu_dict[im], tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.set_attr("n_objects", len(memlist[im]))
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
if options.local_alignment:
if cpu_dict[im] == Blockdata["myid"]:
write_text_row(
plist_dict[im],
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params_avg",
"ali2d_local_params_avg_%03d.txt" % im))
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(plist_dict[im], Blockdata["main_node"],
MPI_COMM_WORLD)
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
plist_dict[im] = dummy
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
else:
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi_barrier(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if options.local_alignment:
if Blockdata["myid"] == Blockdata["main_node"]:
ali3d_local_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
ali3d_local_params[im] = [ptl_list[im]] + plist_dict[
global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
ali3d_local_params,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params.txt"))
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
else:
if Blockdata["myid"] == Blockdata["main_node"]:
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
mpi_barrier(MPI_COMM_WORLD)
target_xr = 3
target_yr = 3
if (Blockdata["myid"] == 0):
cmd = "{} {} {} {} {} {} {} {} {} {}".format("sxchains.py", os.path.join(Tracker["constants"]["masterdir"],"class_averages.hdf"),\
os.path.join(Tracker["constants"]["masterdir"],"junk.hdf"),os.path.join(Tracker["constants"]["masterdir"],"ordered_class_averages.hdf"),\
"--circular","--radius=%d"%Tracker["constants"]["radius"] , "--xr=%d"%(target_xr+1),"--yr=%d"%(target_yr+1),"--align", ">/dev/null")
junk = cmdexecute(cmd)
cmd = "{} {}".format(
"rm -rf",
os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"))
junk = cmdexecute(cmd)
from mpi import mpi_finalize
mpi_finalize()
exit()