def main():
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from logger import Logger, BaseLogger_Files
import sys, os, time
global Tracker, Blockdata
from global_def import ERROR
progname = os.path.basename(sys.argv[0])
usage = progname + " --output_dir=output_dir --isac_dir=output_dir_of_isac "
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--adjust_to_analytic_model",
action="store_true",
default=False,
help="adjust power spectrum of 2-D averages to an analytic model ")
parser.add_option(
"--adjust_to_given_pw2",
action="store_true",
default=False,
help="adjust power spectrum to 2-D averages to given 1D power spectrum"
)
parser.add_option("--B_enhance",
action="store_true",
default=False,
help="using B-factor to enhance 2-D averages")
parser.add_option("--no_adjustment",
action="store_true",
default=False,
help="No power spectrum adjustment")
options_list = []
adjust_to_analytic_model = False
for q in sys.argv[1:]:
if (q[:26] == "--adjust_to_analytic_model"):
adjust_to_analytic_model = True
options_list.append(q)
break
adjust_to_given_pw2 = False
for q in sys.argv[1:]:
if (q[:21] == "--adjust_to_given_pw2"):
adjust_to_given_pw2 = True
options_list.append(q)
break
B_enhance = False
for q in sys.argv[1:]:
if (q[:11] == "--B_enhance"):
B_enhance = True
options_list.append(q)
break
no_adjustment = False
for q in sys.argv[1:]:
if (q[:15] == "--no_adjustment"):
no_adjustment = True
options_list.append(q)
break
if len(options_list) == 0:
if (Blockdata["myid"] == Blockdata["main_node"]):
print(
"specify one of the following options to start: 1. adjust_to_analytic_model; 2. adjust_to_given_pw2; 3. B_enhance; 4. no_adjustment"
)
if len(options_list) > 1:
ERROR(
"The specified options are exclusive. Use only one of them to start",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
# options in common
parser.add_option(
"--isac_dir",
type="string",
default='',
help="ISAC run output directory, input directory for this command")
parser.add_option(
"--output_dir",
type="string",
default='',
help="output directory where computed averages are saved")
parser.add_option("--pixel_size",
type="float",
default=-1.0,
help="pixel_size of raw images")
parser.add_option(
"--fl",
type="float",
default=-1.0,
help=
"low pass filter, =-1, not applied; =1, using FH1 (initial resolution), =2 using FH2 (resolution after local alignment), or user provided value"
)
parser.add_option("--stack",
type="string",
default="",
help="data stack used in ISAC")
parser.add_option("--radius", type="int", default=-1, help="radius")
parser.add_option("--xr",
type="float",
default=-1.0,
help="local alignment search range")
parser.add_option("--ts",
type="float",
default=1.0,
help="local alignment search step")
parser.add_option("--fh",
type="float",
default=-1.,
help="local alignment high frequencies limit")
parser.add_option("--maxit",
type="int",
default=5,
help="local alignment iterations")
parser.add_option("--navg",
type="int",
default=-1,
help="number of aveages")
parser.add_option("--skip_local_alignment",
action="store_true",
default=False,
help="skip local alignment")
parser.add_option(
"--noctf",
action="store_true",
default=False,
help=
"no ctf correction, useful for negative stained data. always ctf for cryo data"
)
if B_enhance:
parser.add_option(
"--B_start",
type="float",
default=10.0,
help=
"start frequency (1./Angstrom) of power spectrum for B_factor estimation"
)
parser.add_option(
"--Bfactor",
type="float",
default=-1.0,
help=
"User defined bactors (e.g. 45.0[A^2]). By default, the program automatically estimates B-factor. "
)
if adjust_to_given_pw2:
parser.add_option("--modelpw",
type="string",
default='',
help="1-D reference power spectrum")
checking_flag = 0
if (Blockdata["myid"] == Blockdata["main_node"]):
if not os.path.exists(options.modelpw): checking_flag = 1
checking_flag = bcast_number_to_all(checking_flag,
Blockdata["main_node"],
MPI_COMM_WORLD)
if checking_flag == 1:
ERROR("User provided power spectrum does not exist",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
(options, args) = parser.parse_args(sys.argv[1:])
Tracker = {}
Constants = {}
Constants["isac_dir"] = options.isac_dir
Constants["masterdir"] = options.output_dir
Constants["pixel_size"] = options.pixel_size
Constants["orgstack"] = options.stack
Constants["radius"] = options.radius
Constants["xrange"] = options.xr
Constants["xstep"] = options.ts
Constants["FH"] = options.fh
Constants["maxit"] = options.maxit
Constants["navg"] = options.navg
Constants["low_pass_filter"] = options.fl
if B_enhance:
Constants["B_start"] = options.B_start
Constants["Bfactor"] = options.Bfactor
if adjust_to_given_pw2: Constants["modelpw"] = options.modelpw
Tracker["constants"] = Constants
# -------------------------------------------------------------
#
# Create and initialize Tracker dictionary with input options # State Variables
#<<<---------------------->>>imported functions<<<---------------------------------------------
from utilities import get_im, bcast_number_to_all, write_text_file, read_text_file, wrap_mpi_bcast, write_text_row
from utilities import cmdexecute
from filter import filt_tanl
from time import sleep
from logger import Logger, BaseLogger_Files
import user_functions
import string
from string import split, atoi, atof
import json
#x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1)
#y_range = x_range
####-----------------------------------------------------------
# Create Master directory
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]:
masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen")
else:
masterdir = Tracker["constants"]["masterdir"]
if (Blockdata["myid"] == Blockdata["main_node"]):
msg = "Postprocessing ISAC 2D averages starts"
print(line, "Postprocessing ISAC 2D averages starts")
if not masterdir:
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "sharpen_" + Tracker["constants"]["isac_dir"]
os.mkdir(masterdir)
else:
if os.path.exists(masterdir):
print("%s already exists" % masterdir)
else:
os.mkdir(masterdir)
li = len(masterdir)
else:
li = 0
li = mpi_bcast(li, 1, MPI_INT, Blockdata["main_node"], MPI_COMM_WORLD)[0]
masterdir = mpi_bcast(masterdir, li, MPI_CHAR, Blockdata["main_node"],
MPI_COMM_WORLD)
masterdir = string.join(masterdir, "")
Tracker["constants"]["masterdir"] = masterdir
log_main = Logger(BaseLogger_Files())
log_main.prefix = Tracker["constants"]["masterdir"] + "/"
while not os.path.exists(Tracker["constants"]["masterdir"]):
print("Node ", Blockdata["myid"], " waiting...",
Tracker["constants"]["masterdir"])
sleep(1)
mpi_barrier(MPI_COMM_WORLD)
if (Blockdata["myid"] == Blockdata["main_node"]):
init_dict = {}
print(Tracker["constants"]["isac_dir"])
Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"],
"2dalignment")
core = read_text_row(
os.path.join(Tracker["directory"], "initial2Dparams.txt"))
for im in xrange(len(core)):
init_dict[im] = core[im]
del core
else:
init_dict = 0
init_dict = wrap_mpi_bcast(init_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
###
if (Blockdata["myid"] == Blockdata["main_node"]):
#Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack")
image = get_im(Tracker["constants"]["orgstack"], 0)
Tracker["constants"]["nnxo"] = image.get_xsize()
try:
ctf_params = image.get_attr("ctf")
if Tracker["constants"]["pixel_size"] == -1.:
Tracker["constants"]["pixel_size"] = ctf_params.apix
except:
print("pixel size value is not given.")
Tracker["ini_shrink"] = float(
get_im(os.path.join(Tracker["directory"], "aqfinal.hdf"),
0).get_xsize()) / Tracker["constants"]["nnxo"]
else:
Tracker["ini_shrink"] = 0
Tracker = wrap_mpi_bcast(Tracker,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
#print(Tracker["constants"]["pixel_size"], "pixel_size")
x_range = max(Tracker["constants"]["xrange"],
int(1. / Tracker["ini_shrink"]) + 1)
y_range = x_range
if (Blockdata["myid"] == Blockdata["main_node"]):
parameters = read_text_row(
os.path.join(Tracker["constants"]["isac_dir"],
"all_parameters.txt"))
else:
parameters = 0
parameters = wrap_mpi_bcast(parameters,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
params_dict = {}
list_dict = {}
#parepare params_dict
if Tracker["constants"]["navg"] < 0:
navg = EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"))
else:
navg = min(
Tracker["constants"]["navg"],
EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf")))
global_dict = {}
ptl_list = []
memlist = []
if (Blockdata["myid"] == Blockdata["main_node"]):
for iavg in xrange(navg):
params_of_this_average = []
image = get_im(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"), iavg)
members = image.get_attr("members")
memlist.append(members)
for im in xrange(len(members)):
abs_id = members[im]
global_dict[abs_id] = [iavg, im]
P = combine_params2( init_dict[abs_id][0], init_dict[abs_id][1], init_dict[abs_id][2], init_dict[abs_id][3], \
parameters[abs_id][0], parameters[abs_id][1]/Tracker["ini_shrink"], parameters[abs_id][2]/Tracker["ini_shrink"], parameters[abs_id][3])
if parameters[abs_id][3] == -1: print("wrong one")
params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0])
ptl_list.append(abs_id)
params_dict[iavg] = params_of_this_average
list_dict[iavg] = members
write_text_row(
params_of_this_average,
os.path.join(Tracker["constants"]["masterdir"],
"params_avg_%03d.txt" % iavg))
ptl_list.sort()
init_params = [None for im in xrange(len(ptl_list))]
for im in xrange(len(ptl_list)):
init_params[im] = [ptl_list[im]] + params_dict[global_dict[
ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
init_params,
os.path.join(Tracker["constants"]["masterdir"],
"init_isac_params.txt"))
else:
params_dict = 0
list_dict = 0
memlist = 0
params_dict = wrap_mpi_bcast(params_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
list_dict = wrap_mpi_bcast(list_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
memlist = wrap_mpi_bcast(memlist,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
# Now computing!
del init_dict
tag_sharpen_avg = 1000
## always apply low pass filter to B_enhanced images to suppress noise in high frequencies
enforced_to_H1 = False
if options.B_enhance:
if Tracker["constants"]["low_pass_filter"] == -1:
print("User does not provide low pass filter")
enforced_to_H1 = True
if navg < Blockdata["nproc"]: # Each CPU do one average
FH_list = [None for im in xrange(navg)]
for iavg in xrange(navg):
if Blockdata["myid"] == iavg:
mlist = [None for i in xrange(len(list_dict[iavg]))]
for im in xrange(len(mlist)):
mlist[im] = get_im(Tracker["constants"]["orgstack"],
list_dict[iavg][im])
set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
FH1 = get_optimistic_res(frc)
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d_before_ali.txt"%iavg))
if not options.skip_local_alignment:
new_average1 = within_group_refinement([mlist[kik] for kik in xrange(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1)
new_average2 = within_group_refinement([mlist[kik] for kik in xrange(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1)
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
FH2 = get_optimistic_res(frc)
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg))
#if Tracker["constants"]["nopwadj"]: # pw adjustment, 1. analytic model 2. PDB model 3. B-facttor enhancement
else:
FH2 = 0.0
FH_list[iavg] = [FH1, FH2]
if options.B_enhance:
new_avg, gb = apply_enhancement(
new_avg, Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"])
print("Process avg %d %f %f %f" %
(iavg, gb, FH1, FH2))
elif options.adjust_to_given_pw2:
roo = read_text_file(Tracker["constants"]["modelpw"], -1)
roo = roo[0] # always put pw in the first column
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], roo)
elif options.adjust_to_analytic_model:
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], None)
elif options.no_adjustment:
pass
print("Process avg %d %f %f" % (iavg, FH1, FH2))
if Tracker["constants"]["low_pass_filter"] != -1.:
if Tracker["constants"]["low_pass_filter"] == 1.:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 2.:
low_pass_filter = FH2
if options.skip_local_alignment: low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"][
"low_pass_filter"]
if low_pass_filter >= 0.45: low_pass_filter = 0.45
new_avg = filt_tanl(new_avg, low_pass_filter, 0.1)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
mpi_barrier(MPI_COMM_WORLD)
for im in xrange(navg): # avg
if im == Blockdata[
"myid"] and Blockdata["myid"] != Blockdata["main_node"]:
send_EMData(new_avg, Blockdata["main_node"], tag_sharpen_avg)
elif Blockdata["myid"] == Blockdata["main_node"]:
if im != Blockdata["main_node"]:
new_avg_other_cpu = recv_EMData(im, tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
else:
new_avg.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
if not options.skip_local_alignment:
if im == Blockdata["myid"]:
write_text_row(
plist,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params_avg_%03d.txt" % im))
if Blockdata["myid"] == im and Blockdata["myid"] != Blockdata[
"main_node"]:
wrap_mpi_send(plist_dict[im], Blockdata["main_node"],
MPI_COMM_WORLD)
elif im != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(im, MPI_COMM_WORLD)
plist_dict[im] = dummy
if im == Blockdata["myid"] and im != Blockdata["main_node"]:
wrap_mpi_send(FH_list[im], Blockdata["main_node"],
MPI_COMM_WORLD)
elif im != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(im, MPI_COMM_WORLD)
FH_list[im] = dummy
else:
if im == Blockdata["myid"] and im != Blockdata["main_node"]:
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif im != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(im, MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi_barrier(MPI_COMM_WORLD)
else:
FH_list = [[0, 0.0, 0.0] for im in xrange(navg)]
image_start, image_end = MPI_start_end(navg, Blockdata["nproc"],
Blockdata["myid"])
if Blockdata["myid"] == Blockdata["main_node"]:
cpu_dict = {}
for iproc in xrange(Blockdata["nproc"]):
local_image_start, local_image_end = MPI_start_end(
navg, Blockdata["nproc"], iproc)
for im in xrange(local_image_start, local_image_end):
cpu_dict[im] = iproc
else:
cpu_dict = 0
cpu_dict = wrap_mpi_bcast(cpu_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
slist = [None for im in xrange(navg)]
ini_list = [None for im in xrange(navg)]
avg1_list = [None for im in xrange(navg)]
avg2_list = [None for im in xrange(navg)]
plist_dict = {}
data_list = [None for im in xrange(navg)]
if Blockdata["myid"] == Blockdata["main_node"]: print("read data")
for iavg in xrange(image_start, image_end):
mlist = [None for i in xrange(len(list_dict[iavg]))]
for im in xrange(len(mlist)):
mlist[im] = get_im(Tracker["constants"]["orgstack"],
list_dict[iavg][im])
set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
data_list[iavg] = mlist
print("read data done %d" % Blockdata["myid"])
#if Blockdata["myid"] == Blockdata["main_node"]: print("start to compute averages")
for iavg in xrange(image_start, image_end):
mlist = data_list[iavg]
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
FH1 = get_optimistic_res(frc)
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d_before_ali.txt"%iavg))
if not options.skip_local_alignment:
new_average1 = within_group_refinement([mlist[kik] for kik in xrange(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH=max(Tracker["constants"]["FH"], FH1), FF=0.1)
new_average2 = within_group_refinement([mlist[kik] for kik in xrange(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou= Tracker["constants"]["radius"], rs=1.0, xrng=[ x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1)
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
plist_dict[iavg] = plist
FH2 = get_optimistic_res(frc)
else:
FH2 = 0.0
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg))
FH_list[iavg] = [iavg, FH1, FH2]
if options.B_enhance:
new_avg, gb = apply_enhancement(
new_avg, Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"])
print("Process avg %d %f %f %f" % (iavg, gb, FH1, FH2))
elif options.adjust_to_given_pw2:
roo = read_text_file(Tracker["constants"]["modelpw"], -1)
roo = roo[0] # always on the first column
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], roo)
print("Process avg %d %f %f" % (iavg, FH1, FH2))
elif adjust_to_analytic_model:
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], None)
print("Process avg %d %f %f" % (iavg, FH1, FH2))
elif options.no_adjustment:
pass
if Tracker["constants"]["low_pass_filter"] != -1.:
new_avg = filt_tanl(new_avg,
Tracker["constants"]["low_pass_filter"],
0.1)
if Tracker["constants"]["low_pass_filter"] != -1.:
if Tracker["constants"]["low_pass_filter"] == 1.:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 2.:
low_pass_filter = FH2
if options.skip_local_alignment: low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"]["low_pass_filter"]
if low_pass_filter >= 0.45: low_pass_filter = 0.45
new_avg = filt_tanl(new_avg, low_pass_filter, 0.1)
else:
if enforced_to_H1: new_avg = filt_tanl(new_avg, FH1, 0.1)
if options.B_enhance: new_avg = fft(new_avg)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
slist[iavg] = new_avg
## send to main node to write
mpi_barrier(MPI_COMM_WORLD)
for im in xrange(navg):
# avg
if cpu_dict[im] == Blockdata[
"myid"] and Blockdata["myid"] != Blockdata["main_node"]:
send_EMData(slist[im], Blockdata["main_node"], tag_sharpen_avg)
elif cpu_dict[im] == Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
slist[im].set_attr("members", memlist[im])
slist[im].write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
elif cpu_dict[im] != Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
new_avg_other_cpu = recv_EMData(cpu_dict[im], tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
if not options.skip_local_alignment:
if cpu_dict[im] == Blockdata["myid"]:
write_text_row(
plist_dict[im],
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params_avg_%03d.txt" % im))
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(plist_dict[im], Blockdata["main_node"],
MPI_COMM_WORLD)
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
plist_dict[im] = dummy
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
else:
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi_barrier(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if not options.skip_local_alignment:
if Blockdata["myid"] == Blockdata["main_node"]:
ali3d_local_params = [None for im in xrange(len(ptl_list))]
for im in xrange(len(ptl_list)):
ali3d_local_params[im] = [ptl_list[im]] + plist_dict[
global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
ali3d_local_params,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params.txt"))
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
else:
if Blockdata["myid"] == Blockdata["main_node"]:
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
mpi_barrier(MPI_COMM_WORLD)
target_xr = 3
target_yr = 3
if (Blockdata["myid"] == 0):
cmd = "{} {} {} {} {} {} {} {} {} {}".format("sxchains.py", os.path.join(Tracker["constants"]["masterdir"],"class_averages.hdf"),\
os.path.join(Tracker["constants"]["masterdir"],"junk.hdf"),os.path.join(Tracker["constants"]["masterdir"],"ordered_class_averages.hdf"),\
"--circular","--radius=%d"%Tracker["constants"]["radius"] , "--xr=%d"%(target_xr+1),"--yr=%d"%(target_yr+1),"--align", ">/dev/null")
junk = cmdexecute(cmd)
cmd = "{} {}".format(
"rm -rf",
os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"))
junk = cmdexecute(cmd)
from mpi import mpi_finalize
mpi_finalize()
exit()
def main():
from logger import Logger, BaseLogger_Files
arglist = []
i = 0
while( i < len(sys.argv) ):
if sys.argv[i]=='-p4pg':
i = i+2
elif sys.argv[i]=='-p4wd':
i = i+2
else:
arglist.append( sys.argv[i] )
i = i+1
progname = os.path.basename(arglist[0])
usage = progname + " stack outdir <mask> --focus=3Dmask --radius=outer_radius --delta=angular_step" +\
"--an=angular_neighborhood --maxit=max_iter --CTF --sym=c1 --function=user_function --independent=indenpendent_runs --number_of_images_per_group=number_of_images_per_group --low_pass_frequency=.25 --seed=random_seed"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--focus", type ="string", default ='', help="bineary 3D mask for focused clustering ")
parser.add_option("--ir", type = "int", default =1, help="inner radius for rotational correlation > 0 (set to 1)")
parser.add_option("--radius", type = "int", default =-1, help="particle radius in pixel for rotational correlation <nx-1 (set to the radius of the particle)")
parser.add_option("--maxit", type = "int", default =25, help="maximum number of iteration")
parser.add_option("--rs", type = "int", default =1, help="step between rings in rotational correlation >0 (set to 1)" )
parser.add_option("--xr", type ="string", default ='1', help="range for translation search in x direction, search is +/-xr ")
parser.add_option("--yr", type ="string", default ='-1', help="range for translation search in y direction, search is +/-yr (default = same as xr)")
parser.add_option("--ts", type ="string", default ='0.25', help="step size of the translation search in both directions direction, search is -xr, -xr+ts, 0, xr-ts, xr ")
parser.add_option("--delta", type ="string", default ='2', help="angular step of reference projections")
parser.add_option("--an", type ="string", default ='-1', help="angular neighborhood for local searches")
parser.add_option("--center", type ="int", default =0, help="0 - if you do not want the volume to be centered, 1 - center the volume using cog (default=0)")
parser.add_option("--nassign", type ="int", default =1, help="number of reassignment iterations performed for each angular step (set to 3) ")
parser.add_option("--nrefine", type ="int", default =0, help="number of alignment iterations performed for each angular step (set to 0)")
parser.add_option("--CTF", action ="store_true", default =False, help="do CTF correction during clustring")
parser.add_option("--stoprnct", type ="float", default =3.0, help="Minimum percentage of assignment change to stop the program")
parser.add_option("--sym", type ="string", default ='c1', help="symmetry of the structure ")
parser.add_option("--function", type ="string", default ='do_volume_mrk05', help="name of the reference preparation function")
parser.add_option("--independent", type ="int", default = 3, help="number of independent run")
parser.add_option("--number_of_images_per_group", type ="int", default =1000, help="number of groups")
parser.add_option("--low_pass_filter", type ="float", default =-1.0, help="absolute frequency of low-pass filter for 3d sorting on the original image size" )
parser.add_option("--nxinit", type ="int", default =64, help="initial image size for sorting" )
parser.add_option("--unaccounted", action ="store_true", default =False, help="reconstruct the unaccounted images")
parser.add_option("--seed", type ="int", default =-1, help="random seed for create initial random assignment for EQ Kmeans")
parser.add_option("--smallest_group", type ="int", default =500, help="minimum members for identified group")
parser.add_option("--sausage", action ="store_true", default =False, help="way of filter volume")
parser.add_option("--chunkdir", type ="string", default ='', help="chunkdir for computing margin of error")
parser.add_option("--PWadjustment", type ="string", default ='', help="1-D power spectrum of PDB file used for EM volume power spectrum correction")
parser.add_option("--protein_shape", type ="string", default ='g', help="protein shape. It defines protein preferred orientation angles. Currently it has g and f two types ")
parser.add_option("--upscale", type ="float", default =0.5, help=" scaling parameter to adjust the power spectrum of EM volumes")
parser.add_option("--wn", type ="int", default =0, help="optimal window size for data processing")
parser.add_option("--interpolation", type ="string", default ="4nn", help="3-d reconstruction interpolation method, two options trl and 4nn")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 1 or len(args) > 4:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
else:
if len(args)>2:
mask_file = args[2]
else:
mask_file = None
orgstack =args[0]
masterdir =args[1]
global_def.BATCH = True
#---initialize MPI related variables
from mpi import mpi_init, mpi_comm_size, MPI_COMM_WORLD, mpi_comm_rank,mpi_barrier,mpi_bcast, mpi_bcast, MPI_INT,MPI_CHAR
sys.argv = mpi_init(len(sys.argv),sys.argv)
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
mpi_comm = MPI_COMM_WORLD
main_node= 0
# import some utilities
from utilities import get_im,bcast_number_to_all,cmdexecute,write_text_file,read_text_file,wrap_mpi_bcast, get_params_proj, write_text_row
from applications import recons3d_n_MPI, mref_ali3d_MPI, Kmref_ali3d_MPI
from statistics import k_means_match_clusters_asg_new,k_means_stab_bbenum
from applications import mref_ali3d_EQ_Kmeans, ali3d_mref_Kmeans_MPI
# Create the main log file
from logger import Logger,BaseLogger_Files
if myid ==main_node:
log_main=Logger(BaseLogger_Files())
log_main.prefix = masterdir+"/"
else:
log_main =None
#--- fill input parameters into dictionary named after Constants
Constants ={}
Constants["stack"] = args[0]
Constants["masterdir"] = masterdir
Constants["mask3D"] = mask_file
Constants["focus3Dmask"] = options.focus
Constants["indep_runs"] = options.independent
Constants["stoprnct"] = options.stoprnct
Constants["number_of_images_per_group"] = options.number_of_images_per_group
Constants["CTF"] = options.CTF
Constants["maxit"] = options.maxit
Constants["ir"] = options.ir
Constants["radius"] = options.radius
Constants["nassign"] = options.nassign
Constants["rs"] = options.rs
Constants["xr"] = options.xr
Constants["yr"] = options.yr
Constants["ts"] = options.ts
Constants["delta"] = options.delta
Constants["an"] = options.an
Constants["sym"] = options.sym
Constants["center"] = options.center
Constants["nrefine"] = options.nrefine
#Constants["fourvar"] = options.fourvar
Constants["user_func"] = options.function
Constants["low_pass_filter"] = options.low_pass_filter # enforced low_pass_filter
#Constants["debug"] = options.debug
Constants["main_log_prefix"] = args[1]
#Constants["importali3d"] = options.importali3d
Constants["myid"] = myid
Constants["main_node"] = main_node
Constants["nproc"] = nproc
Constants["log_main"] = log_main
Constants["nxinit"] = options.nxinit
Constants["unaccounted"] = options.unaccounted
Constants["seed"] = options.seed
Constants["smallest_group"] = options.smallest_group
Constants["sausage"] = options.sausage
Constants["chunkdir"] = options.chunkdir
Constants["PWadjustment"] = options.PWadjustment
Constants["upscale"] = options.upscale
Constants["wn"] = options.wn
Constants["3d-interpolation"] = options.interpolation
Constants["protein_shape"] = options.protein_shape
# -----------------------------------------------------
#
# Create and initialize Tracker dictionary with input options
Tracker = {}
Tracker["constants"] = Constants
Tracker["maxit"] = Tracker["constants"]["maxit"]
Tracker["radius"] = Tracker["constants"]["radius"]
#Tracker["xr"] = ""
#Tracker["yr"] = "-1" # Do not change!
#Tracker["ts"] = 1
#Tracker["an"] = "-1"
#Tracker["delta"] = "2.0"
#Tracker["zoom"] = True
#Tracker["nsoft"] = 0
#Tracker["local"] = False
#Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
Tracker["upscale"] = Tracker["constants"]["upscale"]
#Tracker["upscale"] = 0.5
Tracker["applyctf"] = False # Should the data be premultiplied by the CTF. Set to False for local continuous.
#Tracker["refvol"] = None
Tracker["nxinit"] = Tracker["constants"]["nxinit"]
#Tracker["nxstep"] = 32
Tracker["icurrentres"] = -1
#Tracker["ireachedres"] = -1
#Tracker["lowpass"] = 0.4
#Tracker["falloff"] = 0.2
#Tracker["inires"] = options.inires # Now in A, convert to absolute before using
Tracker["fuse_freq"] = 50 # Now in A, convert to absolute before using
#Tracker["delpreviousmax"] = False
#Tracker["anger"] = -1.0
#Tracker["shifter"] = -1.0
#Tracker["saturatecrit"] = 0.95
#Tracker["pixercutoff"] = 2.0
#Tracker["directory"] = ""
#Tracker["previousoutputdir"] = ""
#Tracker["eliminated-outliers"] = False
#Tracker["mainiteration"] = 0
#Tracker["movedback"] = False
#Tracker["state"] = Tracker["constants"]["states"][0]
#Tracker["global_resolution"] =0.0
Tracker["orgstack"] = orgstack
#--------------------------------------------------------------------
# import from utilities
from utilities import sample_down_1D_curve,get_initial_ID,remove_small_groups,print_upper_triangular_matrix,print_a_line_with_timestamp
from utilities import print_dict,get_resolution_mrk01,partition_to_groups,partition_independent_runs,get_outliers
from utilities import merge_groups, save_alist, margin_of_error, get_margin_of_error, do_two_way_comparison, select_two_runs, get_ali3d_params
from utilities import counting_projections, unload_dict, load_dict, get_stat_proj, create_random_list, get_number_of_groups, recons_mref
from utilities import apply_low_pass_filter, get_groups_from_partition, get_number_of_groups, get_complementary_elements_total, update_full_dict
from utilities import count_chunk_members, set_filter_parameters_from_adjusted_fsc, adjust_fsc_down, get_two_chunks_from_stack
####------------------------------------------------------------------
#
# Get the pixel size; if none, set to 1.0, and the original image size
from utilities import get_shrink_data_huang
if(myid == main_node):
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line+"Initialization of 3-D sorting")
a = get_im(orgstack)
nnxo = a.get_xsize()
if( Tracker["nxinit"] > nnxo ):
ERROR("Image size less than minimum permitted $d"%Tracker["nxinit"],"sxsort3d.py",1)
nnxo = -1
else:
if Tracker["constants"]["CTF"]:
i = a.get_attr('ctf')
pixel_size = i.apix
fq = pixel_size/Tracker["fuse_freq"]
else:
pixel_size = 1.0
# No pixel size, fusing computed as 5 Fourier pixels
fq = 5.0/nnxo
del a
else:
nnxo = 0
fq = 0.0
pixel_size = 1.0
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
if( nnxo < 0 ):
mpi_finalize()
exit()
pixel_size = bcast_number_to_all(pixel_size, source_node = main_node)
fq = bcast_number_to_all(fq, source_node = main_node)
if Tracker["constants"]["wn"]==0:
Tracker["constants"]["nnxo"] = nnxo
else:
Tracker["constants"]["nnxo"] = Tracker["constants"]["wn"]
nnxo = Tracker["constants"]["nnxo"]
Tracker["constants"]["pixel_size"] = pixel_size
Tracker["fuse_freq"] = fq
del fq, nnxo, pixel_size
if(Tracker["constants"]["radius"] < 1):
Tracker["constants"]["radius"] = Tracker["constants"]["nnxo"]//2-2
elif((2*Tracker["constants"]["radius"] +2) > Tracker["constants"]["nnxo"]):
ERROR("Particle radius set too large!","sxsort3d.py",1,myid)
####-----------------------------------------------------------------------------------------
# Master directory
if myid == main_node:
if masterdir =="":
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir ="master_sort3d"+timestring
li =len(masterdir)
cmd="{} {}".format("mkdir", masterdir)
os.system(cmd)
else:
li=0
li = mpi_bcast(li,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
if li>0:
masterdir = mpi_bcast(masterdir,li,MPI_CHAR,main_node,MPI_COMM_WORLD)
import string
masterdir = string.join(masterdir,"")
if myid ==main_node:
print_dict(Tracker["constants"],"Permanent settings of 3-D sorting program")
######### create a vstack from input stack to the local stack in masterdir
# stack name set to default
Tracker["constants"]["stack"] = "bdb:"+masterdir+"/rdata"
Tracker["constants"]["ali3d"] = os.path.join(masterdir, "ali3d_init.txt")
Tracker["constants"]["ctf_params"] = os.path.join(masterdir, "ctf_params.txt")
Tracker["constants"]["partstack"] = Tracker["constants"]["ali3d"] # also serves for refinement
if myid == main_node:
total_stack = EMUtil.get_image_count(Tracker["orgstack"])
else:
total_stack = 0
total_stack = bcast_number_to_all(total_stack, source_node = main_node)
mpi_barrier(MPI_COMM_WORLD)
from time import sleep
while not os.path.exists(masterdir):
print "Node ",myid," waiting..."
sleep(5)
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("Sphire sort3d ")
log_main.add("the sort3d master directory is "+masterdir)
#####
###----------------------------------------------------------------------------------
# Initial data analysis and handle two chunk files
from random import shuffle
# Compute the resolution
#### make chunkdir dictionary for computing margin of error
import user_functions
user_func = user_functions.factory[Tracker["constants"]["user_func"]]
chunk_dict = {}
chunk_list = []
if myid == main_node:
chunk_one = read_text_file(os.path.join(Tracker["constants"]["chunkdir"],"chunk0.txt"))
chunk_two = read_text_file(os.path.join(Tracker["constants"]["chunkdir"],"chunk1.txt"))
else:
chunk_one = 0
chunk_two = 0
chunk_one = wrap_mpi_bcast(chunk_one, main_node)
chunk_two = wrap_mpi_bcast(chunk_two, main_node)
mpi_barrier(MPI_COMM_WORLD)
######################## Read/write bdb: data on main node ############################
if myid==main_node:
if(orgstack[:4] == "bdb:"): cmd = "{} {} {}".format("e2bdb.py", orgstack,"--makevstack="+Tracker["constants"]["stack"])
else: cmd = "{} {} {}".format("sxcpy.py", orgstack, Tracker["constants"]["stack"])
cmdexecute(cmd)
cmd = "{} {} {}".format("sxheader.py --params=xform.projection", "--export="+Tracker["constants"]["ali3d"],orgstack)
cmdexecute(cmd)
cmd = "{} {} {}".format("sxheader.py --params=ctf", "--export="+Tracker["constants"]["ctf_params"],orgstack)
cmdexecute(cmd)
mpi_barrier(MPI_COMM_WORLD)
########-----------------------------------------------------------------------------
Tracker["total_stack"] = total_stack
Tracker["constants"]["total_stack"] = total_stack
Tracker["shrinkage"] = float(Tracker["nxinit"])/Tracker["constants"]["nnxo"]
Tracker["radius"] = Tracker["constants"]["radius"]*Tracker["shrinkage"]
if Tracker["constants"]["mask3D"]:
Tracker["mask3D"] = os.path.join(masterdir,"smask.hdf")
else:
Tracker["mask3D"] = None
if Tracker["constants"]["focus3Dmask"]:
Tracker["focus3D"] = os.path.join(masterdir,"sfocus.hdf")
else:
Tracker["focus3D"] = None
if myid == main_node:
if Tracker["constants"]["mask3D"]:
mask_3D = get_shrink_3dmask(Tracker["nxinit"],Tracker["constants"]["mask3D"])
mask_3D.write_image(Tracker["mask3D"])
if Tracker["constants"]["focus3Dmask"]:
mask_3D = get_shrink_3dmask(Tracker["nxinit"],Tracker["constants"]["focus3Dmask"])
st = Util.infomask(mask_3D, None, True)
if( st[0] == 0.0 ): ERROR("sxrsort3d","incorrect focused mask, after binarize all values zero",1)
mask_3D.write_image(Tracker["focus3D"])
del mask_3D
if Tracker["constants"]["PWadjustment"] !='':
PW_dict = {}
nxinit_pwsp = sample_down_1D_curve(Tracker["constants"]["nxinit"],Tracker["constants"]["nnxo"],Tracker["constants"]["PWadjustment"])
Tracker["nxinit_PW"] = os.path.join(masterdir,"spwp.txt")
if myid == main_node: write_text_file(nxinit_pwsp,Tracker["nxinit_PW"])
PW_dict[Tracker["constants"]["nnxo"]] = Tracker["constants"]["PWadjustment"]
PW_dict[Tracker["constants"]["nxinit"]] = Tracker["nxinit_PW"]
Tracker["PW_dict"] = PW_dict
mpi_barrier(MPI_COMM_WORLD)
#-----------------------From two chunks to FSC, and low pass filter-----------------------------------------###
for element in chunk_one: chunk_dict[element] = 0
for element in chunk_two: chunk_dict[element] = 1
chunk_list =[chunk_one, chunk_two]
Tracker["chunk_dict"] = chunk_dict
Tracker["P_chunk0"] = len(chunk_one)/float(total_stack)
Tracker["P_chunk1"] = len(chunk_two)/float(total_stack)
### create two volumes to estimate resolution
if myid == main_node:
for index in xrange(2): write_text_file(chunk_list[index],os.path.join(masterdir,"chunk%01d.txt"%index))
mpi_barrier(MPI_COMM_WORLD)
vols = []
for index in xrange(2):
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nxinit"], os.path.join(masterdir,"chunk%01d.txt"%index), Tracker["constants"]["partstack"],myid,main_node,nproc,preshift=True)
vol = recons3d_4nn_ctf_MPI(myid=myid, prjlist=data,symmetry=Tracker["constants"]["sym"], finfo=None)
if myid == main_node:
vol.write_image(os.path.join(masterdir, "vol%d.hdf"%index))
vols.append(vol)
mpi_barrier(MPI_COMM_WORLD)
if myid ==main_node:
low_pass, falloff,currentres = get_resolution_mrk01(vols,Tracker["constants"]["radius"],Tracker["constants"]["nxinit"],masterdir,Tracker["mask3D"])
if low_pass >Tracker["constants"]["low_pass_filter"]: low_pass= Tracker["constants"]["low_pass_filter"]
else:
low_pass =0.0
falloff =0.0
currentres =0.0
bcast_number_to_all(currentres,source_node = main_node)
bcast_number_to_all(low_pass,source_node = main_node)
bcast_number_to_all(falloff,source_node = main_node)
Tracker["currentres"] = currentres
Tracker["falloff"] = falloff
if Tracker["constants"]["low_pass_filter"] ==-1.0:
Tracker["low_pass_filter"] = min(.45,low_pass/Tracker["shrinkage"]) # no better than .45
else:
Tracker["low_pass_filter"] = min(.45,Tracker["constants"]["low_pass_filter"]/Tracker["shrinkage"])
Tracker["lowpass"] = Tracker["low_pass_filter"]
Tracker["falloff"] =.1
Tracker["global_fsc"] = os.path.join(masterdir, "fsc.txt")
############################################################################################
if myid == main_node:
log_main.add("The command-line inputs are as following:")
log_main.add("**********************************************************")
for a in sys.argv:
if myid == main_node:log_main.add(a)
if myid == main_node:
log_main.add("number of cpus used in this run is %d"%Tracker["constants"]["nproc"])
log_main.add("**********************************************************")
from filter import filt_tanl
### START 3-D sorting
if myid ==main_node:
log_main.add("----------3-D sorting program------- ")
log_main.add("current resolution %6.3f for images of original size in terms of absolute frequency"%Tracker["currentres"])
log_main.add("equivalent to %f Angstrom resolution"%(Tracker["constants"]["pixel_size"]/Tracker["currentres"]/Tracker["shrinkage"]))
log_main.add("the user provided enforced low_pass_filter is %f"%Tracker["constants"]["low_pass_filter"])
#log_main.add("equivalent to %f Angstrom resolution"%(Tracker["constants"]["pixel_size"]/Tracker["constants"]["low_pass_filter"]))
for index in xrange(2):
filt_tanl(get_im(os.path.join(masterdir,"vol%01d.hdf"%index)), Tracker["low_pass_filter"],Tracker["falloff"]).write_image(os.path.join(masterdir, "volf%01d.hdf"%index))
mpi_barrier(MPI_COMM_WORLD)
from utilities import get_input_from_string
delta = get_input_from_string(Tracker["constants"]["delta"])
delta = delta[0]
from utilities import even_angles
n_angles = even_angles(delta, 0, 180)
this_ali3d = Tracker["constants"]["ali3d"]
sampled = get_stat_proj(Tracker,delta,this_ali3d)
if myid ==main_node:
nc = 0
for a in sampled:
if len(sampled[a])>0:
nc += 1
log_main.add("total sampled direction %10d at angle step %6.3f"%(len(n_angles), delta))
log_main.add("captured sampled directions %10d percentage covered by data %6.3f"%(nc,float(nc)/len(n_angles)*100))
number_of_images_per_group = Tracker["constants"]["number_of_images_per_group"]
if myid ==main_node: log_main.add("user provided number_of_images_per_group %d"%number_of_images_per_group)
Tracker["number_of_images_per_group"] = number_of_images_per_group
number_of_groups = get_number_of_groups(total_stack,number_of_images_per_group)
Tracker["number_of_groups"] = number_of_groups
generation =0
partition_dict ={}
full_dict ={}
workdir =os.path.join(masterdir,"generation%03d"%generation)
Tracker["this_dir"] = workdir
if myid ==main_node:
log_main.add("---- generation %5d"%generation)
log_main.add("number of images per group is set as %d"%number_of_images_per_group)
log_main.add("the initial number of groups is %10d "%number_of_groups)
cmd="{} {}".format("mkdir",workdir)
os.system(cmd)
mpi_barrier(MPI_COMM_WORLD)
list_to_be_processed = range(Tracker["constants"]["total_stack"])
Tracker["this_data_list"] = list_to_be_processed
create_random_list(Tracker)
#################################
full_dict ={}
for iptl in xrange(Tracker["constants"]["total_stack"]):
full_dict[iptl] = iptl
Tracker["full_ID_dict"] = full_dict
#################################
for indep_run in xrange(Tracker["constants"]["indep_runs"]):
Tracker["this_particle_list"] = Tracker["this_indep_list"][indep_run]
ref_vol = recons_mref(Tracker)
if myid == main_node: log_main.add("independent run %10d"%indep_run)
mpi_barrier(MPI_COMM_WORLD)
Tracker["this_data_list"] = list_to_be_processed
Tracker["total_stack"] = len(Tracker["this_data_list"])
Tracker["this_particle_text_file"] = os.path.join(workdir,"independent_list_%03d.txt"%indep_run) # for get_shrink_data
if myid == main_node: write_text_file(Tracker["this_data_list"], Tracker["this_particle_text_file"])
mpi_barrier(MPI_COMM_WORLD)
outdir = os.path.join(workdir, "EQ_Kmeans%03d"%indep_run)
ref_vol = apply_low_pass_filter(ref_vol,Tracker)
mref_ali3d_EQ_Kmeans(ref_vol, outdir, Tracker["this_particle_text_file"], Tracker)
partition_dict[indep_run]=Tracker["this_partition"]
Tracker["partition_dict"] = partition_dict
Tracker["total_stack"] = len(Tracker["this_data_list"])
Tracker["this_total_stack"] = Tracker["total_stack"]
###############################
do_two_way_comparison(Tracker)
###############################
ref_vol_list = []
from time import sleep
number_of_ref_class = []
for igrp in xrange(len(Tracker["two_way_stable_member"])):
Tracker["this_data_list"] = Tracker["two_way_stable_member"][igrp]
Tracker["this_data_list_file"] = os.path.join(workdir,"stable_class%d.txt"%igrp)
if myid == main_node:
write_text_file(Tracker["this_data_list"], Tracker["this_data_list_file"])
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["nxinit"], Tracker["this_data_list_file"], Tracker["constants"]["partstack"], myid, main_node, nproc, preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"], finfo = None)
ref_vol_list.append(volref)
number_of_ref_class.append(len(Tracker["this_data_list"]))
if myid == main_node:
log_main.add("group %d members %d "%(igrp,len(Tracker["this_data_list"])))
Tracker["number_of_ref_class"] = number_of_ref_class
nx_of_image = ref_vol_list[0].get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"] # no PW adjustment
if myid == main_node:
for iref in xrange(len(ref_vol_list)):
refdata = [None]*4
refdata[0] = ref_vol_list[iref]
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
volref.write_image(os.path.join(workdir,"volf_stable.hdf"),iref)
mpi_barrier(MPI_COMM_WORLD)
Tracker["this_data_list"] = Tracker["this_accounted_list"]
outdir = os.path.join(workdir,"Kmref")
empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI(ref_vol_list,outdir,Tracker["this_accounted_text"],Tracker)
Tracker["this_unaccounted_list"] = get_complementary_elements(list_to_be_processed,final_list)
if myid == main_node:
log_main.add("the number of particles not processed is %d"%len(Tracker["this_unaccounted_list"]))
write_text_file(Tracker["this_unaccounted_list"],Tracker["this_unaccounted_text"])
update_full_dict(Tracker["this_unaccounted_list"], Tracker)
#######################################
number_of_groups = len(res_groups)
vol_list = []
number_of_ref_class = []
for igrp in xrange(number_of_groups):
data,old_shifts = get_shrink_data_huang(Tracker, Tracker["constants"]["nnxo"], os.path.join(outdir,"Class%d.txt"%igrp), Tracker["constants"]["partstack"],myid,main_node,nproc,preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"], finfo=None)
vol_list.append(volref)
if( myid == main_node ): npergroup = len(read_text_file(os.path.join(outdir,"Class%d.txt"%igrp)))
else: npergroup = 0
npergroup = bcast_number_to_all(npergroup, main_node )
number_of_ref_class.append(npergroup)
Tracker["number_of_ref_class"] = number_of_ref_class
mpi_barrier(MPI_COMM_WORLD)
nx_of_image = vol_list[0].get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"]=Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"]=Tracker["constants"]["PWadjustment"]
if myid == main_node:
for ivol in xrange(len(vol_list)):
refdata =[None]*4
refdata[0] = vol_list[ivol]
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
volref.write_image(os.path.join(workdir,"volf_of_Classes.hdf"),ivol)
log_main.add("number of unaccounted particles %10d"%len(Tracker["this_unaccounted_list"]))
log_main.add("number of accounted particles %10d"%len(Tracker["this_accounted_list"]))
Tracker["this_data_list"] = Tracker["this_unaccounted_list"] # reset parameters for the next round calculation
Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
Tracker["this_total_stack"] = Tracker["total_stack"]
number_of_groups = get_number_of_groups(len(Tracker["this_unaccounted_list"]),number_of_images_per_group)
Tracker["number_of_groups"] = number_of_groups
while number_of_groups >= 2 :
generation +=1
partition_dict ={}
workdir =os.path.join(masterdir,"generation%03d"%generation)
Tracker["this_dir"] = workdir
if myid ==main_node:
log_main.add("*********************************************")
log_main.add("----- generation %5d "%generation)
log_main.add("number of images per group is set as %10d "%number_of_images_per_group)
log_main.add("the number of groups is %10d "%number_of_groups)
log_main.add(" number of particles for clustering is %10d"%Tracker["total_stack"])
cmd ="{} {}".format("mkdir",workdir)
os.system(cmd)
mpi_barrier(MPI_COMM_WORLD)
create_random_list(Tracker)
for indep_run in xrange(Tracker["constants"]["indep_runs"]):
Tracker["this_particle_list"] = Tracker["this_indep_list"][indep_run]
ref_vol = recons_mref(Tracker)
if myid == main_node:
log_main.add("independent run %10d"%indep_run)
outdir = os.path.join(workdir, "EQ_Kmeans%03d"%indep_run)
Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
#ref_vol=apply_low_pass_filter(ref_vol,Tracker)
mref_ali3d_EQ_Kmeans(ref_vol,outdir,Tracker["this_unaccounted_text"],Tracker)
partition_dict[indep_run] = Tracker["this_partition"]
Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
Tracker["partition_dict"] = partition_dict
Tracker["this_total_stack"] = Tracker["total_stack"]
total_list_of_this_run = Tracker["this_unaccounted_list"]
###############################
do_two_way_comparison(Tracker)
###############################
ref_vol_list = []
number_of_ref_class = []
for igrp in xrange(len(Tracker["two_way_stable_member"])):
Tracker["this_data_list"] = Tracker["two_way_stable_member"][igrp]
Tracker["this_data_list_file"] = os.path.join(workdir,"stable_class%d.txt"%igrp)
if myid == main_node: write_text_file(Tracker["this_data_list"], Tracker["this_data_list_file"])
mpi_barrier(MPI_COMM_WORLD)
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nxinit"],Tracker["this_data_list_file"],Tracker["constants"]["partstack"],myid,main_node,nproc,preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"],finfo= None)
#volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1)
if myid == main_node:volref.write_image(os.path.join(workdir,"vol_stable.hdf"),iref)
#volref = resample(volref,Tracker["shrinkage"])
ref_vol_list.append(volref)
number_of_ref_class.append(len(Tracker["this_data_list"]))
mpi_barrier(MPI_COMM_WORLD)
Tracker["number_of_ref_class"] = number_of_ref_class
Tracker["this_data_list"] = Tracker["this_accounted_list"]
outdir = os.path.join(workdir,"Kmref")
empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI(ref_vol_list,outdir,Tracker["this_accounted_text"],Tracker)
# calculate the 3-D structure of original image size for each group
number_of_groups = len(res_groups)
Tracker["this_unaccounted_list"] = get_complementary_elements(total_list_of_this_run,final_list)
if myid == main_node:
log_main.add("the number of particles not processed is %d"%len(Tracker["this_unaccounted_list"]))
write_text_file(Tracker["this_unaccounted_list"],Tracker["this_unaccounted_text"])
mpi_barrier(MPI_COMM_WORLD)
update_full_dict(Tracker["this_unaccounted_list"],Tracker)
vol_list = []
for igrp in xrange(number_of_groups):
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nnxo"], os.path.join(outdir,"Class%d.txt"%igrp), Tracker["constants"]["partstack"], myid, main_node, nproc,preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"],finfo= None)
vol_list.append(volref)
mpi_barrier(MPI_COMM_WORLD)
nx_of_image=ref_vol_list[0].get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
if myid == main_node:
for ivol in xrange(len(vol_list)):
refdata = [None]*4
refdata[0] = vol_list[ivol]
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
volref.write_image(os.path.join(workdir, "volf_of_Classes.hdf"),ivol)
log_main.add("number of unaccounted particles %10d"%len(Tracker["this_unaccounted_list"]))
log_main.add("number of accounted particles %10d"%len(Tracker["this_accounted_list"]))
del vol_list
mpi_barrier(MPI_COMM_WORLD)
number_of_groups = get_number_of_groups(len(Tracker["this_unaccounted_list"]),number_of_images_per_group)
Tracker["number_of_groups"] = number_of_groups
Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
if Tracker["constants"]["unaccounted"]:
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nnxo"],Tracker["this_unaccounted_text"],Tracker["constants"]["partstack"],myid,main_node,nproc,preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"],finfo= None)
nx_of_image = volref.get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"]=Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"]=Tracker["constants"]["PWadjustment"]
if( myid == main_node ):
refdata = [None]*4
refdata[0] = volref
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
#volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1)
volref.write_image(os.path.join(workdir,"volf_unaccounted.hdf"))
# Finish program
if myid ==main_node: log_main.add("sxsort3d finishes")
mpi_barrier(MPI_COMM_WORLD)
from mpi import mpi_finalize
mpi_finalize()
exit()
def identify_outliers(myid, main_node, rviper_iter, no_of_viper_runs_analyzed_together,
no_of_viper_runs_analyzed_together_from_user_options, masterdir, bdb_stack_location, outlier_percentile,
criterion_name, outlier_index_threshold_method, angle_threshold):
no_of_viper_runs_analyzed_together_must_be_incremented = 0
do_calculation = 1
if (myid == main_node):
mainoutputdir = masterdir + DIR_DELIM + NAME_OF_MAIN_DIR + ("%03d" + DIR_DELIM) % (rviper_iter)
if(os.path.exists(mainoutputdir + DIR_DELIM + "list_of_viper_runs_included_in_outlier_elimination.json")):
# list_of_independent_viper_run_indices_used_for_outlier_elimination = map(int, read_text_file(mainoutputdir + DIR_DELIM + "list_of_viper_runs_included_in_outlier_elimination.txt"))
import json; f = open(mainoutputdir + "list_of_viper_runs_included_in_outlier_elimination.json", 'r')
list_of_independent_viper_run_indices_used_for_outlier_elimination = json.load(f); f.close()
do_calculation = 0
do_calculation = mpi_bcast(do_calculation, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
else:
do_calculation = mpi_bcast(do_calculation, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
if do_calculation:
list_of_independent_viper_run_indices_used_for_outlier_elimination = calculate_list_of_independent_viper_run_indices_used_for_outlier_elimination(no_of_viper_runs_analyzed_together,
no_of_viper_runs_analyzed_together_from_user_options, masterdir, rviper_iter, criterion_name)
# only master has the actual list: list_of_independent_viper_run_indices_used_for_outlier_elimination
# only master has the actual list: list_of_independent_viper_run_indices_used_for_outlier_elimination
# only master has the actual list: list_of_independent_viper_run_indices_used_for_outlier_elimination
error_status = 0
if (myid == main_node):
# if len(list_of_independent_viper_run_indices_used_for_outlier_elimination) == 0:
if list_of_independent_viper_run_indices_used_for_outlier_elimination[0] == EMPTY_VIPER_RUN_INDICES_LIST:
if no_of_viper_runs_analyzed_together > MAXIMUM_NO_OF_VIPER_RUNS_ANALYZED_TOGETHER:
error_status = 1
print "RVIPER reached maximum number of VIPER runs analyzed together without finding a core set of stable projections for the current RVIPER iteration (%d)! Finishing."%rviper_iter
cmd = "{} {}".format("mkdir ", masterdir + "MAXIMUM_NO_OF_VIPER_RUNS_ANALYZED_TOGETHER__Reached"); cmdexecute(cmd)
else:
# No set of solutions has been found to make a selection for outlier elimination.
# A new independent viper run will be performed
no_of_viper_runs_analyzed_together_must_be_incremented = 1
cmd = "{} {}".format("rm ", mainoutputdir + "list_of_viper_runs_included_in_outlier_elimination.json"); cmdexecute(cmd)
else:
# Outliers are eliminated based on the viper runs contained in "list_of_independent_viper_run_indices_used_for_outlier_elimination"
if list_of_independent_viper_run_indices_used_for_outlier_elimination[0] == MUST_END_PROGRAM_THIS_ITERATION:
no_of_viper_runs_analyzed_together_must_be_incremented = MUST_END_PROGRAM_THIS_ITERATION
found_outliers(list_of_independent_viper_run_indices_used_for_outlier_elimination[1:], outlier_percentile,
rviper_iter, masterdir, bdb_stack_location, "use all images", angle_threshold)
else:
# still need to eliminate DUMMY_INDEX_USED_AS_BUFFER
found_outliers(list_of_independent_viper_run_indices_used_for_outlier_elimination[1:], outlier_percentile,
rviper_iter, masterdir, bdb_stack_location, outlier_index_threshold_method, angle_threshold)
if_error_then_all_processes_exit_program(error_status)
no_of_viper_runs_analyzed_together_must_be_incremented = mpi_bcast(no_of_viper_runs_analyzed_together_must_be_incremented, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
return no_of_viper_runs_analyzed_together_must_be_incremented
def main():
from logger import Logger, BaseLogger_Files
import user_functions
from optparse import OptionParser, SUPPRESS_HELP
from global_def import SPARXVERSION
from EMAN2 import EMData
main_node = 0
mpi_init(0, [])
mpi_comm = MPI_COMM_WORLD
myid = mpi_comm_rank(MPI_COMM_WORLD)
mpi_size = mpi_comm_size(MPI_COMM_WORLD) # Total number of processes, passed by --np option.
# mpi_barrier(mpi_comm)
# from mpi import mpi_finalize
# mpi_finalize()
# print "mpi finalize"
# from sys import exit
# exit()
progname = os.path.basename(sys.argv[0])
usage = progname + " stack [output_directory] --ir=inner_radius --radius=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --center=center_type --maxit1=max_iter1 --maxit2=max_iter2 --L2threshold=0.1 --fl --aa --ref_a=S --sym=c1"
usage += """
stack 2D images in a stack file: (default required string)
output_directory: directory name into which the output files will be written. If it does not exist, the directory will be created. If it does exist, the program will continue executing from where it stopped (if it did not already reach the end). The "--use_latest_master_directory" option can be used to choose the most recent directory that starts with "master".
"""
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--radius", type="int", help="radius of the particle: has to be less than < int(nx/2)-1 (default required int)")
parser.add_option("--ir", type="int", default=1, help="inner radius for rotational search: > 0 (default 1)")
parser.add_option("--rs", type="int", default=1, help="step between rings in rotational search: >0 (default 1)")
parser.add_option("--xr", type="string", default='0', help="range for translation search in x direction: search is +/xr in pixels (default '0')")
parser.add_option("--yr", type="string", default='0', help="range for translation search in y direction: if omitted will be set to xr, search is +/yr in pixels (default '0')")
parser.add_option("--ts", type="string", default='1.0', help="step size of the translation search in x-y directions: search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default '1.0')")
parser.add_option("--delta", type="string", default='2.0', help="angular step of reference projections: (default '2.0')")
#parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches (phi and theta)")
parser.add_option("--center", type="float", default=-1.0, help="centering of 3D template: average shift method; 0: no centering; 1: center of gravity (default -1.0)")
parser.add_option("--maxit1", type="int", default=400, help="maximum number of iterations performed for the GA part: (default 400)")
parser.add_option("--maxit2", type="int", default=50, help="maximum number of iterations performed for the finishing up part: (default 50)")
parser.add_option("--L2threshold", type="float", default=0.03, help="stopping criterion of GA: given as a maximum relative dispersion of volumes' L2 norms: (default 0.03)")
parser.add_option("--doga", type="float", default=0.1, help="do GA when fraction of orientation changes less than 1.0 degrees is at least doga: (default 0.1)")
parser.add_option("--n_shc_runs", type="int", default=4, help="number of quasi-independent shc runs (same as '--nruns' parameter from sxviper.py): (default 4)")
parser.add_option("--n_rv_runs", type="int", default=10, help="number of rviper iterations: (default 10)")
parser.add_option("--n_v_runs", type="int", default=3, help="number of viper runs for each r_viper cycle: (default 3)")
parser.add_option("--outlier_percentile", type="float", default=95.0, help="percentile above which outliers are removed every rviper iteration: (default 95.0)")
parser.add_option("--iteration_start", type="int", default=0, help="starting iteration for rviper: 0 means go to the most recent one (default 0)")
#parser.add_option("--CTF", action="store_true", default=False, help="NOT IMPLEMENTED Consider CTF correction during the alignment ")
#parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data (default 1.0)")
parser.add_option("--ref_a", type="string", default='S', help="method for generating the quasi-uniformly distributed projection directions: (default S)")
parser.add_option("--sym", type="string", default='c1', help="point-group symmetry of the structure: (default c1)")
# parser.add_option("--function", type="string", default="ref_ali3d", help="name of the reference preparation function (ref_ali3d by default)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--function", type="string", default="ref_ali3d", help=SUPPRESS_HELP)
parser.add_option("--npad", type="int", default=2, help="padding size for 3D reconstruction: (default 2)")
# parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default 2)")
#options introduced for the do_volume function
parser.add_option("--fl", type="float", default=0.25, help="cut-off frequency applied to the template volume: using a hyperbolic tangent low-pass filter (default 0.25)")
parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass filter: (default 0.1)")
parser.add_option("--pwreference", type="string", default='', help="text file with a reference power spectrum: (default none)")
parser.add_option("--mask3D", type="string", default=None, help="3D mask file: (default sphere)")
parser.add_option("--moon_elimination", type="string", default='', help="elimination of disconnected pieces: two arguments: mass in KDa and pixel size in px/A separated by comma, no space (default none)")
# used for debugging, help is supressed with SUPPRESS_HELP
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--my_random_seed", type="int", default=123, help = SUPPRESS_HELP)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--run_get_already_processed_viper_runs", action="store_true", dest="run_get_already_processed_viper_runs", default=False, help = SUPPRESS_HELP)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False, help = SUPPRESS_HELP)
parser.add_option("--criterion_name", type="string", default='80th percentile',help="criterion deciding if volumes have a core set of stable projections: '80th percentile', other options:'fastest increase in the last quartile' (default '80th percentile')")
parser.add_option("--outlier_index_threshold_method",type="string", default='discontinuity_in_derivative',help="method that decides which images to keep: discontinuity_in_derivative, other options:percentile, angle_measure (default discontinuity_in_derivative)")
parser.add_option("--angle_threshold", type="int", default=30, help="angle threshold for projection removal if using 'angle_measure': (default 30)")
required_option_list = ['radius']
(options, args) = parser.parse_args(sys.argv[1:])
options.CTF = False
options.snr = 1.0
options.an = -1
if options.moon_elimination == "":
options.moon_elimination = []
else:
options.moon_elimination = map(float, options.moon_elimination.split(","))
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
print "\n ==%s== mandatory option is missing.\n"%required_option
print "Please run '" + progname + " -h' for detailed options"
return 1
mpi_barrier(MPI_COMM_WORLD)
if(myid == main_node):
print "****************************************************************"
Util.version()
print "****************************************************************"
sys.stdout.flush()
mpi_barrier(MPI_COMM_WORLD)
# this is just for benefiting from a user friendly parameter name
options.ou = options.radius
my_random_seed = options.my_random_seed
criterion_name = options.criterion_name
outlier_index_threshold_method = options.outlier_index_threshold_method
use_latest_master_directory = options.use_latest_master_directory
iteration_start_default = options.iteration_start
number_of_rrr_viper_runs = options.n_rv_runs
no_of_viper_runs_analyzed_together_from_user_options = options.n_v_runs
no_of_shc_runs_analyzed_together = options.n_shc_runs
outlier_percentile = options.outlier_percentile
angle_threshold = options.angle_threshold
run_get_already_processed_viper_runs = options.run_get_already_processed_viper_runs
get_already_processed_viper_runs(run_get_already_processed_viper_runs)
import random
random.seed(my_random_seed)
if len(args) < 1 or len(args) > 3:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
return 1
# if len(args) > 2:
# ref_vol = get_im(args[2])
# else:
ref_vol = None
# error_status = None
# if myid == 0:
# number_of_images = EMUtil.get_image_count(args[0])
# if mpi_size > number_of_images:
# error_status = ('Number of processes supplied by --np in mpirun needs to be less than or equal to %d (total number of images) ' % number_of_images, getframeinfo(currentframe()))
# if_error_then_all_processes_exit_program(error_status)
bdb_stack_location = ""
masterdir = ""
if len(args) == 2:
masterdir = args[1]
if masterdir[-1] != DIR_DELIM:
masterdir += DIR_DELIM
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re; r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs)>0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
masterdir += DIR_DELIM
log = Logger(BaseLogger_Files())
error_status = 0
if mpi_size % no_of_shc_runs_analyzed_together != 0:
ERROR('Number of processes needs to be a multiple of the number of quasi-independent runs (shc) within each viper run. '
'Total quasi-independent runs by default are 3, you can change it by specifying '
'--n_shc_runs option (in sxviper this option is called --nruns). Also, to improve communication time it is recommended that '
'the number of processes divided by the number of quasi-independent runs is a power '
'of 2 (e.g. 2, 4, 8 or 16 depending on how many physical cores each node has).', 'sxviper', 1)
error_status = 1
if_error_then_all_processes_exit_program(error_status)
#Create folder for all results or check if there is one created already
if(myid == main_node):
#cmd = "{}".format("Rmycounter ccc")
#cmdexecute(cmd)
if( masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime())
masterdir = "master"+timestring
if not os.path.exists(masterdir):
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
if ':' in args[0]:
bdb_stack_location = args[0].split(":")[0] + ":" + masterdir + args[0].split(":")[1]
org_stack_location = args[0]
if(not os.path.exists(os.path.join(masterdir,"EMAN2DB" + DIR_DELIM))):
# cmd = "{} {}".format("cp -rp EMAN2DB", masterdir, "EMAN2DB" DIR_DELIM)
# cmdexecute(cmd)
cmd = "{} {} {}".format("e2bdb.py", org_stack_location,"--makevstack=" + bdb_stack_location + "_000")
cmdexecute(cmd)
from applications import header
try:
header(bdb_stack_location + "_000", params='original_image_index', fprint=True)
print "Images were already indexed!"
except KeyError:
print "Indexing images"
header(bdb_stack_location + "_000", params='original_image_index', consecutive=True)
else:
filename = os.path.basename(args[0])
bdb_stack_location = "bdb:" + masterdir + os.path.splitext(filename)[0]
if(not os.path.exists(os.path.join(masterdir,"EMAN2DB" + DIR_DELIM))):
cmd = "{} {} {}".format("sxcpy.py ", args[0], bdb_stack_location + "_000")
cmdexecute(cmd)
from applications import header
try:
header(bdb_stack_location + "_000", params='original_image_index', fprint=True)
print "Images were already indexed!"
except KeyError:
print "Indexing images"
header(bdb_stack_location + "_000", params='original_image_index', consecutive=True)
# send masterdir to all processes
dir_len = len(masterdir)*int(myid == main_node)
dir_len = mpi_bcast(dir_len,1,MPI_INT,0,MPI_COMM_WORLD)[0]
masterdir = mpi_bcast(masterdir,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD)
masterdir = string.join(masterdir,"")
if masterdir[-1] != DIR_DELIM:
masterdir += DIR_DELIM
global_def.LOGFILE = os.path.join(masterdir, global_def.LOGFILE)
print_program_start_information()
# mpi_barrier(mpi_comm)
# from mpi import mpi_finalize
# mpi_finalize()
# print "mpi finalize"
# from sys import exit
# exit()
# send bdb_stack_location to all processes
dir_len = len(bdb_stack_location)*int(myid == main_node)
dir_len = mpi_bcast(dir_len,1,MPI_INT,0,MPI_COMM_WORLD)[0]
bdb_stack_location = mpi_bcast(bdb_stack_location,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD)
bdb_stack_location = string.join(bdb_stack_location,"")
iteration_start = get_latest_directory_increment_value(masterdir, "main")
if (myid == main_node):
if (iteration_start < iteration_start_default):
ERROR('Starting iteration provided is greater than last iteration performed. Quiting program', 'sxviper', 1)
error_status = 1
if iteration_start_default!=0:
iteration_start = iteration_start_default
if (myid == main_node):
if (number_of_rrr_viper_runs < iteration_start):
ERROR('Please provide number of rviper runs (--n_rv_runs) greater than number of iterations already performed.', 'sxviper', 1)
error_status = 1
if_error_then_all_processes_exit_program(error_status)
for rviper_iter in range(iteration_start, number_of_rrr_viper_runs + 1):
if(myid == main_node):
all_projs = EMData.read_images(bdb_stack_location + "_%03d"%(rviper_iter - 1))
print "XXXXXXXXXXXXXXXXX"
print "Number of projections (in loop): " + str(len(all_projs))
print "XXXXXXXXXXXXXXXXX"
subset = range(len(all_projs))
else:
all_projs = None
subset = None
runs_iter = get_latest_directory_increment_value(masterdir + NAME_OF_MAIN_DIR + "%03d"%rviper_iter, DIR_DELIM + NAME_OF_RUN_DIR, start_value=0) - 1
no_of_viper_runs_analyzed_together = max(runs_iter + 2, no_of_viper_runs_analyzed_together_from_user_options)
first_time_entering_the_loop_need_to_do_full_check_up = True
while True:
runs_iter += 1
if not first_time_entering_the_loop_need_to_do_full_check_up:
if runs_iter >= no_of_viper_runs_analyzed_together:
break
first_time_entering_the_loop_need_to_do_full_check_up = False
this_run_is_NOT_complete = 0
if (myid == main_node):
independent_run_dir = masterdir + DIR_DELIM + NAME_OF_MAIN_DIR + ('%03d' + DIR_DELIM + NAME_OF_RUN_DIR + "%03d" + DIR_DELIM)%(rviper_iter, runs_iter)
if run_get_already_processed_viper_runs:
cmd = "{} {}".format("mkdir -p", masterdir + DIR_DELIM + NAME_OF_MAIN_DIR + ('%03d' + DIR_DELIM)%(rviper_iter)); cmdexecute(cmd)
cmd = "{} {}".format("rm -rf", independent_run_dir); cmdexecute(cmd)
cmd = "{} {}".format("cp -r", get_already_processed_viper_runs() + " " + independent_run_dir); cmdexecute(cmd)
if os.path.exists(independent_run_dir + "log.txt") and (string_found_in_file("Finish VIPER2", independent_run_dir + "log.txt")):
this_run_is_NOT_complete = 0
else:
this_run_is_NOT_complete = 1
cmd = "{} {}".format("rm -rf", independent_run_dir); cmdexecute(cmd)
cmd = "{} {}".format("mkdir -p", independent_run_dir); cmdexecute(cmd)
this_run_is_NOT_complete = mpi_bcast(this_run_is_NOT_complete,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
dir_len = len(independent_run_dir)
dir_len = mpi_bcast(dir_len,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
independent_run_dir = mpi_bcast(independent_run_dir,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD)
independent_run_dir = string.join(independent_run_dir,"")
else:
this_run_is_NOT_complete = mpi_bcast(this_run_is_NOT_complete,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
dir_len = 0
independent_run_dir = ""
dir_len = mpi_bcast(dir_len,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
independent_run_dir = mpi_bcast(independent_run_dir,dir_len,MPI_CHAR,main_node,MPI_COMM_WORLD)
independent_run_dir = string.join(independent_run_dir,"")
if this_run_is_NOT_complete:
mpi_barrier(MPI_COMM_WORLD)
if independent_run_dir[-1] != DIR_DELIM:
independent_run_dir += DIR_DELIM
log.prefix = independent_run_dir
options.user_func = user_functions.factory[options.function]
# for debugging purposes
#if (myid == main_node):
#cmd = "{} {}".format("cp ~/log.txt ", independent_run_dir)
#cmdexecute(cmd)
#cmd = "{} {}{}".format("cp ~/paramdir/params$(mycounter ccc).txt ", independent_run_dir, "param%03d.txt"%runs_iter)
#cmd = "{} {}{}".format("cp ~/paramdir/params$(mycounter ccc).txt ", independent_run_dir, "params.txt")
#cmdexecute(cmd)
if (myid == main_node):
store_value_of_simple_vars_in_json_file(masterdir + 'program_state_stack.json', locals(), exclude_list_of_vars=["usage"],
vars_that_will_show_only_size = ["subset"])
store_value_of_simple_vars_in_json_file(masterdir + 'program_state_stack.json', options.__dict__, write_or_append='a')
# mpi_barrier(mpi_comm)
# from mpi import mpi_finalize
# mpi_finalize()
# print "mpi finalize"
# from sys import exit
# exit()
out_params, out_vol, out_peaks = multi_shc(all_projs, subset, no_of_shc_runs_analyzed_together, options,
mpi_comm=mpi_comm, log=log, ref_vol=ref_vol)
# end of: if this_run_is_NOT_complete:
if runs_iter >= (no_of_viper_runs_analyzed_together_from_user_options - 1):
increment_for_current_iteration = identify_outliers(myid, main_node, rviper_iter,
no_of_viper_runs_analyzed_together, no_of_viper_runs_analyzed_together_from_user_options, masterdir,
bdb_stack_location, outlier_percentile, criterion_name, outlier_index_threshold_method, angle_threshold)
if increment_for_current_iteration == MUST_END_PROGRAM_THIS_ITERATION:
break
no_of_viper_runs_analyzed_together += increment_for_current_iteration
# end of independent viper loop
calculate_volumes_after_rotation_and_save_them(options, rviper_iter, masterdir, bdb_stack_location, myid,
mpi_size, no_of_viper_runs_analyzed_together, no_of_viper_runs_analyzed_together_from_user_options)
if increment_for_current_iteration == MUST_END_PROGRAM_THIS_ITERATION:
if (myid == main_node):
print "RVIPER found a core set of stable projections for the current RVIPER iteration (%d), the maximum angle difference between corresponding projections from different VIPER volumes is less than %.2f. Finishing."%(rviper_iter, ANGLE_ERROR_THRESHOLD)
break
else:
if (myid == main_node):
print "After running the last iteration (%d), RVIPER did not find a set of projections with the maximum angle difference between corresponding projections from different VIPER volumes less than %.2f Finishing."%(rviper_iter, ANGLE_ERROR_THRESHOLD)
# end of RVIPER loop
#mpi_finalize()
#sys.exit()
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def found_outliers(list_of_projection_indices, outlier_percentile, rviper_iter, masterdir, bdb_stack_location,
outlier_index_threshold_method, angle_threshold):
# sxheader.py bdb:nj --consecutive --params=OID
import numpy as np
mainoutputdir = masterdir + DIR_DELIM + NAME_OF_MAIN_DIR + ("%03d" + DIR_DELIM) %(rviper_iter)
# if this data analysis step was already performed in the past then return
for check_run in list_of_projection_indices:
if not (os.path.exists(mainoutputdir + DIR_DELIM + NAME_OF_RUN_DIR + "%03d"%(check_run) + DIR_DELIM + "rotated_reduced_params.txt")):
break
else:
return
print "identify_outliers"
projs = []
for i1 in list_of_projection_indices:
projs.append(read_text_row(mainoutputdir + NAME_OF_RUN_DIR + "%03d"%(i1) + DIR_DELIM + "params.txt"))
# ti1, ti3, out = find_common_subset(projs, 1.0)
subset, avg_diff_per_image, rotated_params = find_common_subset(projs, target_threshold = 0)
# subset, avg_diff_per_image, rotated_params = find_common_subset(projs, target_threshold = 1.0)
error_values_and_indices = []
for i in xrange(len(avg_diff_per_image)):
error_values_and_indices.append([avg_diff_per_image[i], i])
del subset, avg_diff_per_image
error_values_and_indices.sort()
if outlier_index_threshold_method == "discontinuity_in_derivative":
outlier_index_threshold = find_index_of_discontinuity_in_derivative([i[0] for i in error_values_and_indices],
list_of_projection_indices, mainoutputdir, outlier_percentile)
elif outlier_index_threshold_method == "percentile":
outlier_index_threshold = outlier_percentile * (len(error_values_and_indices) - 1)/ 100.0
elif outlier_index_threshold_method == "angle_measure":
error_values = [i[0] for i in error_values_and_indices]
outlier_index_threshold = min(range(len(error_values)), key=lambda i: abs(error_values[i]-angle_threshold))
elif outlier_index_threshold_method == "use all images":
outlier_index_threshold = len(error_values_and_indices)
index_keep_images = [i[1] for i in error_values_and_indices[:outlier_index_threshold]]
index_outliers = [i[1] for i in error_values_and_indices[outlier_index_threshold:]]
# print "error_values_and_indices: %f"%error_values_and_indices
print "index_outliers: ", index_outliers
import copy
reversed_sorted_index_outliers = copy.deepcopy(index_outliers)
reversed_sorted_index_outliers.sort(reverse=True)
for k in xrange(len(projs)):
for l in reversed_sorted_index_outliers:
del rotated_params[k][l]
index_outliers.sort()
index_keep_images.sort()
write_text_file(index_outliers, mainoutputdir + "this_iteration_index_outliers.txt")
write_text_file(index_keep_images, mainoutputdir + "this_iteration_index_keep_images.txt")
#if len(index_outliers) < 3:
#return False
if len(index_outliers) > 0:
cmd = "{} {} {} {}".format("e2bdb.py ", bdb_stack_location + "_%03d"%(rviper_iter - 1), "--makevstack=" + bdb_stack_location + "_outliers_%03d"%(rviper_iter), "--list=" + mainoutputdir + "this_iteration_index_outliers.txt")
cmdexecute(cmd)
cmd = "{} {} {} {}".format("e2bdb.py ", bdb_stack_location + "_%03d"%(rviper_iter - 1), "--makevstack=" + bdb_stack_location + "_%03d"%(rviper_iter), "--list=" + mainoutputdir + "this_iteration_index_keep_images.txt")
cmdexecute(cmd)
dat = EMData.read_images(bdb_stack_location + "_%03d"%(rviper_iter - 1))
write_text_file([dat[i].get_attr("original_image_index") for i in index_outliers],mainoutputdir + "index_outliers.txt")
write_text_file([dat[i].get_attr("original_image_index") for i in index_keep_images],mainoutputdir + "index_keep_images.txt")
print "index_outliers:: " + str(index_outliers)
# write rotated param files
for i1 in range(len(list_of_projection_indices)):
write_text_row(rotated_params[i1], mainoutputdir + NAME_OF_RUN_DIR + "%03d"%(list_of_projection_indices[i1]) + DIR_DELIM + "rotated_reduced_params.txt")
return True
def main():
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from logger import Logger, BaseLogger_Files
import sys, os, time
global Tracker, Blockdata
from global_def import ERROR
progname = os.path.basename(sys.argv[0])
usage = progname + " --output_dir=output_dir --isac_dir=output_dir_of_isac "
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--pw_adjustment", type ="string", default ='analytical_model', \
help="adjust power spectrum of 2-D averages to an analytic model. Other opions: no_adjustment; bfactor; a text file of 1D rotationally averaged PW")
#### Four options for --pw_adjustment:
# 1> analytical_model(default);
# 2> no_adjustment;
# 3> bfactor;
# 4> adjust_to_given_pw2(user has to provide a text file that contains 1D rotationally averaged PW)
# options in common
parser.add_option(
"--isac_dir",
type="string",
default='',
help="ISAC run output directory, input directory for this command")
parser.add_option(
"--output_dir",
type="string",
default='',
help="output directory where computed averages are saved")
parser.add_option(
"--pixel_size",
type="float",
default=-1.0,
help=
"pixel_size of raw images. one can put 1.0 in case of negative stain data"
)
parser.add_option(
"--fl",
type="float",
default=-1.0,
help=
"low pass filter, = -1.0, not applied; =0.0, using FH1 (initial resolution), = 1.0 using FH2 (resolution after local alignment), or user provided value in absolute freqency [0.0:0.5]"
)
parser.add_option("--stack",
type="string",
default="",
help="data stack used in ISAC")
parser.add_option("--radius", type="int", default=-1, help="radius")
parser.add_option("--xr",
type="float",
default=-1.0,
help="local alignment search range")
#parser.add_option("--ts", type ="float", default =1.0, help= "local alignment search step")
parser.add_option("--fh",
type="float",
default=-1.0,
help="local alignment high frequencies limit")
#parser.add_option("--maxit", type ="int", default =5, help= "local alignment iterations")
parser.add_option("--navg",
type="int",
default=1000000,
help="number of aveages")
parser.add_option("--local_alignment",
action="store_true",
default=False,
help="do local alignment")
parser.add_option(
"--noctf",
action="store_true",
default=False,
help=
"no ctf correction, useful for negative stained data. always ctf for cryo data"
)
parser.add_option(
"--B_start",
type="float",
default=45.0,
help=
"start frequency (Angstrom) of power spectrum for B_factor estimation")
parser.add_option(
"--Bfactor",
type="float",
default=-1.0,
help=
"User defined bactors (e.g. 25.0[A^2]). By default, the program automatically estimates B-factor. "
)
(options, args) = parser.parse_args(sys.argv[1:])
adjust_to_analytic_model = False
adjust_to_given_pw2 = False
B_enhance = False
no_adjustment = False
if options.pw_adjustment == 'analytical_model':
adjust_to_analytic_model = True
elif options.pw_adjustment == 'no_adjustment':
no_adjustment = True
elif options.pw_adjustment == 'bfactor':
B_enhance = True
else:
adjust_to_given_pw2 = True
from utilities import get_im, bcast_number_to_all, write_text_file, read_text_file, wrap_mpi_bcast, write_text_row
from utilities import cmdexecute
from filter import filt_tanl
from logger import Logger, BaseLogger_Files
import user_functions
import string
from string import split, atoi, atof
import json
mpi_init(0, [])
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
Blockdata = {}
# MPI stuff
Blockdata["nproc"] = nproc
Blockdata["myid"] = myid
Blockdata["main_node"] = 0
Blockdata["shared_comm"] = mpi_comm_split_type(MPI_COMM_WORLD,
MPI_COMM_TYPE_SHARED, 0,
MPI_INFO_NULL)
Blockdata["myid_on_node"] = mpi_comm_rank(Blockdata["shared_comm"])
Blockdata["no_of_processes_per_group"] = mpi_comm_size(
Blockdata["shared_comm"])
masters_from_groups_vs_everything_else_comm = mpi_comm_split(
MPI_COMM_WORLD, Blockdata["main_node"] == Blockdata["myid_on_node"],
Blockdata["myid_on_node"])
Blockdata["color"], Blockdata["no_of_groups"], balanced_processor_load_on_nodes = get_colors_and_subsets(Blockdata["main_node"], MPI_COMM_WORLD, Blockdata["myid"], \
Blockdata["shared_comm"], Blockdata["myid_on_node"], masters_from_groups_vs_everything_else_comm)
# We need two nodes for processing of volumes
Blockdata["node_volume"] = [
Blockdata["no_of_groups"] - 3, Blockdata["no_of_groups"] - 2,
Blockdata["no_of_groups"] - 1
] # For 3D stuff take three last nodes
# We need two CPUs for processing of volumes, they are taken to be main CPUs on each volume
# We have to send the two myids to all nodes so we can identify main nodes on two selected groups.
Blockdata["nodes"] = [Blockdata["node_volume"][0]*Blockdata["no_of_processes_per_group"],Blockdata["node_volume"][1]*Blockdata["no_of_processes_per_group"], \
Blockdata["node_volume"][2]*Blockdata["no_of_processes_per_group"]]
# End of Blockdata: sorting requires at least three nodes, and the used number of nodes be integer times of three
global_def.BATCH = True
global_def.MPI = True
if adjust_to_given_pw2:
checking_flag = 0
if (Blockdata["myid"] == Blockdata["main_node"]):
if not os.path.exists(options.pw_adjustment): checking_flag = 1
checking_flag = bcast_number_to_all(checking_flag,
Blockdata["main_node"],
MPI_COMM_WORLD)
if checking_flag == 1:
ERROR("User provided power spectrum does not exist",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
Tracker = {}
Constants = {}
Constants["isac_dir"] = options.isac_dir
Constants["masterdir"] = options.output_dir
Constants["pixel_size"] = options.pixel_size
Constants["orgstack"] = options.stack
Constants["radius"] = options.radius
Constants["xrange"] = options.xr
Constants["FH"] = options.fh
Constants["low_pass_filter"] = options.fl
#Constants["maxit"] = options.maxit
Constants["navg"] = options.navg
Constants["B_start"] = options.B_start
Constants["Bfactor"] = options.Bfactor
if adjust_to_given_pw2: Constants["modelpw"] = options.pw_adjustment
Tracker["constants"] = Constants
# -------------------------------------------------------------
#
# Create and initialize Tracker dictionary with input options # State Variables
#<<<---------------------->>>imported functions<<<---------------------------------------------
#x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1)
#y_range = x_range
####-----------------------------------------------------------
# Create Master directory and associated subdirectories
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]:
masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen")
else:
masterdir = Tracker["constants"]["masterdir"]
if (Blockdata["myid"] == Blockdata["main_node"]):
msg = "Postprocessing ISAC 2D averages starts"
print(line, "Postprocessing ISAC 2D averages starts")
if not masterdir:
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "sharpen_" + Tracker["constants"]["isac_dir"]
os.mkdir(masterdir)
else:
if os.path.exists(masterdir):
print("%s already exists" % masterdir)
else:
os.mkdir(masterdir)
subdir_path = os.path.join(masterdir, "ali2d_local_params_avg")
if not os.path.exists(subdir_path): os.mkdir(subdir_path)
subdir_path = os.path.join(masterdir, "params_avg")
if not os.path.exists(subdir_path): os.mkdir(subdir_path)
li = len(masterdir)
else:
li = 0
li = mpi_bcast(li, 1, MPI_INT, Blockdata["main_node"], MPI_COMM_WORLD)[0]
masterdir = mpi_bcast(masterdir, li, MPI_CHAR, Blockdata["main_node"],
MPI_COMM_WORLD)
masterdir = string.join(masterdir, "")
Tracker["constants"]["masterdir"] = masterdir
log_main = Logger(BaseLogger_Files())
log_main.prefix = Tracker["constants"]["masterdir"] + "/"
while not os.path.exists(Tracker["constants"]["masterdir"]):
print("Node ", Blockdata["myid"], " waiting...",
Tracker["constants"]["masterdir"])
sleep(1)
mpi_barrier(MPI_COMM_WORLD)
if (Blockdata["myid"] == Blockdata["main_node"]):
init_dict = {}
print(Tracker["constants"]["isac_dir"])
Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"],
"2dalignment")
core = read_text_row(
os.path.join(Tracker["directory"], "initial2Dparams.txt"))
for im in range(len(core)):
init_dict[im] = core[im]
del core
else:
init_dict = 0
init_dict = wrap_mpi_bcast(init_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
###
do_ctf = True
if options.noctf: do_ctf = False
if (Blockdata["myid"] == Blockdata["main_node"]):
if do_ctf: print("CTF correction is on")
else: print("CTF correction is off")
if options.local_alignment: print("local refinement is on")
else: print("local refinement is off")
if B_enhance: print("Bfactor is to be applied on averages")
elif adjust_to_given_pw2:
print("PW of averages is adjusted to a given 1D PW curve")
elif adjust_to_analytic_model:
print("PW of averages is adjusted to analytical model")
else:
print("PW of averages is not adjusted")
#Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack")
image = get_im(Tracker["constants"]["orgstack"], 0)
Tracker["constants"]["nnxo"] = image.get_xsize()
if Tracker["constants"]["pixel_size"] == -1.0:
print(
"Pixel size value is not provided by user. extracting it from ctf header entry of the original stack."
)
try:
ctf_params = image.get_attr("ctf")
Tracker["constants"]["pixel_size"] = ctf_params.apix
except:
ERROR(
"Pixel size could not be extracted from the original stack.",
"sxcompute_isac_avg.py", 1,
Blockdata["myid"]) # action=1 - fatal error, exit
## Now fill in low-pass filter
isac_shrink_path = os.path.join(Tracker["constants"]["isac_dir"],
"README_shrink_ratio.txt")
if not os.path.exists(isac_shrink_path):
ERROR(
"%s does not exist in the specified ISAC run output directory"
% (isac_shrink_path), "sxcompute_isac_avg.py", 1,
Blockdata["myid"]) # action=1 - fatal error, exit
isac_shrink_file = open(isac_shrink_path, "r")
isac_shrink_lines = isac_shrink_file.readlines()
isac_shrink_ratio = float(
isac_shrink_lines[5]
) # 6th line: shrink ratio (= [target particle radius]/[particle radius]) used in the ISAC run
isac_radius = float(
isac_shrink_lines[6]
) # 7th line: particle radius at original pixel size used in the ISAC run
isac_shrink_file.close()
print("Extracted parameter values")
print("ISAC shrink ratio : {0}".format(isac_shrink_ratio))
print("ISAC particle radius : {0}".format(isac_radius))
Tracker["ini_shrink"] = isac_shrink_ratio
else:
Tracker["ini_shrink"] = 0.0
Tracker = wrap_mpi_bcast(Tracker,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
#print(Tracker["constants"]["pixel_size"], "pixel_size")
x_range = max(Tracker["constants"]["xrange"],
int(1. / Tracker["ini_shrink"] + 0.99999))
a_range = y_range = x_range
if (Blockdata["myid"] == Blockdata["main_node"]):
parameters = read_text_row(
os.path.join(Tracker["constants"]["isac_dir"],
"all_parameters.txt"))
else:
parameters = 0
parameters = wrap_mpi_bcast(parameters,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
params_dict = {}
list_dict = {}
#parepare params_dict
#navg = min(Tracker["constants"]["navg"]*Blockdata["nproc"], EMUtil.get_image_count(os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf")))
navg = min(
Tracker["constants"]["navg"],
EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf")))
global_dict = {}
ptl_list = []
memlist = []
if (Blockdata["myid"] == Blockdata["main_node"]):
print("Number of averages computed in this run is %d" % navg)
for iavg in range(navg):
params_of_this_average = []
image = get_im(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"), iavg)
members = sorted(image.get_attr("members"))
memlist.append(members)
for im in range(len(members)):
abs_id = members[im]
global_dict[abs_id] = [iavg, im]
P = combine_params2( init_dict[abs_id][0], init_dict[abs_id][1], init_dict[abs_id][2], init_dict[abs_id][3], \
parameters[abs_id][0], parameters[abs_id][1]/Tracker["ini_shrink"], parameters[abs_id][2]/Tracker["ini_shrink"], parameters[abs_id][3])
if parameters[abs_id][3] == -1:
print(
"WARNING: Image #{0} is an unaccounted particle with invalid 2D alignment parameters and should not be the member of any classes. Please check the consitency of input dataset."
.format(abs_id)
) # How to check what is wrong about mirror = -1 (Toshio 2018/01/11)
params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0])
ptl_list.append(abs_id)
params_dict[iavg] = params_of_this_average
list_dict[iavg] = members
write_text_row(
params_of_this_average,
os.path.join(Tracker["constants"]["masterdir"], "params_avg",
"params_avg_%03d.txt" % iavg))
ptl_list.sort()
init_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
init_params[im] = [ptl_list[im]] + params_dict[global_dict[
ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
init_params,
os.path.join(Tracker["constants"]["masterdir"],
"init_isac_params.txt"))
else:
params_dict = 0
list_dict = 0
memlist = 0
params_dict = wrap_mpi_bcast(params_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
list_dict = wrap_mpi_bcast(list_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
memlist = wrap_mpi_bcast(memlist,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
# Now computing!
del init_dict
tag_sharpen_avg = 1000
## always apply low pass filter to B_enhanced images to suppress noise in high frequencies
enforced_to_H1 = False
if B_enhance:
if Tracker["constants"]["low_pass_filter"] == -1.0:
enforced_to_H1 = True
if navg < Blockdata["nproc"]: # Each CPU do one average
ERROR("number of nproc is larger than number of averages",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
else:
FH_list = [[0, 0.0, 0.0] for im in range(navg)]
image_start, image_end = MPI_start_end(navg, Blockdata["nproc"],
Blockdata["myid"])
if Blockdata["myid"] == Blockdata["main_node"]:
cpu_dict = {}
for iproc in range(Blockdata["nproc"]):
local_image_start, local_image_end = MPI_start_end(
navg, Blockdata["nproc"], iproc)
for im in range(local_image_start, local_image_end):
cpu_dict[im] = iproc
else:
cpu_dict = 0
cpu_dict = wrap_mpi_bcast(cpu_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
slist = [None for im in range(navg)]
ini_list = [None for im in range(navg)]
avg1_list = [None for im in range(navg)]
avg2_list = [None for im in range(navg)]
plist_dict = {}
data_list = [None for im in range(navg)]
if Blockdata["myid"] == Blockdata["main_node"]:
if B_enhance:
print(
"Avg ID B-factor FH1(Res before ali) FH2(Res after ali)"
)
else:
print("Avg ID FH1(Res before ali) FH2(Res after ali)")
for iavg in range(image_start, image_end):
mlist = EMData.read_images(Tracker["constants"]["orgstack"],
list_dict[iavg])
for im in range(len(mlist)):
#mlist[im]= get_im(Tracker["constants"]["orgstack"], list_dict[iavg][im])
set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
if options.local_alignment:
"""
new_average1 = within_group_refinement([mlist[kik] for kik in range(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH=max(Tracker["constants"]["FH"], FH1), FF=0.02, method="")
new_average2 = within_group_refinement([mlist[kik] for kik in range(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou= Tracker["constants"]["radius"], rs=1.0, xrng=[ x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.02, method="")
new_avg, frc, plist = compute_average(mlist, Tracker["constants"]["radius"], do_ctf)
"""
new_avg, plist, FH2 = refinement_2d_local(
mlist,
Tracker["constants"]["radius"],
a_range,
x_range,
y_range,
CTF=do_ctf,
SNR=1.0e10)
plist_dict[iavg] = plist
FH1 = -1.0
else:
new_avg, frc, plist = compute_average(
mlist, Tracker["constants"]["radius"], do_ctf)
FH1 = get_optimistic_res(frc)
FH2 = -1.0
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg))
FH_list[iavg] = [iavg, FH1, FH2]
if B_enhance:
new_avg, gb = apply_enhancement(
new_avg, Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"])
print(" %6d %6.3f %4.3f %4.3f" % (iavg, gb, FH1, FH2))
elif adjust_to_given_pw2:
roo = read_text_file(Tracker["constants"]["modelpw"], -1)
roo = roo[0] # always on the first column
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], roo)
print(" %6d %4.3f %4.3f " % (iavg, FH1, FH2))
elif adjust_to_analytic_model:
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], None)
print(" %6d %4.3f %4.3f " % (iavg, FH1, FH2))
elif no_adjustment:
pass
if Tracker["constants"]["low_pass_filter"] != -1.0:
if Tracker["constants"]["low_pass_filter"] == 0.0:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 1.0:
low_pass_filter = FH2
if not options.local_alignment: low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"]["low_pass_filter"]
if low_pass_filter >= 0.45: low_pass_filter = 0.45
new_avg = filt_tanl(new_avg, low_pass_filter, 0.02)
else: # No low pass filter but if enforced
if enforced_to_H1: new_avg = filt_tanl(new_avg, FH1, 0.02)
if B_enhance: new_avg = fft(new_avg)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
slist[iavg] = new_avg
print(
strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>",
"Refined average %7d" % iavg)
## send to main node to write
mpi_barrier(MPI_COMM_WORLD)
for im in range(navg):
# avg
if cpu_dict[im] == Blockdata[
"myid"] and Blockdata["myid"] != Blockdata["main_node"]:
send_EMData(slist[im], Blockdata["main_node"], tag_sharpen_avg)
elif cpu_dict[im] == Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
slist[im].set_attr("members", memlist[im])
slist[im].set_attr("n_objects", len(memlist[im]))
slist[im].write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
elif cpu_dict[im] != Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
new_avg_other_cpu = recv_EMData(cpu_dict[im], tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.set_attr("n_objects", len(memlist[im]))
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
if options.local_alignment:
if cpu_dict[im] == Blockdata["myid"]:
write_text_row(
plist_dict[im],
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params_avg",
"ali2d_local_params_avg_%03d.txt" % im))
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(plist_dict[im], Blockdata["main_node"],
MPI_COMM_WORLD)
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
plist_dict[im] = dummy
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
else:
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi_barrier(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if options.local_alignment:
if Blockdata["myid"] == Blockdata["main_node"]:
ali3d_local_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
ali3d_local_params[im] = [ptl_list[im]] + plist_dict[
global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
ali3d_local_params,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params.txt"))
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
else:
if Blockdata["myid"] == Blockdata["main_node"]:
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
mpi_barrier(MPI_COMM_WORLD)
target_xr = 3
target_yr = 3
if (Blockdata["myid"] == 0):
cmd = "{} {} {} {} {} {} {} {} {} {}".format("sxchains.py", os.path.join(Tracker["constants"]["masterdir"],"class_averages.hdf"),\
os.path.join(Tracker["constants"]["masterdir"],"junk.hdf"),os.path.join(Tracker["constants"]["masterdir"],"ordered_class_averages.hdf"),\
"--circular","--radius=%d"%Tracker["constants"]["radius"] , "--xr=%d"%(target_xr+1),"--yr=%d"%(target_yr+1),"--align", ">/dev/null")
junk = cmdexecute(cmd)
cmd = "{} {}".format(
"rm -rf",
os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"))
junk = cmdexecute(cmd)
from mpi import mpi_finalize
mpi_finalize()
exit()