def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
540.0 - psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
360.0 - psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=15. --aa=0.01 --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--output_dir",
type="string",
default="./",
help="output directory")
parser.add_option("--ave2D",
type="string",
default=False,
help="write to the disk a stack of 2D averages")
parser.add_option("--var2D",
type="string",
default=False,
help="write to the disk a stack of 2D variances")
parser.add_option("--ave3D",
type="string",
default=False,
help="write to the disk reconstructed 3D average")
parser.add_option("--var3D",
type="string",
default=False,
help="compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp",
type="int",
default=10,
help="number of neighbouring projections")
parser.add_option("--no_norm",
action="store_true",
default=False,
help="do not use normalization")
#parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option("--npad",
type="int",
default=2,
help="number of time to pad the original images")
parser.add_option("--sym", type="string", default="c1", help="symmetry")
parser.add_option(
"--fl",
type="float",
default=0.0,
help=
"cutoff freqency in absolute frequency (0.0-0.5). (Default - no filtration)"
)
parser.add_option(
"--aa",
type="float",
default=0.0,
help=
"fall off of the filter. Put 0.01 if user has no clue about falloff (Default - no filtration)"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="use CFT correction")
parser.add_option("--VERBOSE",
action="store_true",
default=False,
help="Long output for debugging")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option(
"--VAR",
action="store_true",
default=False,
help="stack on input consists of 2D variances (Default False)")
parser.add_option(
"--decimate",
type="float",
default=1.0,
help=
"image decimate rate, a number larger (expand image) or less (shrink image) than 1. default is 1"
)
parser.add_option(
"--window",
type="int",
default=0,
help=
"reduce images to a small image size without changing pixel_size. Default value is zero."
)
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
parser.add_option(
"--nvec",
type="int",
default=0,
help="number of eigenvectors, default = 0 meaning no PCA calculated")
parser.add_option(
"--symmetrize",
action="store_true",
default=False,
help="Prepare input stack for handling symmetry (Default False)")
(options, args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
# Set up global variables related to bdb cache
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
# Set up global variables related to ERROR function
global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
if RUNNING_UNDER_MPI:
global_def.MPI = True
if options.symmetrize:
if RUNNING_UNDER_MPI:
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if (number_of_proc > 1):
ERROR(
"Cannot use more than one CPU for symmetry prepration",
"sx3dvariability", 1)
except:
pass
except:
pass
if options.output_dir != "./" and not os.path.exists(
options.output_dir):
os.mkdir(options.output_dir)
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
from logger import Logger, BaseLogger_Files
if os.path.exists(os.path.join(options.output_dir, "log.txt")):
os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(options.output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if (sym == "c1"):
ERROR("There is no need to symmetrize stack for C1 symmetry",
"sx3dvariability", 1)
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
if (instack[:4] != "bdb:"):
if output_dir == "./": stack = "bdb:data"
else: stack = "bdb:" + options.output_dir + "/data"
delete_bdb(stack)
junk = cmdexecute("sxcpy.py " + instack + " " + stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack, 'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None] * na
for k in xrange(ks):
#Qfile = "Q%1d"%k
if options.output_dir != "./":
Qfile = os.path.join(options.output_dir, "Q%1d" % k)
else:
Qfile = os.path.join(options.output_dir, "Q%1d" % k)
#delete_bdb("bdb:Q%1d"%k)
delete_bdb("bdb:" + Qfile)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = cmdexecute("e2bdb.py " + stack + " --makevstack=bdb:" +
Qfile)
#DB = db_open_dict("bdb:Q%1d"%k)
DB = db_open_dict("bdb:" + Qfile)
for i in xrange(na):
ut = qt[i] * ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
if options.output_dir == "./": delete_bdb("bdb:sdata")
else: delete_bdb("bdb:" + options.output_dir + "/" + "sdata")
#junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:" + options.output_dir + "/" + "sdata"
print(sdata)
junk = cmdexecute("e2bdb.py " + options.output_dir +
" --makevstack=" + sdata + " --filt=Q")
#junk = cmdexecute("ls EMAN2DB/sdata*")
#a = get_im("bdb:sdata")
a = get_im(sdata)
a.set_attr("variabilitysymmetry", sym)
#a.write_image("bdb:sdata")
a.write_image(sdata)
else:
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 1:
stack = args[0]
else:
print(("usage: " + usage))
print(("Please run '" + progname + " -h' for detailed options"))
return 1
t0 = time()
# obsolete flags
options.MPI = True
options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter",
"sx3dvariability", 1, myid)
if options.VAR and options.SND:
ERROR("Only one of var and SND can be set!", "sx3dvariability",
myid)
exit()
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR(
"When VAR is set, the program cannot output ave2D, ave3D or var2D",
"sx3dvariability", 1, myid)
exit()
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
# exit()
if options.nvec > 0:
ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
exit()
if options.nvec > 0 and options.ave3D == None:
ERROR("When doing PCA analysis, one must set ave3D",
"sx3dvariability",
myid=myid)
exit()
import string
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if options.output_dir != "./" and not os.path.exists(
options.output_dir):
os.mkdir(options.output_dir)
img_per_grp = options.img_per_grp
nvec = options.nvec
radiuspca = options.radiuspca
from logger import Logger, BaseLogger_Files
#if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(options.output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("instack :" + stack)
log_main.add("output_dir :" + options.output_dir)
log_main.add("var3d :" + options.var3D)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
#print_begin_msg("sx3dvariability")
msg = "sx3dvariability"
log_main.add(msg)
print(line, msg)
msg = ("%-70s: %s\n" % ("Input stack", stack))
log_main.add(msg)
print(line, msg)
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
if options.sym != "c1":
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if (i != options.sym):
ERROR(
"The symmetry provided does not agree with the symmetry of the input stack",
"sx3dvariability",
myid=myid)
except:
ERROR(
"Input stack is not prepared for symmetry, please follow instructions",
"sx3dvariability",
myid=myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if ((nima / i) * i != nima):
ERROR(
"The length of the input stack is incorrect for symmetry processing",
"sx3dvariability",
myid=myid)
symbaselen = nima / i
else:
symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
Tracker = {}
Tracker["total_stack"] = nima
if options.decimate == 1.:
if options.window != 0:
nx = options.window
ny = options.window
else:
if options.window == 0:
nx = int(nx * options.decimate)
ny = int(ny * options.decimate)
else:
nx = int(options.window * options.decimate)
ny = nx
Tracker["nx"] = nx
Tracker["ny"] = ny
Tracker["nz"] = nx
symbaselen = bcast_number_to_all(symbaselen)
if radiuspca == -1: radiuspca = nx / 2 - 2
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = "%-70s: %d\n" % ("Number of projection", nima)
log_main.add(msg)
print(line, msg)
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR:
#varList = EMData.read_images(stack, range(img_begin, img_end))
varList = []
this_image = EMData()
for index_of_particle in xrange(img_begin, img_end):
this_image.read_image(stack, index_of_particle)
varList.append(
image_decimate_window_xform_ctf(this_image,
options.decimate,
options.window,
options.CTF))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in xrange(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x * 10000 + psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = "%-70s: %d\n" % ("Number of neighboring projections",
img_per_grp)
log_main.add(msg)
print(line, msg)
msg = "...... Finding neighboring projections\n"
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
msg = "Number of images per group: %d" % img_per_grp
log_main.add(msg)
print(line, msg)
msg = "Now grouping projections"
log_main.add(msg)
print(line, msg)
proj_angles.sort()
proj_angles_list = [0.0] * (nima * 4)
if myid == main_node:
for i in xrange(nima):
proj_angles_list[i * 4] = proj_angles[i][1]
proj_angles_list[i * 4 + 1] = proj_angles[i][2]
proj_angles_list[i * 4 + 2] = proj_angles[i][3]
proj_angles_list[i * 4 + 3] = proj_angles[i][4]
proj_angles_list = bcast_list_to_all(proj_angles_list, myid,
main_node)
proj_angles = []
for i in xrange(nima):
proj_angles.append([
proj_angles_list[i * 4], proj_angles_list[i * 4 + 1],
proj_angles_list[i * 4 + 2],
int(proj_angles_list[i * 4 + 3])
])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp,
range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
if options.VERBOSE:
print("On node %2d, number of images needed to be read = %5d" %
(myid, len(all_proj)))
index = {}
for i in xrange(len(all_proj)):
index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Finding neighboring projections lasted [s]",
time() - t2))
log_main.add(msg)
print(msg)
msg = ("%-70s: %d\n" %
("Number of groups processed on the main node",
len(proj_list)))
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Grouping projections took: ", (time() - t2) / 60,
"[min]")
print("Number of groups on main node: ", len(proj_list))
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("...... calculating the stack of 2D variances \n")
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Now calculating the stack of 2D variances")
proj_params = [0.0] * (nima * 5)
aveList = []
varList = []
if nvec > 0:
eigList = [[] for i in xrange(nvec)]
if options.VERBOSE:
print("Begin to read images on processor %d" % (myid))
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
imgdata = []
for index_of_proj in xrange(len(all_proj)):
#img = EMData()
#img.read_image(stack, all_proj[index_of_proj])
dmg = image_decimate_window_xform_ctf(
get_im(stack, all_proj[index_of_proj]), options.decimate,
options.window, options.CTF)
#print dmg.get_xsize(), "init"
imgdata.append(dmg)
if options.VERBOSE:
print("Reading images on processor %d done, time = %.2f" %
(myid, time() - ttt))
print("On processor %d, we got %d images" %
(myid, len(imgdata)))
mpi_barrier(MPI_COMM_WORLD)
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
for i in xrange(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])
grp_imgdata = []
for j in xrange(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
alpha, sx, sy, mirror = params_3D_2D_NEW(
phi, theta, psi, s2x, s2y, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, phiM - phi, 0.0, 0.0, 1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, 180 - (phiM - phi), 0.0,
0.0, 1.0)
else:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(phiM - phi), 0.0, 0.0,
1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(180 - (phiM - phi)), 0.0,
0.0, 1.0)
set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
grp_imgdata.append(imgdata[mj])
#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()
if not options.no_norm:
#print grp_imgdata[j].get_xsize()
mask = model_circle(nx / 2 - 2, nx, nx)
for k in xrange(img_per_grp):
ave, std, minn, maxx = Util.infomask(
grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
del mask
if options.fl > 0.0:
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_tanl(
filt_ctf(
fft(
pad(grp_imgdata[k], nx2, ny2,
1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1), options.fl,
options.aa)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
for k in xrange(img_per_grp):
grp_imgdata[k] = filt_tanl(grp_imgdata[k],
options.fl, options.aa)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
from utilities import pad, read_text_file
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_ctf(fft(
pad(grp_imgdata[k], nx2, ny2, 1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
'''
if i < 10 and myid == main_node:
for k in xrange(10):
grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
'''
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("pp.hdf", pp)
"""
ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("qq.hdf", pp)
"""
var = model_blank(nx, ny)
for q in grp_imgdata:
Util.add_img2(var, q)
Util.mul_scalar(var, 1.0 / (len(grp_imgdata) - 1))
# Switch to std dev
var = square_root(threshold(var))
#if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a")
#else: ave, var = avgvar(grp_imgdata, mode="a")
"""
if myid == main_node:
ave.write_image("avgv.hdf",i)
var.write_image("varv.hdf",i)
"""
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
if options.VERBOSE:
print("%5.2f%% done on processor %d" %
(i * 100.0 / len(proj_list), myid))
if nvec > 0:
eig = pca(input_stacks=grp_imgdata,
subavg="",
mask_radius=radiuspca,
nvec=nvec,
incore=True,
shuffle=False,
genbuf=True)
for k in xrange(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
del imgdata
# To this point, all averages, variances, and eigenvectors are computed
if options.ave2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(aveList)):
aveList[im].write_image(
os.path.join(options.output_dir,
options.ave2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
"""
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol = fpol(ave, Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.ave2D), km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
"""
if options.ave3D:
from fundamentals import fpol
if options.VERBOSE:
print("Reconstructing 3D average volume")
ave3D = recons3d_4nn_MPI(myid,
aveList,
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
ave3D = fpol(ave3D, Tracker["nx"], Tracker["nx"],
Tracker["nx"])
ave3D.write_image(
os.path.join(options.output_dir, options.ave3D))
msg = ("%-70s: %s\n" % (
"Writing to the disk volume reconstructed from averages as",
options.ave3D))
log_main.add(msg)
print(line, msg)
del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList
if nvec > 0:
for k in xrange(nvec):
if options.VERBOSE:
print("Reconstruction eigenvolumes", k)
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid,
eigList[k],
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image(
os.path.join(options.outpout_dir,
"eig3d_%03d.hdf" % (k, ITER)))
Util.mul_img(eig3D,
model_circle(radiuspca, nx, nx, nx))
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in xrange(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(
eigList[k][l])
proj = prgs(eig3Df, kb,
[phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node,
MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM,
main_node, MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S",
localtime()) + " =>"
u = int(u[0])
msg = (" Eigenvector: ", k, " number changed ",
int(icont[0]))
log_main.add(msg)
print(line, msg)
else:
u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(varList)):
tmpvol = fpol(varList[im], Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.var2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
tmpvol = fpol(ave, Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.var2D, km))
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in xrange(len(varList)):
send_EMData(varList[im], main_node, im + myid +
70000) # What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node and options.VERBOSE:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("Reconstructing 3D variability volume")
log_main.add(msg)
print(line, msg)
t6 = time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid,
varList,
symmetry=options.sym,
npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
res = fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
res.write_image(os.path.join(options.output_dir,
options.var3D))
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Reconstructing 3D variability took [s]", time() - t6))
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Reconstruction took: %.2f [min]" %
((time() - t6) / 60))
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Total time for these computations [s]", time() - t0))
print(line, msg)
log_main.add(msg)
if options.VERBOSE:
print("Total time for these computations: %.2f [min]" %
((time() - t0) / 60))
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("sx3dvariability")
print(line, msg)
log_main.add(msg)
from mpi import mpi_finalize
mpi_finalize()
if RUNNING_UNDER_MPI:
global_def.MPI = False
global_def.BATCH = False
def runcheck(classavgstack, recon, outdir, inangles=None, selectdoc=None, displayYN=False,
projstack='proj.hdf', outangles='angles.txt', outstack='comp-proj-reproj.hdf', normstack='comp-proj-reproj-norm.hdf'):
print
# Check if inputs exist
check(classavgstack)
check(recon)
# Create directory if it doesn't exist
if not os.path.isdir(outdir):
os.makedirs(outdir) # os.mkdir() can only operate one directory deep
print("mkdir -p %s" % outdir)
# Expand path for outputs
projstack = os.path.join(outdir, projstack)
outangles = os.path.join(outdir, outangles)
outstack = os.path.join(outdir, outstack)
normstack = os.path.join(outdir, normstack)
# Get number of images
nimg0 = EMUtil.get_image_count(classavgstack)
#print("nimg0: %s" % nimg0)
# In case class averages include discarded images, apply selection file
if selectdoc:
goodavgs, extension = os.path.splitext(classavgstack)
newclasses = goodavgs + "_kept" + extension
# e2proc2d appends to existing files, so rename existing output
if os.path.exists(newclasses):
renamefile = newclasses + '.bak'
os.rename(newclasses, renamefile)
print("mv %s %s" % (newclasses, renamefile))
cmd7="e2proc2d.py %s %s --list=%s" % (classavgstack, newclasses, selectdoc)
print cmd7
os.system(cmd7)
# Update class-averages
classavgstack = newclasses
# Import Euler angles
if inangles:
cmd6="sxheader.py %s --params=xform.projection --import=%s" % (classavgstack, inangles)
print cmd6
header(classavgstack, 'xform.projection', fimport=inangles)
cmd1="sxheader.py %s --params=xform.projection --export=%s" % (classavgstack, outangles)
print cmd1
#os.system(cmd1)
try:
header(classavgstack, 'xform.projection', fexport=outangles)
except RuntimeError:
print("\nERROR!! No projection angles found in class-average stack header!\n")
exit()
cmd2="sxproject3d.py %s %s --angles=%s" % (recon, projstack, outangles)
print cmd2
#os.system(cmd2)
project3d(recon, stack=projstack, listagls=outangles)
imgcounter = 0 # montage will have double the number of images as number of class-averages
result=[]
# Number of images may have changed
nimg1 = EMUtil.get_image_count(classavgstack)
for imgnum in xrange(nimg1):
#print imgnum
classimg = get_im(classavgstack, imgnum)
ccc1 = classimg.get_attr_default('cross-corr', -1.0)
prjimg = get_im(projstack,imgnum)
ccc1 = prjimg.get_attr_default('cross-corr', -1.0)
cccoeff = ccc(prjimg,classimg)
#print imgnum, cccoeff
classimg.set_attr_dict({'cross-corr':cccoeff})
prjimg.set_attr_dict({'cross-corr':cccoeff})
prjimg.write_image(outstack,imgcounter)
imgcounter += 1
classimg.write_image(outstack, imgcounter)
imgcounter += 1
result.append(cccoeff)
result1 = sum(result)
#print result1
nimg2 = EMUtil.get_image_count(outstack)
meanccc = result1/nimg1
print("Mean CCC is %s" % meanccc)
for imgnum in xrange(nimg2):
if (imgnum % 2 ==0):
prjimg = get_im(outstack,imgnum)
meanccc1 = prjimg.get_attr_default('mean-cross-corr', -1.0)
prjimg.set_attr_dict({'mean-cross-corr':meanccc})
write_header(outstack,prjimg,imgnum)
if (imgnum % 100) == 0:
print imgnum
# e2proc2d appends to existing files, so delete existing output
if os.path.exists(normstack):
os.remove(normstack)
print("rm %s" % normstack)
cmd5="e2proc2d.py %s %s --process=normalize" % (outstack, normstack)
print cmd5
os.system(cmd5)
# Optionally pop up e2display
if displayYN:
cmd8 = "e2display.py %s" % normstack
print cmd8
os.system(cmd8)
print("Done!")
def main():
import sys
import os
import math
import random
import pyemtbx.options
import time
from random import random, seed, randint
from optparse import OptionParser
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Forms chains of 2D images based on their similarities.
Functionality:
4. Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent).
Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers.
The ways to use the program:
4.1 Use option initial to specify which image will be used as an initial seed to form the chain.
sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25
4.2 If options initial is omitted, the program will determine which image best serves as initial seed to form the chain
sxprocess.py input_stack.hdf output_stack.hdf --radius=25
4.3 Use option circular to form a circular chain.
sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25
4.4 New circular code based on pairwise alignments
sxprocess.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt
4.5 Circular ordering based on pairwise alignments
sxprocess.py vols.hdf chain.hdf mask.hdf --dd --radius=25
"""
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False)
parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False)
parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False)
parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering")
# import params for 2D alignment
parser.add_option("--ou", type="int", default=-1, help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)")
parser.add_option("--xr", type="int", default=0, help="range for translation search in x direction, search is +/xr (0)")
parser.add_option("--yr", type="int", default=0, help="range for translation search in y direction, search is +/yr (0)")
#parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)")
parser.add_option("--pairwiseccc", type="string", default= None, help="Input/output pairwise ccc file")
(options, args) = parser.parse_args()
global_def.BATCH = True
if options.dd:
nargs = len(args)
if nargs != 3:
print "must provide name of input and two output files!"
return
stack = args[0]
new_stack = args[1]
from utilities import model_circle
from statistics import ccc
from statistics import mono
lend = EMUtil.get_image_count(stack)
lccc = [None]*(lend*(lend-1)/2)
for i in xrange(lend-1):
v1 = get_im( stack, i )
if( i == 0 and nargs == 2):
nx = v1.get_xsize()
ny = v1.get_ysize()
nz = v1.get_ysize()
if options.ou < 1 : radius = nx//2-2
else: radius = options.ou
mask = model_circle(radius, nx, ny, nz)
else:
mask = get_im(args[2])
for j in xrange(i+1, lend):
lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0]
order = tsp(lccc)
if(len(order) != lend):
print " problem with data length"
from sys import exit
exit()
print "Total sum of cccs :",TotalDistance(order, lccc)
print "ordering :",order
for i in xrange(lend): get_im(stack, order[i]).write_image( new_stack, i )
elif options.align:
nargs = len(args)
if nargs != 3:
print "must provide name of input and two output files!"
return
from utilities import get_params2D, model_circle
from fundamentals import rot_shift2D
from statistics import ccc
from time import time
from alignment import align2d, align2d_scf
from multi_shc import mult_transform
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
"""
# will align anyway
try:
ttt = d[0].get_attr('xform.params2d')
for i in xrange(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
"""
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.ou < 1 : radius = nx//2-2
else: radius = options.ou
mask = model_circle(radius, nx, ny)
if(options.xr < 0): xrng = 0
else: xrng = options.xr
if(options.yr < 0): yrng = xrng
else: yrng = options.yr
initial = max(options.initial, 0)
from statistics import mono
lend = len(d)
lccc = [None]*(lend*(lend-1)/2)
from utilities import read_text_row
if options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) :
st = time()
for i in xrange(lend-1):
for j in xrange(i+1, lend):
# j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger)
#alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F")
alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius)
lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir]
#print " %4d %10.1f"%(i,time()-st)
if(not os.path.exists(options.pairwiseccc)):
from utilities import write_text_row
write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc)
elif(os.path.exists(options.pairwiseccc)):
lccc = read_text_row(options.pairwiseccc)
initial = int(lccc[0][0] + 0.1)
del lccc[0]
for i in xrange(len(lccc)):
T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)})
lccc[i] = [lccc[i][0],T]
tdummy = Transform({"type":"2D"})
maxsum = -1.023
for m in xrange(0,lend):#initial, initial+1):
indc = range( lend )
lsnake = [[m, tdummy, 0.0]]
del indc[m]
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in xrange(len(indc)):
cuc = lccc[mono(indc[i], lsnake[-1][0])][0]
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
# Here we need transformation from the current to the previous,
# meaning indc[i] -> lsnake[-1][0]
T = lccc[mono(indc[i], lsnake[-1][0])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if( indc[i] > lsnake[-1][0] ): T = T.inverse()
lsnake.append([qi,T, maxcit])
lsum += maxcit
del indc[indc.index(qi)]
T = lccc[mono(indc[-1], lsnake[-1][0])][1]
if( indc[-1] > lsnake[-1][0]): T = T.inverse()
lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]])
print " initial image and lsum ",m,lsum
#print lsnake
if(lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in xrange(lend)]
print " Initial image selected : ",init,maxsum," ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc)
#for q in snake: print q
from copy import deepcopy
trans=deepcopy([snake[i][1] for i in xrange(len(snake))])
print [snake[i][0] for i in xrange(len(snake))]
"""
for m in xrange(lend):
prms = trans[m].get_params("2D")
print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
"""
for k in xrange(lend-2,0,-1):
T = snake[k][1]
for i in xrange(k+1, lend):
trans[i] = T*trans[i]
# To add - apply all transformations and do the overall centering.
for m in xrange(lend):
prms = trans[m].get_params("2D")
#print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
#rot_shift2D(d[snake[m][0]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image(new_stack, m)
rot_shift2D(d[snake[m][0]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(new_stack, m)
order = tsp(lccc)
if(len(order) != lend):
print " problem with data length"
from sys import exit
exit()
print TotalDistance(order, lccc)
print order
ibeg = order.index(init)
order = [order[(i+ibeg)%lend] for i in xrange(lend)]
print TotalDistance(order, lccc)
print order
snake = [tdummy]
for i in xrange(1,lend):
# Here we need transformation from the current to the previous,
# meaning order[i] -> order[i-1]]
T = lccc[mono(order[i], order[i-1])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if( order[i] > order[i-1] ): T = T.inverse()
snake.append(T)
assert(len(snake) == lend)
from copy import deepcopy
trans = deepcopy(snake)
for k in xrange(lend-2,0,-1):
T = snake[k]
for i in xrange(k+1, lend):
trans[i] = T*trans[i]
# Try to smooth the angles - complicated, I am afraid one would have to use angles forward and backwards
# and find their average??
# In addition, one would have to recenter them
"""
trms = []
for m in xrange(lend):
prms = trans[m].get_params("2D")
trms.append([prms["alpha"], prms["mirror"]])
for i in xrange(3):
for m in xrange(lend):
mb = (m-1)%lend
me = (m+1)%lend
# angles order mb,m,me
# calculate predicted angles mb->m
"""
for m in xrange(lend):
prms = trans[m].get_params("2D")
#rot_shift2D(d[order[m]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image("metro.hdf", m)
rot_shift2D(d[order[m]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(args[2], m)
"""
# This was an effort to get number of loops, inconclusive, to say the least
from numpy import outer, zeros, float32, sqrt
lend = len(d)
cor = zeros(lend,float32)
cor = outer(cor, cor)
for i in xrange(lend): cor[i][i] = 1.0
for i in xrange(lend-1):
for j in xrange(i+1, lend):
cor[i,j] = lccc[mono(i,j)][0]
cor[j,i] = cor[i,j]
lmbd, eigvec = pca(cor)
from utilities import write_text_file
nvec=20
print [lmbd[j] for j in xrange(nvec)]
print " G"
mm = [-1]*lend
for i in xrange(lend): # row
mi = -1.0e23
for j in xrange(nvec):
qt = eigvec[j][i]
if(abs(qt)>mi):
mi = abs(qt)
mm[i] = j
for j in xrange(nvec):
qt = eigvec[j][i]
print round(qt,3), # eigenvector
print mm[i]
print
for j in xrange(nvec):
qt = []
for i in xrange(lend):
if(mm[i] == j): qt.append(i)
if(len(qt)>0): write_text_file(qt,"loop%02d.txt"%j)
"""
"""
print [lmbd[j] for j in xrange(nvec)]
print " B"
mm = [-1]*lend
for i in xrange(lend): # row
mi = -1.0e23
for j in xrange(nvec):
qt = eigvec[j][i]/sqrt(lmbd[j])
if(abs(qt)>mi):
mi = abs(qt)
mm[i] = j
for j in xrange(nvec):
qt = eigvec[j][i]/sqrt(lmbd[j])
print round(qt,3), # eigenvector
print mm[i]
print
"""
"""
lend=3
cor = zeros(lend,float32)
cor = outer(cor, cor)
cor[0][0] =136.77
cor[0][1] = 79.15
cor[0][2] = 37.13
cor[1][0] = 79.15
cor[2][0] = 37.13
cor[1][1] = 50.04
cor[1][2] = 21.65
cor[2][1] = 21.65
cor[2][2] = 13.26
lmbd, eigvec = pca(cor)
print lmbd
print eigvec
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i], # eigenvector
print
print " B"
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i]/sqrt(lmbd[j]), # eigenvector
print
print " G"
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i]*sqrt(lmbd[j]), # eigenvector
print
"""
else:
nargs = len(args)
if nargs != 2:
print "must provide name of input and output file!"
return
from utilities import get_params2D, model_circle
from fundamentals import rot_shift2D
from statistics import ccc
from time import time
from alignment import align2d
from multi_shc import mult_transform
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
try:
ttt = d[0].get_attr('xform.params2d')
for i in xrange(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.radius < 1 : radius = nx//2-2
else: radius = options.radius
mask = model_circle(radius, nx, ny)
init = options.initial
if init > -1 :
print " initial image: %d" % init
temp = d[init].copy()
temp.write_image(new_stack, 0)
del d[init]
k = 1
lsum = 0.0
while len(d) > 1:
maxcit = -111.
for i in xrange(len(d)):
cuc = ccc(d[i], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = i
# print k, maxcit
lsum += maxcit
temp = d[qi].copy()
del d[qi]
temp.write_image(new_stack, k)
k += 1
print lsum
d[0].write_image(new_stack, k)
else:
if options.circular :
print "Using options.circular"
# figure the "best circular" starting image
maxsum = -1.023
for m in xrange(len(d)):
indc = range(len(d) )
lsnake = [-1]*(len(d)+1)
lsnake[0] = m
lsnake[-1] = m
del indc[m]
temp = d[m].copy()
lsum = 0.0
direction = +1
k = 1
while len(indc) > 1:
maxcit = -111.
for i in xrange(len(indc)):
cuc = ccc(d[indc[i]], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
lsnake[k] = qi
lsum += maxcit
del indc[indc.index(qi)]
direction = -direction
for i in xrange( 1,len(d) ):
if( direction > 0 ):
if(lsnake[i] == -1):
temp = d[lsnake[i-1]].copy()
#print " forw ",lsnake[i-1]
k = i
break
else:
if(lsnake[len(d) - i] == -1):
temp = d[lsnake[len(d) - i +1]].copy()
#print " back ",lsnake[len(d) - i +1]
k = len(d) - i
break
lsnake[lsnake.index(-1)] = indc[-1]
#print " initial image and lsum ",m,lsum
#print lsnake
if(lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in xrange(len(d))]
print " Initial image selected : ",init,maxsum
print lsnake
for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m)
else:
# figure the "best" starting image
maxsum = -1.023
for m in xrange(len(d)):
indc = range(len(d) )
lsnake = [m]
del indc[m]
temp = d[m].copy()
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in xrange(len(indc)):
cuc = ccc(d[indc[i]], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
lsnake.append(qi)
lsum += maxcit
temp = d[qi].copy()
del indc[indc.index(qi)]
lsnake.append(indc[-1])
#print " initial image and lsum ",m,lsum
#print lsnake
if(lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in xrange(len(d))]
print " Initial image selected : ",init,maxsum
print lsnake
for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m)
def main():
import sys
import os
import math
import random
import pyemtbx.options
import time
from random import random, seed, randint
from optparse import OptionParser
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Forms chains of 2D images based on their similarities.
Functionality:
4. Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent).
Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers.
The ways to use the program:
4.1 Use option initial to specify which image will be used as an initial seed to form the chain.
sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25
4.2 If options initial is omitted, the program will determine which image best serves as initial seed to form the chain
sxprocess.py input_stack.hdf output_stack.hdf --radius=25
4.3 Use option circular to form a circular chain.
sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25
4.4 New circular code based on pairwise alignments
sxprocess.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt
4.5 Circular ordering based on pairwise alignments
sxprocess.py vols.hdf chain.hdf mask.hdf --dd --radius=25
"""
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False)
parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False)
parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False)
parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering")
# import params for 2D alignment
parser.add_option("--ou", type="int", default=-1, help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)")
parser.add_option("--xr", type="int", default=0, help="range for translation search in x direction, search is +/xr (0)")
parser.add_option("--yr", type="int", default=0, help="range for translation search in y direction, search is +/yr (0)")
#parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)")
parser.add_option("--pairwiseccc", type="string", default= None, help="Input/output pairwise ccc file")
(options, args) = parser.parse_args()
global_def.BATCH = True
if options.dd:
nargs = len(args)
if nargs != 3:
print "must provide name of input and two output files!"
return
stack = args[0]
new_stack = args[1]
from utilities import model_circle
from statistics import ccc
from statistics import mono
lend = EMUtil.get_image_count(stack)
lccc = [None]*(lend*(lend-1)/2)
for i in xrange(lend-1):
v1 = get_im( stack, i )
if( i == 0 and nargs == 2):
nx = v1.get_xsize()
ny = v1.get_ysize()
nz = v1.get_ysize()
if options.ou < 1 : radius = nx//2-2
else: radius = options.ou
mask = model_circle(radius, nx, ny, nz)
else:
mask = get_im(args[2])
for j in xrange(i+1, lend):
lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0]
order = tsp(lccc)
if(len(order) != lend):
print " problem with data length"
from sys import exit
exit()
print "Total sum of cccs :",TotalDistance(order, lccc)
print "ordering :",order
for i in xrange(lend): get_im(stack, order[i]).write_image( new_stack, i )
elif options.align:
nargs = len(args)
if nargs != 3:
print "must provide name of input and two output files!"
return
from utilities import get_params2D, model_circle
from fundamentals import rot_shift2D
from statistics import ccc
from time import time
from alignment import align2d, align2d_scf
from multi_shc import mult_transform
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
"""
# will align anyway
try:
ttt = d[0].get_attr('xform.params2d')
for i in xrange(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
"""
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.ou < 1 : radius = nx//2-2
else: radius = options.ou
mask = model_circle(radius, nx, ny)
if(options.xr < 0): xrng = 0
else: xrng = options.xr
if(options.yr < 0): yrng = xrng
else: yrng = options.yr
initial = max(options.initial, 0)
from statistics import mono
lend = len(d)
lccc = [None]*(lend*(lend-1)/2)
from utilities import read_text_row
if options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) :
st = time()
for i in xrange(lend-1):
for j in xrange(i+1, lend):
# j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger)
#alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F")
alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius)
lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir]
#print " %4d %10.1f"%(i,time()-st)
if(not os.path.exists(options.pairwiseccc)):
from utilities import write_text_row
write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc)
elif(os.path.exists(options.pairwiseccc)):
lccc = read_text_row(options.pairwiseccc)
initial = int(lccc[0][0] + 0.1)
del lccc[0]
for i in xrange(len(lccc)):
T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)})
lccc[i] = [lccc[i][0],T]
tdummy = Transform({"type":"2D"})
maxsum = -1.023
for m in xrange(0,lend):#initial, initial+1):
indc = range( lend )
lsnake = [[m, tdummy, 0.0]]
del indc[m]
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in xrange(len(indc)):
cuc = lccc[mono(indc[i], lsnake[-1][0])][0]
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
# Here we need transformation from the current to the previous,
# meaning indc[i] -> lsnake[-1][0]
T = lccc[mono(indc[i], lsnake[-1][0])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if( indc[i] > lsnake[-1][0] ): T = T.inverse()
lsnake.append([qi,T, maxcit])
lsum += maxcit
del indc[indc.index(qi)]
T = lccc[mono(indc[-1], lsnake[-1][0])][1]
if( indc[-1] > lsnake[-1][0]): T = T.inverse()
lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]])
print " initial image and lsum ",m,lsum
#print lsnake
if(lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in xrange(lend)]
print " Initial image selected : ",init,maxsum," ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc)
#for q in snake: print q
from copy import deepcopy
trans=deepcopy([snake[i][1] for i in xrange(len(snake))])
print [snake[i][0] for i in xrange(len(snake))]
else: radius = options.radius mask = model_circle(radius, nx, ny) init = options.initial if init > -1 : print " initial image: %d" % init temp = d[init].copy() temp.write_image(new_stack, 0) del d[init] k = 1 lsum = 0.0 while len(d) > 1: maxcit = -111. for i in xrange(len(d)): cuc = ccc(d[i], temp, mask) if cuc > maxcit: maxcit = cuc qi = i # print k, maxcit lsum += maxcit temp = d[qi].copy() del d[qi] temp.write_image(new_stack, k) k += 1 print lsum d[0].write_image(new_stack, k) else: if options.circular : print "Using options.circular" # figure the "best circular" starting image
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=0.2 --aa=0.1 --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--ave2D", type="string" , default=False, help="write to the disk a stack of 2D averages")
parser.add_option("--var2D", type="string" , default=False, help="write to the disk a stack of 2D variances")
parser.add_option("--ave3D", type="string" , default=False, help="write to the disk reconstructed 3D average")
parser.add_option("--var3D", type="string" , default=False, help="compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp", type="int" , default=10 , help="number of neighbouring projections")
parser.add_option("--no_norm", action="store_true", default=False, help="do not use normalization")
parser.add_option("--radiusvar", type="int" , default=-1 , help="radius for 3D var" )
parser.add_option("--npad", type="int" , default=2 , help="number of time to pad the original images")
parser.add_option("--sym" , type="string" , default="c1" , help="symmetry")
parser.add_option("--fl", type="float" , default=0.0 , help="stop-band frequency (Default - no filtration)")
parser.add_option("--aa", type="float" , default=0.0 , help="fall off of the filter (Default - no filtration)")
parser.add_option("--CTF", action="store_true", default=False, help="use CFT correction")
parser.add_option("--VERBOSE", action="store_true", default=False, help="Long output for debugging")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option("--VAR" , action="store_true", default=False, help="stack on input consists of 2D variances (Default False)")
parser.add_option("--decimate", type="float", default=1.0, help="image decimate rate, a number large than 1. default is 1")
parser.add_option("--window", type="int", default=0, help="reduce images to a small image size without changing pixel_size. Default value is zero.")
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
parser.add_option("--nvec", type="int" , default=0 , help="number of eigenvectors, default = 0 meaning no PCA calculated")
parser.add_option("--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)")
(options,args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD, MPI_TAG_UB
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
if options.symmetrize :
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if( number_of_proc > 1 ):
ERROR("Cannot use more than one CPU for symmetry prepration","sx3dvariability",1)
except:
pass
except:
pass
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
instack = args[0]
sym = options.sym
if( sym == "c1" ):
ERROR("Thre is no need to symmetrize stack for C1 symmetry","sx3dvariability",1)
if(instack[:4] !="bdb:"):
stack = "bdb:data"
delete_bdb(stack)
cmdexecute("sxcpy.py "+instack+" "+stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack,'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None]*na
for k in xrange(ks):
delete_bdb("bdb:Q%1d"%k)
cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
DB = db_open_dict("bdb:Q%1d"%k)
for i in xrange(na):
ut = qt[i]*ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
delete_bdb("bdb:sdata")
cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
#cmdexecute("ls EMAN2DB/sdata*")
a = get_im("bdb:sdata")
a.set_attr("variabilitysymmetry",sym)
a.write_image("bdb:sdata")
else:
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 1:
stack = args[0]
else:
print( "usage: " + usage)
print( "Please run '" + progname + " -h' for detailed options")
return 1
t0 = time()
# obsolete flags
options.MPI = True
options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid)
if options.VAR and options.SND:
ERROR("Only one of var and SND can be set!", "sx3dvariability", myid)
exit()
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid)
exit()
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
# exit()
if options.nvec > 0 :
ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
exit()
if options.nvec > 0 and options.ave3D == None:
ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid)
exit()
import string
options.sym = options.sym.lower()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
if myid == main_node:
print_begin_msg("sx3dvariability")
print_msg("%-70s: %s\n"%("Input stack", stack))
img_per_grp = options.img_per_grp
nvec = options.nvec
radiuspca = options.radiuspca
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
if options.sym != "c1" :
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry")
if(i != options.sym):
ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", myid=myid)
except:
ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", myid=myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if((nima/i)*i != nima):
ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", myid=myid)
symbaselen = nima/i
else: symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
Tracker ={}
Tracker["nx"] =nx
Tracker["ny"] =ny
Tracker["total_stack"]=nima
if options.decimate==1.:
if options.window !=0:
nx = options.window
ny = options.window
else:
if options.window ==0:
nx = int(nx/options.decimate)
ny = int(ny/options.decimate)
else:
nx = int(options.window/options.decimate)
ny = nx
symbaselen = bcast_number_to_all(symbaselen)
if radiuspca == -1: radiuspca = nx/2-2
if myid == main_node:
print_msg("%-70s: %d\n"%("Number of projection", nima))
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR:
#varList = EMData.read_images(stack, range(img_begin, img_end))
varList = []
this_image = EMData()
for index_of_particle in xrange(img_begin,img_end):
this_image.read_image(stack,index_of_particle)
varList.append(image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in xrange(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x*10000+psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
print_msg("%-70s: %d\n"%("Number of neighboring projections", img_per_grp))
print_msg("...... Finding neighboring projections\n")
if options.VERBOSE:
print "Number of images per group: ", img_per_grp
print "Now grouping projections"
proj_angles.sort()
proj_angles_list = [0.0]*(nima*4)
if myid == main_node:
for i in xrange(nima):
proj_angles_list[i*4] = proj_angles[i][1]
proj_angles_list[i*4+1] = proj_angles[i][2]
proj_angles_list[i*4+2] = proj_angles[i][3]
proj_angles_list[i*4+3] = proj_angles[i][4]
proj_angles_list = bcast_list_to_all(proj_angles_list, myid, main_node)
proj_angles = []
for i in xrange(nima):
proj_angles.append([proj_angles_list[i*4], proj_angles_list[i*4+1], proj_angles_list[i*4+2], int(proj_angles_list[i*4+3])])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
if options.VERBOSE:
print "On node %2d, number of images needed to be read = %5d"%(myid, len(all_proj))
index = {}
for i in xrange(len(all_proj)): index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2))
print_msg("%-70s: %d\n"%("Number of groups processed on the main node", len(proj_list)))
if options.VERBOSE:
print "Grouping projections took: ", (time()-t2)/60 , "[min]"
print "Number of groups on main node: ", len(proj_list)
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("...... calculating the stack of 2D variances \n")
if options.VERBOSE:
print "Now calculating the stack of 2D variances"
proj_params = [0.0]*(nima*5)
aveList = []
varList = []
if nvec > 0:
eigList = [[] for i in xrange(nvec)]
if options.VERBOSE: print "Begin to read images on processor %d"%(myid)
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
img = EMData()
imgdata = []
for index_of_proj in xrange(len(all_proj)):
img.read_image(stack, all_proj[index_of_proj])
dmg = image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF)
#print dmg.get_xsize(), "init"
imgdata.append(dmg)
if options.VERBOSE:
print "Reading images on processor %d done, time = %.2f"%(myid, time()-ttt)
print "On processor %d, we got %d images"%(myid, len(imgdata))
mpi_barrier(MPI_COMM_WORLD)
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
for i in xrange(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])
grp_imgdata = []
for j in xrange(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
alpha, sx, sy, mirror = params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM-phi, 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM-phi), 0.0, 0.0, 1.0)
else:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM-phi), 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM-phi)), 0.0, 0.0, 1.0)
set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
grp_imgdata.append(imgdata[mj])
#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()
if not options.no_norm:
#print grp_imgdata[j].get_xsize()
mask = model_circle(nx/2-2, nx, nx)
for k in xrange(img_per_grp):
ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
del mask
if options.fl > 0.0:
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2*nx
ny2 = 2*ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(fft( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa) ),nx,ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
for k in xrange(img_per_grp):
grp_imgdata[k] = filt_tanl( grp_imgdata[k], options.fl, options.aa)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
from utilities import pad, read_text_file
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2*nx
ny2 = 2*ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d( fft( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1) ) , nx,ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
'''
if i < 10 and myid == main_node:
for k in xrange(10):
grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
'''
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("pp.hdf", pp)
"""
ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("qq.hdf", pp)
"""
var = model_blank(nx,ny)
for q in grp_imgdata: Util.add_img2( var, q )
Util.mul_scalar( var, 1.0/(len(grp_imgdata)-1))
# Switch to std dev
var = square_root(threshold(var))
#if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a")
#else: ave, var = avgvar(grp_imgdata, mode="a")
"""
if myid == main_node:
ave.write_image("avgv.hdf",i)
var.write_image("varv.hdf",i)
"""
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
if options.VERBOSE:
print "%5.2f%% done on processor %d"%(i*100.0/len(proj_list), myid)
if nvec > 0:
eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
for k in xrange(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
del imgdata
# To this point, all averages, variances, and eigenvectors are computed
if options.ave2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(aveList)):
aveList[im].write_image(options.ave2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
"""
nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol=fpol(ave, Tracker["nx"],Tracker["nx"],Tracker["nx"])
tmpvol.write_image(options.ave2D, km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node,im+myid+70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
"""
if options.ave3D:
from fundamentals import fpol
if options.VERBOSE:
print "Reconstructing 3D average volume"
ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
ave3D=fpol(ave3D,Tracker["nx"],Tracker["nx"],Tracker["nx"])
ave3D.write_image(options.ave3D)
print_msg("%-70s: %s\n"%("Writing to the disk volume reconstructed from averages as", options.ave3D))
del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList
if nvec > 0:
for k in xrange(nvec):
if options.VERBOSE:
print "Reconstruction eigenvolumes", k
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image("eig3d_%03d.hdf"%k, ITER)
Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in xrange(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
u = int(u[0])
print " Eigenvector: ",k," number changed ",int(icont[0])
else: u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(varList)):
tmpvol=fpol(varList[im], Tracker["nx"], Tracker["nx"],1)
tmpvol.write_image(options.var2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
tmpvol=fpol(ave, Tracker["nx"], Tracker["nx"],1)
tmpvol.write_image(options.var2D, km)
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(varList)):
send_EMData(varList[im], main_node, im+myid+70000)# What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node and options.VERBOSE:
print "Reconstructing 3D variability volume"
t6 = time()
radiusvar = options.radiusvar
if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid, varList, symmetry=options.sym, npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
res =fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
res.write_image(options.var3D)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Reconstructing 3D variability took [s]", time()-t6))
if options.VERBOSE:
print "Reconstruction took: %.2f [min]"%((time()-t6)/60)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Total time for these computations [s]", time()-t0))
if options.VERBOSE:
print "Total time for these computations: %.2f [min]"%((time()-t0)/60)
print_end_msg("sx3dvariability")
global_def.BATCH = False
from mpi import mpi_finalize
mpi_finalize()
def runcheck(classavgstack,
reconfile,
outdir,
inangles=None,
selectdoc=None,
prjmethod='trilinear',
displayYN=False,
projstack='proj.hdf',
outangles='angles.txt',
outstack='comp-proj-reproj.hdf',
normstack='comp-proj-reproj-norm.hdf'):
print("\n%s, Modified 2018-12-07\n" % __file__)
# Check if inputs exist
check(classavgstack)
check(reconfile)
# Create directory if it doesn't exist
if not os.path.isdir(outdir):
os.makedirs(outdir) # os.mkdir() can only operate one directory deep
print("mkdir -p %s" % outdir)
# Expand path for outputs
projstack = os.path.join(outdir, projstack)
outangles = os.path.join(outdir, outangles)
outstack = os.path.join(outdir, outstack)
normstack = os.path.join(outdir, normstack)
# Get number of images
nimg0 = EMAN2_cppwrap.EMUtil.get_image_count(classavgstack)
recon = EMAN2_cppwrap.EMData(reconfile)
nx = recon.get_xsize()
# In case class averages include discarded images, apply selection file
if selectdoc:
goodavgs, extension = os.path.splitext(classavgstack)
newclasses = goodavgs + "_kept" + extension
# e2proc2d appends to existing files, so rename existing output
if os.path.exists(newclasses):
renamefile = newclasses + '.bak'
os.rename(newclasses, renamefile)
print("mv %s %s" % (newclasses, renamefile))
cmd7 = "e2proc2d.py %s %s --list=%s" % (classavgstack, newclasses,
selectdoc)
print(cmd7)
os.system(cmd7)
# Update class-averages
classavgstack = newclasses
# Import Euler angles
if inangles:
cmd6 = "sxheader.py %s --params=xform.projection --import=%s" % (
classavgstack, inangles)
print(cmd6)
header(classavgstack, 'xform.projection', fimport=inangles)
try:
header(classavgstack, 'xform.projection', fexport=outangles)
cmd1 = "sxheader.py %s --params=xform.projection --export=%s" % (
classavgstack, outangles)
print(cmd1)
except RuntimeError:
print(
"\nERROR!! No projection angles found in class-average stack header!\n"
)
print('Usage:', USAGE)
exit()
#cmd2="sxproject3d.py %s %s --angles=%s" % (recon, projstack, outangles)
#print(cmd2)
#os.system(cmd2)
# Here if you want to be fancy, there should be an option to chose the projection method,
# the mechanism can be copied from sxproject3d.py PAP
if prjmethod == 'trilinear':
method_num = 1
elif prjmethod == 'gridding':
method_num = -1
elif prjmethod == 'nn':
method_num = 0
else:
print(
"\nERROR!! Valid projection methods are: trilinear (default), gridding, and nn (nearest neighbor)."
)
print('Usage:', USAGE)
exit()
#project3d(recon, stack=projstack, listagls=outangles)
recon = prep_vol(recon, npad=2, interpolation_method=1)
result = []
# Here you need actual radius to compute proper ccc's, but if you do, you have to deal with translations, PAP
mask = model_circle(nx // 2 - 2, nx, nx)
# Number of images may have changed
nimg1 = EMAN2_cppwrap.EMUtil.get_image_count(classavgstack)
outangles = read_text_row(outangles)
for imgnum in range(nimg1):
# get class average
classimg = get_im(classavgstack, imgnum)
# compute re-projection
prjimg = prgl(recon, outangles[imgnum], 1, False)
# calculate 1D power spectra
rops_dst = rops_table(classimg * mask)
rops_src = rops_table(prjimg)
# Set power spectrum of reprojection to the data.
# Since data has an envelope, it would make more sense to set data to reconstruction,
# but to do it one would have to know the actual resolution of the data.
# you can check sxprocess.py --adjpw to see how this is done properly PAP
table = [0.0] * len(rops_dst) # initialize table
for j in range(len(rops_dst)):
table[j] = sqrt(rops_dst[j] / rops_src[j])
prjimg = fft(filt_table(
prjimg,
table)) # match FFT amplitdes of re-projection and class average
cccoeff = ccc(prjimg, classimg, mask)
#print(imgnum, cccoeff)
classimg.set_attr_dict({'cross-corr': cccoeff})
prjimg.set_attr_dict({'cross-corr': cccoeff})
prjimg.write_image(outstack, 2 * imgnum)
classimg.write_image(outstack, 2 * imgnum + 1)
result.append(cccoeff)
del outangles
meanccc = sum(result) / nimg1
print("Average CCC is %s" % meanccc)
nimg2 = EMAN2_cppwrap.EMUtil.get_image_count(outstack)
for imgnum in xrange(nimg2):
if (imgnum % 2 == 0):
prjimg = get_im(outstack, imgnum)
meanccc1 = prjimg.get_attr_default('mean-cross-corr', -1.0)
prjimg.set_attr_dict({'mean-cross-corr': meanccc})
write_header(outstack, prjimg, imgnum)
if (imgnum % 100) == 0:
print(imgnum)
# e2proc2d appends to existing files, so delete existing output
if os.path.exists(normstack):
os.remove(normstack)
print("rm %s" % normstack)
# Why would you want to do it? If you do, it should have been done during ccc calculations,
# otherwise what is see is not corresponding to actual data, thus misleading. PAP
#cmd5="e2proc2d.py %s %s --process=normalize" % (outstack, normstack)
#print(cmd5)
#os.system(cmd5)
# Optionally pop up e2display
if displayYN:
cmd8 = "e2display.py %s" % outstack
print(cmd8)
os.system(cmd8)
print("Done!")
