def comp_rep(refrings, data, itout, modout, vol, group, nima, nx, myid,
main_node, outdir):
import os
from fundamentals import rot_shift2D
from utilities import get_params_proj, params_3D_2D
from mpi import mpi_reduce, MPI_COMM_WORLD, MPI_FLOAT, MPI_SUM
avg = [EMData() for i in xrange(len(refrings))]
avg_csum = [0.0 for i in xrange(len(refrings))]
for i in xrange(len(refrings)):
avg[i] = EMData()
avg[i].set_size(nx, nx)
phi = refrings[i].get_attr("phi")
theta = refrings[i].get_attr("theta")
t = Transform({
"type": "spider",
"phi": phi,
"theta": theta,
"psi": 0.0
})
avg[i].set_attr("xform.projection", t)
for im in xrange(nima):
iref = data[im].get_attr("assign")
gim = data[im].get_attr("group")
if gim == group:
[phi, theta, psi, s2x, s2y] = get_params_proj(data[im])
[alpha, sx, sy, mirror] = params_3D_2D(phi, theta, psi, s2x, s2y)
temp = rot_shift2D(data[im], alpha, sx, sy, mirror, 1.0)
avg[iref] = avg[iref] + temp
avg_csum[iref] = avg_csum[iref] + 1
from utilities import reduce_EMData_to_root
for i in xrange(len(refrings)):
reduce_EMData_to_root(avg[i], myid, main_node)
avg_sum = mpi_reduce(avg_csum[i], 1, MPI_FLOAT, MPI_SUM, 0,
MPI_COMM_WORLD)
outfile_repro = os.path.join(outdir,
"repro_%s%s.hdf" % (itout, modout))
if myid == 0:
outfile = os.path.join(outdir,
"compare_repro_%s%s.hdf" % (itout, modout))
avg[i].write_image(outfile, -1)
t = avg[i].get_attr("xform.projection")
proj = vol.project("pawel", t)
proj.set_attr("xform.projection", t)
proj.set_attr("Raw_im_count", float(avg_sum))
proj.write_image(outfile, -1)
proj.write_image(outfile_repro, -1)
return outfile_repro
def rot_shift2D(img, psi, tx=None, ty=None, m=None, out=None):
''' Rotate and shift an image in 2D
* has alternative
:Parameters:
img : array
Image data
psi : float
Inplane rotation
tx : int
Translation in the x-direction
ty : int
Translation in the y-direction
m : bool
True if image should be mirrored around x-axis
out : array
Output array
:Returns:
img : array
Transformed image
'''
if fundamentals is None:
raise ImportError, "EMAN2/Sparx library not available, rot_shift2D requires EMAN2/Sparx"
if tx is None:
#m = psi[1] > 179.9
tx = psi[6]
ty = psi[7]
psi = psi[5]
if out is None: out = img.copy()
emdata = numpy2em(img)
emdata = fundamentals.rot_shift2D(emdata,
psi,
tx,
ty,
m,
interpolation_method="gridding")
out[:, :] = em2numpy(emdata)
return out
def comp_rep(refrings, data, itout, modout, vol, group, nima, nx, myid, main_node, outdir):
import os
from fundamentals import rot_shift2D
from utilities import get_params_proj, params_3D_2D
from mpi import mpi_reduce, MPI_COMM_WORLD, MPI_FLOAT, MPI_SUM
avg = [EMData() for i in xrange(len(refrings))]
avg_csum = [0.0 for i in xrange(len(refrings))]
for i in xrange(len(refrings)):
avg[i] = EMData()
avg[i].set_size(nx,nx)
phi = refrings[i].get_attr("phi")
theta = refrings[i].get_attr("theta")
t = Transform({"type":"spider","phi":phi,"theta":theta,"psi":0.0})
avg[i].set_attr("xform.projection",t)
for im in xrange(nima):
iref = data[im].get_attr("assign")
gim = data[im].get_attr("group")
if gim == group:
[phi, theta, psi, s2x, s2y] = get_params_proj(data[im])
[alpha, sx,sy,mirror] = params_3D_2D(phi,theta,psi,s2x,s2y)
temp = rot_shift2D(data[im],alpha, sx, sy, mirror, 1.0)
avg[iref] = avg[iref] + temp
avg_csum[iref] = avg_csum[iref] + 1
from utilities import reduce_EMData_to_root
for i in xrange(len(refrings)):
reduce_EMData_to_root(avg[i], myid, main_node)
avg_sum = mpi_reduce(avg_csum[i],1,MPI_FLOAT,MPI_SUM,0,MPI_COMM_WORLD)
outfile_repro = os.path.join(outdir, "repro_%s%s.hdf"%(itout,modout))
if myid ==0:
outfile = os.path.join(outdir, "compare_repro_%s%s.hdf"%(itout,modout))
avg[i].write_image(outfile,-1)
t = avg[i].get_attr("xform.projection")
proj = vol.project("pawel",t)
proj.set_attr("xform.projection",t)
proj.set_attr("Raw_im_count", float(avg_sum))
proj.write_image(outfile,-1)
proj.write_image(outfile_repro,-1)
return outfile_repro
def rot_shift2D(img, psi, tx=None, ty=None, m=None, out=None):
''' Rotate and shift an image in 2D
* has alternative
:Parameters:
img : array
Image data
psi : float
Inplane rotation
tx : int
Translation in the x-direction
ty : int
Translation in the y-direction
m : bool
True if image should be mirrored around x-axis
out : array
Output array
:Returns:
img : array
Transformed image
'''
if fundamentals is None: raise ImportError, "EMAN2/Sparx library not available, rot_shift2D requires EMAN2/Sparx"
if tx is None:
#m = psi[1] > 179.9
tx = psi[6]
ty = psi[7]
psi = psi[5]
if out is None: out = img.copy()
emdata = numpy2em(img)
emdata = fundamentals.rot_shift2D(emdata, psi, tx, ty, m, interpolation_method="gridding")
out[:, :] = em2numpy(emdata)
return out
def main():
import sys
import os
import math
import random
import pyemtbx.options
import time
from random import random, seed, randint
from optparse import OptionParser
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Forms chains of 2D images based on their similarities.
Functionality:
4. Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent).
Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers.
The ways to use the program:
4.1 Use option initial to specify which image will be used as an initial seed to form the chain.
sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25
4.2 If options initial is omitted, the program will determine which image best serves as initial seed to form the chain
sxprocess.py input_stack.hdf output_stack.hdf --radius=25
4.3 Use option circular to form a circular chain.
sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25
4.4 New circular code based on pairwise alignments
sxprocess.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt
4.5 Circular ordering based on pairwise alignments
sxprocess.py vols.hdf chain.hdf mask.hdf --dd --radius=25
"""
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False)
parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False)
parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False)
parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering")
# import params for 2D alignment
parser.add_option("--ou", type="int", default=-1, help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)")
parser.add_option("--xr", type="int", default=0, help="range for translation search in x direction, search is +/xr (0)")
parser.add_option("--yr", type="int", default=0, help="range for translation search in y direction, search is +/yr (0)")
#parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)")
parser.add_option("--pairwiseccc", type="string", default= None, help="Input/output pairwise ccc file")
(options, args) = parser.parse_args()
global_def.BATCH = True
if options.dd:
nargs = len(args)
if nargs != 3:
print "must provide name of input and two output files!"
return
stack = args[0]
new_stack = args[1]
from utilities import model_circle
from statistics import ccc
from statistics import mono
lend = EMUtil.get_image_count(stack)
lccc = [None]*(lend*(lend-1)/2)
for i in xrange(lend-1):
v1 = get_im( stack, i )
if( i == 0 and nargs == 2):
nx = v1.get_xsize()
ny = v1.get_ysize()
nz = v1.get_ysize()
if options.ou < 1 : radius = nx//2-2
else: radius = options.ou
mask = model_circle(radius, nx, ny, nz)
else:
mask = get_im(args[2])
for j in xrange(i+1, lend):
lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0]
order = tsp(lccc)
if(len(order) != lend):
print " problem with data length"
from sys import exit
exit()
print "Total sum of cccs :",TotalDistance(order, lccc)
print "ordering :",order
for i in xrange(lend): get_im(stack, order[i]).write_image( new_stack, i )
elif options.align:
nargs = len(args)
if nargs != 3:
print "must provide name of input and two output files!"
return
from utilities import get_params2D, model_circle
from fundamentals import rot_shift2D
from statistics import ccc
from time import time
from alignment import align2d, align2d_scf
from multi_shc import mult_transform
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
"""
# will align anyway
try:
ttt = d[0].get_attr('xform.params2d')
for i in xrange(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
"""
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.ou < 1 : radius = nx//2-2
else: radius = options.ou
mask = model_circle(radius, nx, ny)
if(options.xr < 0): xrng = 0
else: xrng = options.xr
if(options.yr < 0): yrng = xrng
else: yrng = options.yr
initial = max(options.initial, 0)
from statistics import mono
lend = len(d)
lccc = [None]*(lend*(lend-1)/2)
from utilities import read_text_row
if options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) :
st = time()
for i in xrange(lend-1):
for j in xrange(i+1, lend):
# j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger)
#alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F")
alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius)
lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir]
#print " %4d %10.1f"%(i,time()-st)
if(not os.path.exists(options.pairwiseccc)):
from utilities import write_text_row
write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc)
elif(os.path.exists(options.pairwiseccc)):
lccc = read_text_row(options.pairwiseccc)
initial = int(lccc[0][0] + 0.1)
del lccc[0]
for i in xrange(len(lccc)):
T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)})
lccc[i] = [lccc[i][0],T]
tdummy = Transform({"type":"2D"})
maxsum = -1.023
for m in xrange(0,lend):#initial, initial+1):
indc = range( lend )
lsnake = [[m, tdummy, 0.0]]
del indc[m]
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in xrange(len(indc)):
cuc = lccc[mono(indc[i], lsnake[-1][0])][0]
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
# Here we need transformation from the current to the previous,
# meaning indc[i] -> lsnake[-1][0]
T = lccc[mono(indc[i], lsnake[-1][0])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if( indc[i] > lsnake[-1][0] ): T = T.inverse()
lsnake.append([qi,T, maxcit])
lsum += maxcit
del indc[indc.index(qi)]
T = lccc[mono(indc[-1], lsnake[-1][0])][1]
if( indc[-1] > lsnake[-1][0]): T = T.inverse()
lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]])
print " initial image and lsum ",m,lsum
#print lsnake
if(lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in xrange(lend)]
print " Initial image selected : ",init,maxsum," ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc)
#for q in snake: print q
from copy import deepcopy
trans=deepcopy([snake[i][1] for i in xrange(len(snake))])
print [snake[i][0] for i in xrange(len(snake))]
print [snake[i][0] for i in xrange(len(snake))]
"""
for m in xrange(lend):
prms = trans[m].get_params("2D")
print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
"""
for k in xrange(lend-2,0,-1):
T = snake[k][1]
for i in xrange(k+1, lend):
trans[i] = T*trans[i]
# To add - apply all transformations and do the overall centering.
for m in xrange(lend):
prms = trans[m].get_params("2D")
#print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
#rot_shift2D(d[snake[m][0]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image(new_stack, m)
rot_shift2D(d[snake[m][0]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(new_stack, m)
order = tsp(lccc)
if(len(order) != lend):
print " problem with data length"
from sys import exit
exit()
print TotalDistance(order, lccc)
print order
ibeg = order.index(init)
order = [order[(i+ibeg)%lend] for i in xrange(lend)]
print TotalDistance(order, lccc)
print order
snake = [tdummy]
def compare(compare_ref_free, outfile_repro,ref_free_output,yrng, xrng, rstep,nx,apix,ref_free_cutoff, nproc, myid, main_node):
from alignment import Numrinit, ringwe, Applyws
from random import seed, randint
from utilities import get_params2D, set_params2D, model_circle, inverse_transform2, combine_params2
from fundamentals import rot_shift2D
from mpi import MPI_COMM_WORLD, mpi_barrier, mpi_bcast, MPI_INT
from statistics import fsc_mask
from filter import fit_tanh
from numpy import array
fout = "%s.hdf" % ref_free_output
frc_out = "%s_frc" % ref_free_output
res_out = "%s_res" % ref_free_output
nima = EMUtil.get_image_count(compare_ref_free)
image_start, image_end = MPI_start_end(nima, nproc, myid)
ima = EMData()
ima.read_image(compare_ref_free, image_start)
last_ring = nx/2-2
first_ring = 1
mask = model_circle(last_ring, nx, nx)
refi = []
numref = EMUtil.get_image_count(outfile_repro)
cnx = nx/2 +1
cny = cnx
mode = "F"
numr = Numrinit(first_ring, last_ring, rstep, mode)
wr = ringwe(numr, mode)
ima.to_zero()
for j in xrange(numref):
temp = EMData()
temp.read_image(outfile_repro, j)
# even, odd, numer of even, number of images. After frc, totav
refi.append(temp)
# for each node read its share of data
data = EMData.read_images(compare_ref_free, range(image_start, image_end))
for im in xrange(image_start, image_end):
data[im-image_start].set_attr('ID', im)
set_params2D(data[im-image_start],[0,0,0,0,1])
ringref = []
for j in xrange(numref):
refi[j].process_inplace("normalize.mask", {"mask":mask, "no_sigma":1}) # normalize reference images to N(0,1)
cimage = Util.Polar2Dm(refi[j], cnx, cny, numr, mode)
Util.Frngs(cimage, numr)
Applyws(cimage, numr, wr)
ringref.append(cimage)
if myid == main_node: seed(1000)
data_shift = []
frc = []
res = []
for im in xrange(image_start, image_end):
alpha, sx, sy, mirror, scale = get_params2D(data[im-image_start])
alphai, sxi, syi, scalei = inverse_transform2(alpha, sx, sy, 1.0)
# normalize
data[im-image_start].process_inplace("normalize.mask", {"mask":mask, "no_sigma":1}) # subtract average under the mask
# align current image to the reference
[angt, sxst, syst, mirrort, xiref, peakt] = Util.multiref_polar_ali_2d(data[im-image_start], ringref, xrng, yrng, 1, mode, numr, cnx+sxi, cny+syi)
iref = int(xiref)
[alphan, sxn, syn, mn] = combine_params2(0.0, -sxi, -syi, 0, angt, sxst, syst, (int)(mirrort))
set_params2D(data[im-image_start], [alphan, sxn, syn, int(mn), scale])
temp = rot_shift2D(data[im-image_start], alphan, sxn, syn, mn)
temp.set_attr('assign',iref)
tfrc = fsc_mask(temp,refi[iref],mask = mask)
temp.set_attr('frc',tfrc[1])
res = fit_tanh(tfrc)
temp.set_attr('res',res)
data_shift.append(temp)
for node in xrange(nproc):
if myid == node:
for image in data_shift:
image.write_image(fout,-1)
refindex = image.get_attr('assign')
refi[refindex].write_image(fout,-1)
mpi_barrier(MPI_COMM_WORLD)
rejects = []
if myid == main_node:
a = EMData()
index = 0
frc = []
res = []
temp = []
classes = []
for im in xrange(nima):
a.read_image(fout, index)
frc.append(a.get_attr("frc"))
if ref_free_cutoff != -1: classes.append(a.get_attr("class_ptcl_idxs"))
tmp = a.get_attr("res")
temp.append(tmp[0])
res.append("%12f" %(apix/tmp[0]))
res.append("\n")
index = index + 2
res_num = array(temp)
mean_score = res_num.mean(axis=0)
std_score = res_num.std(axis=0)
std = std_score / 2
if ref_free_cutoff !=-1:
cutoff = mean_score - std * ref_free_cutoff
reject = res_num < cutoff
index = 0
for i in reject:
if i: rejects.extend(classes[index])
index = index + 1
rejects.sort()
length = mpi_bcast(len(rejects),1,MPI_INT,main_node, MPI_COMM_WORLD)
rejects = mpi_bcast(rejects,length , MPI_INT, main_node, MPI_COMM_WORLD)
del a
fout_frc = open(frc_out,'w')
fout_res = open(res_out,'w')
fout_res.write("".join(res))
temp = zip(*frc)
datstrings = []
for i in temp:
for j in i:
datstrings.append(" %12f" % (j))
datstrings.append("\n")
fout_frc.write("".join(datstrings))
fout_frc.close()
del refi
del ringref
return rejects
def main(args):
progname = os.path.basename(sys.argv[0])
usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" +
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" +
" --generation=generation --rand_seed=rand_seed>" )
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--radius", type="int", help="particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)")
parser.add_option("--target_radius", type="int", default=29, help="target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)")
parser.add_option("--target_nx", type="int", default=76, help="target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1' (default 76)")
parser.add_option("--img_per_grp", type="int", default=100, help="number of images per class: in the ideal case (essentially maximum size of class) (default 100)")
parser.add_option("--CTF", action="store_true", default=False, help="apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)")
parser.add_option("--ir", type="int", default=1, help="inner ring: of the resampling to polar coordinates. units - pixels (default 1)")
parser.add_option("--rs", type="int", default=1, help="ring step: of the resampling to polar coordinates. units - pixels (default 1)")
parser.add_option("--xr", type="int", default=1, help="x range: of translational search. By default, set by the program. (default 1)")
parser.add_option("--yr", type="int", default=-1, help="y range: of translational search. By default, same as xr. (default -1)")
parser.add_option("--ts", type="float", default=1.0, help="search step: of translational search: units - pixels (default 1.0)")
parser.add_option("--maxit", type="int", default=30, help="number of iterations for reference-free alignment: (default 30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option("--center_method", type="int", default=-1, help="method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)")
parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment: (default 90.0)")
parser.add_option("--FL", type="float", default=0.2, help="lowest stopband: frequency used in the tangent filter (default 0.2)")
parser.add_option("--FH", type="float", default=0.3, help="highest stopband: frequency used in the tangent filter (default 0.3)")
parser.add_option("--FF", type="float", default=0.2, help="fall-off of the tangent filter: (default 0.2)")
parser.add_option("--init_iter", type="int", default=3, help="SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)")
parser.add_option("--main_iter", type="int", default=3, help="SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)")
parser.add_option("--iter_reali", type="int", default=1, help="SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)")
parser.add_option("--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase: (default 1)")
parser.add_option("--max_round", type="int", default=20, help="maximum rounds: of generating candidate class averages in the first phase (default 20)")
parser.add_option("--match_second", type="int", default=5, help="number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)")
parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability: (default 5)")
parser.add_option("--thld_err", type="float", default=0.7, help="threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)")
parser.add_option("--indep_run", type="int", default=4, help="level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching. Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)")
parser.add_option("--thld_grp", type="int", default=10, help="minimum size of reproducible class (default 10)")
parser.add_option("--n_generations", type="int", default=100, help="maximum number of generations: program stops when reaching this total number of generations: (default 100)")
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option("--rand_seed", type="int", help="random seed set before calculations: useful for testing purposes. By default, total randomness (type int)")
parser.add_option("--new", action="store_true", default=False, help="use new code: (default False)")
parser.add_option("--debug", action="store_true", default=False, help="debug info printout: (default False)")
# must be switched off in production
parser.add_option("--use_latest_master_directory",action="store_true", default=False, help="use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)")
parser.add_option("--restart_section", type="string", default=' ', help="restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')")
parser.add_option("--stop_after_candidates", action="store_true", default=False, help="stop after candidates: stops after the 'candidate_class_averages' section. (default False)")
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP)
parser.add_option("--skip_prealignment", action="store_true", default=False, help="skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)")
required_option_list = ['radius']
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
from isac import iter_isac
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
command_line_provided_stack_filename = args[0]
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if(myid == main_node):
print "****************************************************************"
Util.version()
print "****************************************************************"
sys.stdout.flush()
mpi_barrier(MPI_COMM_WORLD)
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
print "\n ==%s== mandatory option is missing.\n"%required_option
print "Please run '" + progname + " -h' for detailed options"
return 1
radi = options.radius
target_radius = options.target_radius
target_nx = options.target_nx
center_method = options.center_method
if(radi < 1): ERROR("Particle radius has to be provided!","sxisac",1,myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
# program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re; r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs)>0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if(myid == main_node):
if( masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime())
masterdir = "master"+timestring
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
if(args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(masterdir, filename )
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_then_all_processes_exit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != " ":
if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(",")
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(generation_str_format)
stored_state[-1]["isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# previous code 2016-05-05--20-14-12-153
# # PARAMETERS OF THE PROCEDURE
# if( options.xr == -1 ):
# # Default values
# # target_nx = 76
# # target_radius = 29
# target_xr = 1
# else: # nx//2
# # Check below!
# target_xr = options.xr
# # target_nx = 76 + target_xr - 1 # subtract one, which is default
# target_nx += target_xr - 1 # subtract one, which is default
# # target_radius = 29
target_xr = options.xr
target_nx += target_xr - 1 # subtract one, which is default
if (options.yr == -1):
yr = options.xr
else:
yr = options.yr
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if(myid == main_node):
print "command_line_provided_stack_filename", command_line_provided_stack_filename
number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir,"2dalignment")
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
if not os.path.exists(os.path.join(init2dir, "Finished_initial_2d_alignment.txt")):
if(myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p "+log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if(myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
image_start, image_end = MPI_start_end(number_of_images_in_stack, nproc, myid)
if options.skip_prealignment:
params2d = [[0.0,0.0,0.0,0] for i in xrange(image_start, image_end)]
else:
original_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end))
# We assume the target radius will be 29, and xr = 1.
shrink_ratio = float(target_radius)/float(radi)
for im in xrange(len(original_images)):
if(shrink_ratio != 1.0):
original_images[im] = resample(original_images[im], shrink_ratio)
nx = original_images[0].get_xsize()
# nx = int(nx*shrink_ratio + 0.5)
txrm = (nx - 2*(target_radius+1))//2
if(txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d"%(radi), "sxisac",1, myid)
if(txrm/nxrsteps>0):
tss = ""
txr = ""
while(txrm/nxrsteps>0):
tts=txrm/nxrsteps
tss += " %d"%tts
txr += " %d"%(tts*nxrsteps)
txrm =txrm//2
else:
tss = "1"
txr = "%d"%txrm
# print "nx, txr, txrm, tss", nx, txr, txrm, tss
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
# section ali2d_base
params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
del original_images
for i in xrange(len(params2d)):
alpha, sx, sy, mirror = combine_params2(0, params2d[i][1],params2d[i][2], 0, -params2d[i][0], 0, 0, 0)
sx /= shrink_ratio
sy /= shrink_ratio
params2d[i][0] = 0.0
params2d[i][1] = sx
params2d[i][2] = sy
params2d[i][3] = 0
#set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0])
mpi_barrier(MPI_COMM_WORLD)
tmp = params2d[:]
tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD)
if( myid == main_node ):
if options.skip_prealignment:
print "========================================="
print "Even though there is no alignment step, '%s' params are set to zero for later use."%os.path.join(init2dir, "initial2Dparams.txt")
print "========================================="
write_text_row(tmp,os.path.join(init2dir, "initial2Dparams.txt"))
del tmp
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius)/float(radi)
aligned_images = EMData.read_images(command_line_provided_stack_filename, range(image_start,image_end))
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx*shrink_ratio + 0.5)
while not os.path.exists(os.path.join(init2dir, "initial2Dparams.txt")):
import time
time.sleep(1)
mpi_barrier(MPI_COMM_WORLD)
params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt"))
params = params[image_start:image_end]
msk = model_circle(radi, nx, nx)
for im in xrange(nima):
st = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= st[0]
if options.CTF:
aligned_images[im] = filt_ctf(aligned_images[im], aligned_images[im].get_attr("ctf"), binary = True)
if(shrink_ratio < 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
#aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio == 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
#aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio > 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, params[im][1], params[im][2], 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid)
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
if( myid == main_node ):
for i in range(number_of_images_in_stack): aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
fp = open(os.path.join(masterdir,"README_shrink_ratio.txt"), "w")
output_text = """
Since, for processing purposes, isac changes the image dimensions,
adjustment of pixel size needs to be made in subsequent steps, (e.g.
running sxviper.py). The shrink ratio for this particular isac run is
--------
%.5f
%.5f
--------
To get the pixel size for the isac output the user needs to divide
the original pixel size by the above value. This info is saved in
the following file: README_shrink_ratio.txt
"""%(shrink_ratio, radi)
fp.write(output_text); fp.flush() ;fp.close()
print output_text
fp = open(os.path.join(init2dir, "Finished_initial_2d_alignment.txt"), "w"); fp.flush() ;fp.close()
else:
if( myid == main_node ):
print "Skipping 2d alignment since it was already done!"
mpi_barrier(MPI_COMM_WORLD)
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
cmdexecute("sxheader.py --consecutive --params=originalid %s"%stack_processed_by_ali2d_base__filename__without_master_dir)
cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1):
if i == isac_generation_from_command_line+1:
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%i + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line )
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line)
# cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line)
cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation))
write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
# reference the original stack
list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1))
cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if(myid == main_node):
if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation):
print "This generation (%d) no class average candidates were generated! Finishing."%isac_generation
exit_program = 1
exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if(myid == main_node):
accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_accounted.txt"%(isac_generation)))
number_of_accounted_images = len(accounted_images)
un_accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation),"generation_%d_unaccounted.txt"%(isac_generation)))
number_of_un_accounted_images = len(un_accounted_images)
else:
number_of_accounted_images = 0
number_of_un_accounted_images = 0
number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
number_of_un_accounted_images = int(mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
if(myid == main_node):
print "This generation (%d) there are no accounted images! Finishing."%isac_generation
break
while (myid == main_node):
def files_are_missing(isac_generation):
for i in xrange(1, isac_generation + 1):
if not os.path.exists("generation_%04d/class_averages_generation_%d.hdf"%(i,i)):
print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting."%(i,i)
return 1
return 0
if files_are_missing(isac_generation):
break
cmdexecute("rm -f class_averages.hdf")
cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation + 1)], "class_averages.hdf")
break
if number_of_un_accounted_images == 0:
if(myid == main_node):
print "This generation (%d) there are no un accounted images! Finishing."%isac_generation
break
program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall")
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def shiftali_MPI(stack,
maskfile=None,
maxit=100,
CTF=False,
snr=1.0,
Fourvar=False,
search_rng=-1,
oneDx=False,
search_rng_y=-1):
from applications import MPI_start_end
from utilities import model_circle, model_blank, get_image, peak_search, get_im
from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
from pap_statistics import varf2d_MPI
from fundamentals import fft, ccf, rot_shift3D, rot_shift2D
from utilities import get_params2D, set_params2D
from utilities import print_msg, print_begin_msg, print_end_msg
import os
import sys
from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
from EMAN2 import Processor
from time import time
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("shiftali_MPI")
max_iter = int(maxit)
if myid == main_node:
if ftp == "bdb":
from EMAN2db import db_open_dict
dummy = db_open_dict(stack, True)
nima = EMUtil.get_image_count(stack)
else:
nima = 0
nima = bcast_number_to_all(nima, source_node=main_node)
list_of_particles = list(range(nima))
image_start, image_end = MPI_start_end(nima, number_of_proc, myid)
list_of_particles = list_of_particles[image_start:image_end]
# read nx and ctf_app (if CTF) and broadcast to all nodes
if myid == main_node:
ima = EMData()
ima.read_image(stack, list_of_particles[0], True)
nx = ima.get_xsize()
ny = ima.get_ysize()
if CTF: ctf_app = ima.get_attr_default('ctf_applied', 2)
del ima
else:
nx = 0
ny = 0
if CTF: ctf_app = 0
nx = bcast_number_to_all(nx, source_node=main_node)
ny = bcast_number_to_all(ny, source_node=main_node)
if CTF:
ctf_app = bcast_number_to_all(ctf_app, source_node=main_node)
if ctf_app > 0:
ERROR("data cannot be ctf-applied", "shiftali_MPI", 1, myid)
if maskfile == None:
mrad = min(nx, ny)
mask = model_circle(mrad // 2 - 2, nx, ny)
else:
mask = get_im(maskfile)
if CTF:
from filter import filt_ctf
from morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
from global_def import CACHE_DISABLE
if CACHE_DISABLE:
data = EMData.read_images(stack, list_of_particles)
else:
for i in range(number_of_proc):
if myid == i:
data = EMData.read_images(stack, list_of_particles)
if ftp == "bdb": mpi_barrier(MPI_COMM_WORLD)
for im in range(len(data)):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
else:
ctf_2_sum = None
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
for i in range(0, nx, 2):
for j in range(ny):
temp.set_value_at(i, j, snr)
Util.add_img(ctf_2_sum, temp)
del temp
total_iter = 0
# apply initial xform.align2d parameters stored in header
init_params = []
for im in range(len(data)):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im],
p['alpha'],
sx=p['tx'],
sy=p['ty'],
mirror=p['mirror'],
scale=p['scale'])
# fourier transform all images, and apply ctf if CTF
for im in range(len(data)):
if CTF:
ctf_params = data[im].get_attr("ctf")
data[im] = filt_ctf(fft(data[im]), ctf_params)
else:
data[im] = fft(data[im])
sx_sum = 0
sy_sum = 0
sx_sum_total = 0
sy_sum_total = 0
shift_x = [0.0] * len(data)
shift_y = [0.0] * len(data)
ishift_x = [0.0] * len(data)
ishift_y = [0.0] * len(data)
for Iter in range(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n" % (total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in data:
Util.add_img(avg, im)
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF:
tavg = Util.divn_filter(avg, ctf_2_sum)
else:
tavg = Util.mult_scalar(avg, 1.0 / float(nima))
else:
tavg = EMData(nx, ny, 1, False)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask(tavg, mask, False)
tavg -= stat[0]
Util.mul_img(tavg, mask)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
tavg = fft(tavg)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
sx_sum = 0
sy_sum = 0
if search_rng > 0: nwx = 2 * search_rng + 1
else: nwx = nx
if search_rng_y > 0: nwy = 2 * search_rng_y + 1
else: nwy = ny
not_zero = 0
for im in range(len(data)):
if oneDx:
ctx = Util.window(ccf(data[im], tavg), nwx, 1)
p1 = peak_search(ctx)
p1_x = -int(p1[0][3])
ishift_x[im] = p1_x
sx_sum += p1_x
else:
p1 = peak_search(Util.window(ccf(data[im], tavg), nwx, nwy))
p1_x = -int(p1[0][4])
p1_y = -int(p1[0][5])
ishift_x[im] = p1_x
ishift_y[im] = p1_y
sx_sum += p1_x
sy_sum += p1_y
if not_zero == 0:
if (not (ishift_x[im] == 0.0)) or (not (ishift_y[im] == 0.0)):
not_zero = 1
sx_sum = mpi_reduce(sx_sum, 1, MPI_INT, MPI_SUM, main_node,
MPI_COMM_WORLD)
if not oneDx:
sy_sum = mpi_reduce(sy_sum, 1, MPI_INT, MPI_SUM, main_node,
MPI_COMM_WORLD)
if myid == main_node:
sx_sum_total = int(sx_sum[0])
if not oneDx:
sy_sum_total = int(sy_sum[0])
else:
sx_sum_total = 0
sy_sum_total = 0
sx_sum_total = bcast_number_to_all(sx_sum_total, source_node=main_node)
if not oneDx:
sy_sum_total = bcast_number_to_all(sy_sum_total,
source_node=main_node)
sx_ave = round(float(sx_sum_total) / nima)
sy_ave = round(float(sy_sum_total) / nima)
for im in range(len(data)):
p1_x = ishift_x[im] - sx_ave
p1_y = ishift_y[im] - sy_ave
params2 = {
"filter_type": Processor.fourier_filter_types.SHIFT,
"x_shift": p1_x,
"y_shift": p1_y,
"z_shift": 0.0
}
data[im] = Processor.EMFourierFilter(data[im], params2)
shift_x[im] += p1_x
shift_y[im] += p1_y
# stop if all shifts are zero
not_zero = mpi_reduce(not_zero, 1, MPI_INT, MPI_SUM, main_node,
MPI_COMM_WORLD)
if myid == main_node:
not_zero_all = int(not_zero[0])
else:
not_zero_all = 0
not_zero_all = bcast_number_to_all(not_zero_all, source_node=main_node)
if myid == main_node:
print_msg("Time of iteration = %12.2f\n" % (time() - start_time))
start_time = time()
if not_zero_all == 0: break
#for im in xrange(len(data)): data[im] = fft(data[im]) This should not be required as only header information is used
# combine shifts found with the original parameters
for im in range(len(data)):
t0 = init_params[im]
t1 = Transform()
t1.set_params({
"type": "2D",
"alpha": 0,
"scale": t0.get_scale(),
"mirror": 0,
"tx": shift_x[im],
"ty": shift_y[im]
})
# combine t0 and t1
tt = t1 * t0
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from utilities import file_type
if (file_type(stack) == "bdb"):
from utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, image_start,
image_end, number_of_proc)
else:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, image_start,
image_end, number_of_proc)
else:
send_attr_dict(main_node, data, par_str, image_start, image_end)
if myid == main_node: print_end_msg("shiftali_MPI")
def shiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1, oneDx=False, search_rng_y=-1):
from applications import MPI_start_end
from utilities import model_circle, model_blank, get_image, peak_search, get_im
from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
from statistics import varf2d_MPI
from fundamentals import fft, ccf, rot_shift3D, rot_shift2D
from utilities import get_params2D, set_params2D
from utilities import print_msg, print_begin_msg, print_end_msg
import os
import sys
from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
from EMAN2 import Processor
from time import time
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("shiftali_MPI")
max_iter=int(maxit)
if myid == main_node:
if ftp == "bdb":
from EMAN2db import db_open_dict
dummy = db_open_dict(stack, True)
nima = EMUtil.get_image_count(stack)
else:
nima = 0
nima = bcast_number_to_all(nima, source_node = main_node)
list_of_particles = range(nima)
image_start, image_end = MPI_start_end(nima, number_of_proc, myid)
list_of_particles = list_of_particles[image_start: image_end]
# read nx and ctf_app (if CTF) and broadcast to all nodes
if myid == main_node:
ima = EMData()
ima.read_image(stack, list_of_particles[0], True)
nx = ima.get_xsize()
ny = ima.get_ysize()
if CTF: ctf_app = ima.get_attr_default('ctf_applied', 2)
del ima
else:
nx = 0
ny = 0
if CTF: ctf_app = 0
nx = bcast_number_to_all(nx, source_node = main_node)
ny = bcast_number_to_all(ny, source_node = main_node)
if CTF:
ctf_app = bcast_number_to_all(ctf_app, source_node = main_node)
if ctf_app > 0: ERROR("data cannot be ctf-applied", "shiftali_MPI", 1, myid)
if maskfile == None:
mrad = min(nx, ny)
mask = model_circle(mrad//2-2, nx, ny)
else:
mask = get_im(maskfile)
if CTF:
from filter import filt_ctf
from morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
from global_def import CACHE_DISABLE
if CACHE_DISABLE:
data = EMData.read_images(stack, list_of_particles)
else:
for i in xrange(number_of_proc):
if myid == i:
data = EMData.read_images(stack, list_of_particles)
if ftp == "bdb": mpi_barrier(MPI_COMM_WORLD)
for im in xrange(len(data)):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
else: ctf_2_sum = None
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
for i in xrange(0,nx,2):
for j in xrange(ny):
temp.set_value_at(i,j,snr)
Util.add_img(ctf_2_sum, temp)
del temp
total_iter = 0
# apply initial xform.align2d parameters stored in header
init_params = []
for im in xrange(len(data)):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im], p['alpha'], sx=p['tx'], sy=p['ty'], mirror=p['mirror'], scale=p['scale'])
# fourier transform all images, and apply ctf if CTF
for im in xrange(len(data)):
if CTF:
ctf_params = data[im].get_attr("ctf")
data[im] = filt_ctf(fft(data[im]), ctf_params)
else:
data[im] = fft(data[im])
sx_sum=0
sy_sum=0
sx_sum_total=0
sy_sum_total=0
shift_x = [0.0]*len(data)
shift_y = [0.0]*len(data)
ishift_x = [0.0]*len(data)
ishift_y = [0.0]*len(data)
for Iter in xrange(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n"%(total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in data: Util.add_img(avg, im)
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF:
tavg = Util.divn_filter(avg, ctf_2_sum)
else: tavg = Util.mult_scalar(avg, 1.0/float(nima))
else:
tavg = EMData(nx, ny, 1, False)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask( tavg, mask, False )
tavg -= stat[0]
Util.mul_img(tavg, mask)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
tavg = fft(tavg)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
sx_sum=0
sy_sum=0
if search_rng > 0: nwx = 2*search_rng+1
else: nwx = nx
if search_rng_y > 0: nwy = 2*search_rng_y+1
else: nwy = ny
not_zero = 0
for im in xrange(len(data)):
if oneDx:
ctx = Util.window(ccf(data[im],tavg),nwx,1)
p1 = peak_search(ctx)
p1_x = -int(p1[0][3])
ishift_x[im] = p1_x
sx_sum += p1_x
else:
p1 = peak_search(Util.window(ccf(data[im],tavg), nwx,nwy))
p1_x = -int(p1[0][4])
p1_y = -int(p1[0][5])
ishift_x[im] = p1_x
ishift_y[im] = p1_y
sx_sum += p1_x
sy_sum += p1_y
if not_zero == 0:
if (not(ishift_x[im] == 0.0)) or (not(ishift_y[im] == 0.0)):
not_zero = 1
sx_sum = mpi_reduce(sx_sum, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if not oneDx:
sy_sum = mpi_reduce(sy_sum, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
sx_sum_total = int(sx_sum[0])
if not oneDx:
sy_sum_total = int(sy_sum[0])
else:
sx_sum_total = 0
sy_sum_total = 0
sx_sum_total = bcast_number_to_all(sx_sum_total, source_node = main_node)
if not oneDx:
sy_sum_total = bcast_number_to_all(sy_sum_total, source_node = main_node)
sx_ave = round(float(sx_sum_total)/nima)
sy_ave = round(float(sy_sum_total)/nima)
for im in xrange(len(data)):
p1_x = ishift_x[im] - sx_ave
p1_y = ishift_y[im] - sy_ave
params2 = {"filter_type" : Processor.fourier_filter_types.SHIFT, "x_shift" : p1_x, "y_shift" : p1_y, "z_shift" : 0.0}
data[im] = Processor.EMFourierFilter(data[im], params2)
shift_x[im] += p1_x
shift_y[im] += p1_y
# stop if all shifts are zero
not_zero = mpi_reduce(not_zero, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
not_zero_all = int(not_zero[0])
else:
not_zero_all = 0
not_zero_all = bcast_number_to_all(not_zero_all, source_node = main_node)
if myid == main_node:
print_msg("Time of iteration = %12.2f\n"%(time()-start_time))
start_time = time()
if not_zero_all == 0: break
#for im in xrange(len(data)): data[im] = fft(data[im]) This should not be required as only header information is used
# combine shifts found with the original parameters
for im in xrange(len(data)):
t0 = init_params[im]
t1 = Transform()
t1.set_params({"type":"2D","alpha":0,"scale":t0.get_scale(),"mirror":0,"tx":shift_x[im],"ty":shift_y[im]})
# combine t0 and t1
tt = t1*t0
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from utilities import file_type
if(file_type(stack) == "bdb"):
from utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, image_start, image_end, number_of_proc)
else:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, image_start, image_end, number_of_proc)
else: send_attr_dict(main_node, data, par_str, image_start, image_end)
if myid == main_node: print_end_msg("shiftali_MPI")
def helicalshiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1):
from applications import MPI_start_end
from utilities import model_circle, model_blank, get_image, peak_search, get_im, pad
from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
from statistics import varf2d_MPI
from fundamentals import fft, ccf, rot_shift3D, rot_shift2D, fshift
from utilities import get_params2D, set_params2D, chunks_distribution
from utilities import print_msg, print_begin_msg, print_end_msg
import os
import sys
from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
from time import time
from pixel_error import ordersegments
from math import sqrt, atan2, tan, pi
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("helical-shiftali_MPI")
max_iter=int(maxit)
if( myid == main_node):
infils = EMUtil.get_all_attributes(stack, "filament")
ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord')
filaments = ordersegments(infils, ptlcoords)
total_nfils = len(filaments)
inidl = [0]*total_nfils
for i in xrange(total_nfils): inidl[i] = len(filaments[i])
linidl = sum(inidl)
nima = linidl
tfilaments = []
for i in xrange(total_nfils): tfilaments += filaments[i]
del filaments
else:
total_nfils = 0
linidl = 0
total_nfils = bcast_number_to_all(total_nfils, source_node = main_node)
if myid != main_node:
inidl = [-1]*total_nfils
inidl = bcast_list_to_all(inidl, myid, source_node = main_node)
linidl = bcast_number_to_all(linidl, source_node = main_node)
if myid != main_node:
tfilaments = [-1]*linidl
tfilaments = bcast_list_to_all(tfilaments, myid, source_node = main_node)
filaments = []
iendi = 0
for i in xrange(total_nfils):
isti = iendi
iendi = isti+inidl[i]
filaments.append(tfilaments[isti:iendi])
del tfilaments,inidl
if myid == main_node:
print_msg( "total number of filaments: %d"%total_nfils)
if total_nfils< nproc:
ERROR('number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'%(nproc, total_nfils), "ehelix_MPI", 1,myid)
# balanced load
temp = chunks_distribution([[len(filaments[i]), i] for i in xrange(len(filaments))], nproc)[myid:myid+1][0]
filaments = [filaments[temp[i][1]] for i in xrange(len(temp))]
nfils = len(filaments)
#filaments = [[0,1]]
#print "filaments",filaments
list_of_particles = []
indcs = []
k = 0
for i in xrange(nfils):
list_of_particles += filaments[i]
k1 = k+len(filaments[i])
indcs.append([k,k1])
k = k1
data = EMData.read_images(stack, list_of_particles)
ldata = len(data)
print "ldata=", ldata
nx = data[0].get_xsize()
ny = data[0].get_ysize()
if maskfile == None:
mrad = min(nx, ny)//2-2
mask = pad( model_blank(2*mrad+1, ny, 1, 1.0), nx, ny, 1, 0.0)
else:
mask = get_im(maskfile)
# apply initial xform.align2d parameters stored in header
init_params = []
for im in xrange(ldata):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'], p['mirror'], p['scale'])
if CTF:
from filter import filt_ctf
from morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
ctf_abs_sum = None
from utilities import info
for im in xrange(ldata):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
qctf = data[im].get_attr("ctf_applied")
if qctf == 0:
data[im] = filt_ctf(fft(data[im]), ctf_params)
data[im].set_attr('ctf_applied', 1)
elif qctf != 1:
ERROR('Incorrectly set qctf flag', "helicalshiftali_MPI", 1,myid)
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
else: data[im] = fft(data[im])
del list_of_particles
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
tsnr = 1./snr
for i in xrange(0,nx+2,2):
for j in xrange(ny):
temp.set_value_at(i,j,tsnr)
temp.set_value_at(i+1,j,0.0)
#info(ctf_2_sum)
Util.add_img(ctf_2_sum, temp)
#info(ctf_2_sum)
del temp
total_iter = 0
shift_x = [0.0]*ldata
for Iter in xrange(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n"%(total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in xrange(ldata):
Util.add_img(avg, fshift(data[im], shift_x[im]))
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF: tavg = Util.divn_filter(avg, ctf_2_sum)
else: tavg = Util.mult_scalar(avg, 1.0/float(nima))
else:
tavg = model_blank(nx,ny)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask( tavg, mask, False )
tavg -= stat[0]
Util.mul_img(tavg, mask)
tavg.write_image("tavg.hdf",Iter)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
tavg = fft(tavg)
sx_sum = 0.0
nxc = nx//2
for ifil in xrange(nfils):
"""
# Calculate filament average
avg = EMData(nx, ny, 1, False)
filnima = 0
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
Util.add_img(avg, data[im])
filnima += 1
tavg = Util.mult_scalar(avg, 1.0/float(filnima))
"""
# Calculate 1D ccf between each segment and filament average
nsegms = indcs[ifil][1]-indcs[ifil][0]
ctx = [None]*nsegms
pcoords = [None]*nsegms
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
ctx[im-indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx, 1)
pcoords[im-indcs[ifil][0]] = data[im].get_attr('ptcl_source_coord')
#ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0])
#print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True)
# search for best x-shift
cents = nsegms//2
dst = sqrt(max((pcoords[cents][0] - pcoords[0][0])**2 + (pcoords[cents][1] - pcoords[0][1])**2, (pcoords[cents][0] - pcoords[-1][0])**2 + (pcoords[cents][1] - pcoords[-1][1])**2))
maxincline = atan2(ny//2-2-float(search_rng),dst)
kang = int(dst*tan(maxincline)+0.5)
#print " settings ",nsegms,cents,dst,search_rng,maxincline,kang
# ## C code for alignment. @ming
results = [0.0]*3;
results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline, kang, search_rng,nxc)
sib = int(results[0])
bang = results[1]
qm = results[2]
#print qm, sib, bang
# qm = -1.e23
#
# for six in xrange(-search_rng, search_rng+1,1):
# q0 = ctx[cents].get_value_at(six+nxc)
# for incline in xrange(kang+1):
# qt = q0
# qu = q0
# if(kang>0): tang = tan(maxincline/kang*incline)
# else: tang = 0.0
# for kim in xrange(cents+1,nsegms):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# #print " A ", ifil,six,incline,kim,xl,ixl,dxl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# for kim in xrange(cents):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# if( qt > qm ):
# qm = qt
# sib = six
# bang = tang
# if( qu > qm ):
# qm = qu
# sib = six
# bang = -tang
#if incline == 0: print "incline = 0 ",six,tang,qt,qu
#print qm,six,sib,bang
#print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True)
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
kim = im-indcs[ifil][0]
dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
if(kim < cents): xl = -dst*bang+sib
else: xl = dst*bang+sib
shift_x[im] = xl
# Average shift
sx_sum += shift_x[indcs[ifil][0]+cents]
# #print myid,sx_sum,total_nfils
sx_sum = mpi_reduce(sx_sum, 1, MPI_FLOAT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
sx_sum = float(sx_sum[0])/total_nfils
print_msg("Average shift %6.2f\n"%(sx_sum))
else:
sx_sum = 0.0
sx_sum = 0.0
sx_sum = bcast_number_to_all(sx_sum, source_node = main_node)
for im in xrange(ldata):
shift_x[im] -= sx_sum
#print " %3d %6.3f"%(im,shift_x[im])
#exit()
# combine shifts found with the original parameters
for im in xrange(ldata):
t1 = Transform()
##import random
##shix=random.randint(-10, 10)
##t1.set_params({"type":"2D","tx":shix})
t1.set_params({"type":"2D","tx":shift_x[im]})
# combine t0 and t1
tt = t1*init_params[im]
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from utilities import file_type
if(file_type(stack) == "bdb"):
from utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata, nproc)
else:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc)
else: send_attr_dict(main_node, data, par_str, 0, ldata)
if myid == main_node: print_end_msg("helical-shiftali_MPI")
def main(args):
progname = os.path.basename(sys.argv[0])
usage = ( progname + " stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter" +
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp" +
" --generation=generation --rand_seed=rand_seed>" )
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--radius", type="int", default=-1, help="<Particle radius>, it has to be provided.")
parser.add_option("--img_per_grp", type="int", default=100, help="<number of images per group> in the ideal case (essentially maximum size of class) (100)")
parser.add_option("--CTF", action="store_true", default=False, help="<CTF flag>, if set the data will be phase-flipped")
parser.add_option("--ir", type="int", default=1, help="<inner ring> of the resampling to polar coordinates (1)")
parser.add_option("--rs", type="int", default=1, help="<ring step> of the resampling to polar coordinates (1)")
parser.add_option("--xr", type="int", default=-1, help="<x range> of translational search (By default set by the program) (advanced)")
parser.add_option("--yr", type="int", default=-1, help="<y range> of translational search (same as xr) (advanced)")
parser.add_option("--ts", type="float", default=1.0, help="<search step> of translational search (1.0)")
parser.add_option("--maxit", type="int", default=30, help="number of iterations for reference-free alignment (30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option("--center_method", type="int", default=7, help="<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)")
parser.add_option("--dst", type="float", default=90.0, help="discrete angle used in within group alignment ")
parser.add_option("--FL", type="float", default=0.2, help="<lowest stopband> frequency used in the tangent filter (0.2)")
parser.add_option("--FH", type="float", default=0.3, help="<highest stopband> frequency used in the tangent filter (0.3)")
parser.add_option("--FF", type="float", default=0.2, help="<fall-off of the tangent> filter (0.2)")
parser.add_option("--init_iter", type="int", default=3, help="<init_iter> number of iterations of ISAC program in initialization (3)")
parser.add_option("--main_iter", type="int", default=3, help="<main_iter> number of iterations of ISAC program in main part (3)")
parser.add_option("--iter_reali", type="int", default=1, help="<iter_reali> number of iterations in ISAC before checking stability (1)")
parser.add_option("--match_first", type="int", default=1, help="number of iterations to run 2-way matching in the first phase (1)")
parser.add_option("--max_round", type="int", default=20, help="maximum rounds of generating candidate averages in the first phase (20)")
parser.add_option("--match_second", type="int", default=5, help="number of iterations to run 2-way (or 3-way) matching in the second phase (5)")
parser.add_option("--stab_ali", type="int", default=5, help="number of alignments when checking stability (5)")
parser.add_option("--thld_err", type="float", default=0.7, help="the threshold of pixel error when checking stability (0.7)")
parser.add_option("--indep_run", type="int", default=4, help="number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)")
parser.add_option("--thld_grp", type="int", default=10, help="minimum size of class (10)")
parser.add_option("--n_generations", type="int", default=100, help="<n_generations> program stops when reaching this total number of generations (advanced)")
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option("--rand_seed", type="int", default=None, help="random seed set before calculations, useful for testing purposes (default None - total randomness)")
parser.add_option("--new", action="store_true", default=False, help="use new code (default = False)")
parser.add_option("--debug", action="store_true", default=False, help="debug info printout (default = False)")
# must be switched off in production
parser.add_option("--use_latest_master_directory", action="store_true", dest="use_latest_master_directory", default=False)
parser.add_option("--restart_section", type="string", default="", help="<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)")
parser.add_option("--stop_after_candidates", action="store_true", default=False, help="<stop_after_candidates> stops after the 'candidate_class_averages' section")
parser.add_option("--return_options", action="store_true", dest="return_options", default=False, help = SUPPRESS_HELP)
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from isac import iter_isac
global_def.BATCH = True
global_def.BATCH = True
command_line_provided_stack_filename = args[0]
global_def.BATCH = True
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
radi = options.radius
center_method = options.center_method
if(radi < 1): ERROR("Particle radius has to be provided!","sxisac",1,myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re; r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs)>0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if(myid == main_node):
if( masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM, localtime())
masterdir = "master"+timestring
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
if(args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(masterdir, filename )
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_all_processes_quit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != "":
if os.path.exists(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(",")
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%parsed_restart_section_option[0], stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(generation_str_format)
stored_state[-1]["isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(r"___.*$", "___%s"%options.restart_section, stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(os.path.join(masterdir,NAME_OF_JSON_STATE_FILE), stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()), os.path.join(masterdir,NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# PARAMETERS OF THE PROCEDURE
if( options.xr == -1 ):
# Default values
target_nx = 76
target_radius = 29
target_xr = 1
else: # nx//2
# Check below!
target_xr = options.xr
target_nx = 76 + target_xr - 1 # subtract one, which is default
target_radius = 29
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if(myid == main_node):
number_of_images_in_stack = EMUtil.get_image_count(command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir,"2dalignment")
if(myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p "+log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if(myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
txrm = (nnxo - 2*(radi+1))//2
if(txrm < 0): ERROR( "ERROR!! Radius of the structure larger than the window data size permits %d"%(radi), "sxisac",1, myid)
if(txrm/nxrsteps>0):
tss = ""
txr = ""
while(txrm/nxrsteps>0):
tts=txrm/nxrsteps
tss += " %d"%tts
txr += " %d"%(tts*nxrsteps)
txrm =txrm//2
else:
tss = "1"
txr = "%d"%txrm
# section ali2d_base
# centering method is set to #7
params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
if( myid == main_node ):
write_text_row(params2d,os.path.join(init2dir, "initial2Dparams.txt"))
del params2d
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius)/float(radi)
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx*shrink_ratio + 0.5)
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
if(shrink_ratio < 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(newx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio == 1.0):
if newx > target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = Util.window(aligned_images[im], target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx :
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx :
msk = model_circle(nx//2-2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
#aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
elif(shrink_ratio > 1.0):
target_radius = radi
msk = model_circle(target_radius, nx, nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx, target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images, myid)
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack, source_node = main_node)
if( myid == main_node ):
for i in range(number_of_images_in_stack): aligned_images[i].write_image(stack_processed_by_ali2d_base__filename,i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
fp = open("README_shrink_ratio.txt", "w")
output_text = """
Since, for processing purposes, isac changes the image dimensions,
adjustment of pixel size needs to be made in subsequent steps, (e.g.
running sxviper.py). The shrink ratio for this particular isac run is
--------
%.5f
--------
To get the pixel size for the isac output the user needs to divide
the original pixel size by the above value. This info is saved in
the following file: README_shrink_ratio.txt
"""%shrink_ratio
fp.write(output_text); fp.flush() ;fp.close()
print output_text
mpi_barrier(MPI_COMM_WORLD)
global_def.BATCH = True
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
cmdexecute("sxheader.py --consecutive --params=originalid %s"%stack_processed_by_ali2d_base__filename__without_master_dir)
cmdexecute("e2bdb.py %s --makevstack=%s_000"%(stack_processed_by_ali2d_base__filename__without_master_dir, stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./", NAME_OF_MAIN_DIR, myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line+1, main_dir_no + 1):
if i == isac_generation_from_command_line+1:
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%i + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split("___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("rm " + "EMAN2DB/"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d.bdb"%isac_generation_from_command_line )
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value("./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" + "%05d"%backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line + " 000_backup" + "%05d"%backup_dir_no)
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation_from_command_line)
# cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
cmdexecute("rm -rf " + NAME_OF_MAIN_DIR + "%04d/ali_params_generation_*"%isac_generation_from_command_line)
cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_generation*"%isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
write_text_file([1], os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_accounted.txt"%isac_generation))
write_text_file(range(number_of_images_in_stack), os.path.join(NAME_OF_MAIN_DIR + "%04d"%isac_generation, "generation_%d_unaccounted.txt"%isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../"+stack_processed_by_ali2d_base__filename__without_master_dir[4:]+"_%03d"%isac_generation
if(myid == main_node):
accounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_accounted.txt"%(isac_generation - 1)))
number_of_accounted_images = len(accounted_images)
# unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1)))
# number_of_unaccounted_images = len(unaccounted_images)
else:
number_of_accounted_images = 0
number_of_accounted_images = int(mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
os.chdir("..")
break
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR + "%04d"%isac_generation)
# reference the original stack
list_file = os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1), "generation_%d_unaccounted.txt"%(isac_generation - 1))
cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d"%isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if(myid == main_node):
if not os.path.exists("class_averages_candidate_generation_%d.hdf"%isac_generation):
print "This generation (%d) no class averages were generated!"%isac_generation
exit_program = 1
exit_program = int(mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if (myid == main_node):
cmdexecute("rm -f class_averages.hdf")
cpy(["generation_%04d/class_averages_generation_%d.hdf"%(i,i) for i in xrange(1, isac_generation)], "class_averages.hdf")
# program_state_stack.restart_location_title = "stopped_program2"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack(locals(), getframeinfo(currentframe()), last_call="__LastCall")
mpi_finalize()
def main(args):
progname = os.path.basename(sys.argv[0])
usage = (
progname +
" stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter"
+
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp"
+ " --generation=generation --rand_seed=rand_seed>")
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--radius",
type="int",
help=
"particle radius: there is no default, a sensible number has to be provided, units - pixels (default required int)"
)
parser.add_option(
"--target_radius",
type="int",
default=29,
help=
"target particle radius: actual particle radius on which isac will process data. Images will be shrinked/enlarged to achieve this radius (default 29)"
)
parser.add_option(
"--target_nx",
type="int",
default=76,
help=
"target particle image size: actual image size on which isac will process data. Images will be shrinked/enlarged according to target particle radius and then cut/padded to achieve target_nx size. When xr > 0, the final image size for isac processing is 'target_nx + xr - 1' (default 76)"
)
parser.add_option(
"--img_per_grp",
type="int",
default=100,
help=
"number of images per class: in the ideal case (essentially maximum size of class) (default 100)"
)
parser.add_option(
"--CTF",
action="store_true",
default=False,
help=
"apply phase-flip for CTF correction: if set the data will be phase-flipped using CTF information included in image headers (default False)"
)
parser.add_option(
"--ir",
type="int",
default=1,
help=
"inner ring: of the resampling to polar coordinates. units - pixels (default 1)"
)
parser.add_option(
"--rs",
type="int",
default=1,
help=
"ring step: of the resampling to polar coordinates. units - pixels (default 1)"
)
parser.add_option(
"--xr",
type="int",
default=1,
help=
"x range: of translational search. By default, set by the program. (default 1)"
)
parser.add_option(
"--yr",
type="int",
default=-1,
help=
"y range: of translational search. By default, same as xr. (default -1)"
)
parser.add_option(
"--ts",
type="float",
default=1.0,
help=
"search step: of translational search: units - pixels (default 1.0)")
parser.add_option(
"--maxit",
type="int",
default=30,
help="number of iterations for reference-free alignment: (default 30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option(
"--center_method",
type="int",
default=-1,
help=
"method for centering: of global 2D average during initial prealignment of data (0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7) (default -1)"
)
parser.add_option(
"--dst",
type="float",
default=90.0,
help="discrete angle used in within group alignment: (default 90.0)")
parser.add_option(
"--FL",
type="float",
default=0.2,
help=
"lowest stopband: frequency used in the tangent filter (default 0.2)")
parser.add_option(
"--FH",
type="float",
default=0.3,
help=
"highest stopband: frequency used in the tangent filter (default 0.3)")
parser.add_option("--FF",
type="float",
default=0.2,
help="fall-off of the tangent filter: (default 0.2)")
parser.add_option(
"--init_iter",
type="int",
default=3,
help=
"SAC initialization iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC initialization (default 3)"
)
parser.add_option(
"--main_iter",
type="int",
default=3,
help=
"SAC main iterations: number of runs of ab-initio within-cluster alignment for stability evaluation in SAC (default 3)"
)
parser.add_option(
"--iter_reali",
type="int",
default=1,
help=
"SAC stability check interval: every iter_reali iterations of SAC stability checking is performed (default 1)"
)
parser.add_option(
"--match_first",
type="int",
default=1,
help=
"number of iterations to run 2-way matching in the first phase: (default 1)"
)
parser.add_option(
"--max_round",
type="int",
default=20,
help=
"maximum rounds: of generating candidate class averages in the first phase (default 20)"
)
parser.add_option(
"--match_second",
type="int",
default=5,
help=
"number of iterations to run 2-way (or 3-way) matching in the second phase: (default 5)"
)
parser.add_option(
"--stab_ali",
type="int",
default=5,
help="number of alignments when checking stability: (default 5)")
parser.add_option(
"--thld_err",
type="float",
default=0.7,
help=
"threshold of pixel error when checking stability: equals root mean square of distances between corresponding pixels from set of found transformations and theirs average transformation, depends linearly on square of radius (parameter ou). units - pixels. (default 0.7)"
)
parser.add_option(
"--indep_run",
type="int",
default=4,
help=
"level of m-way matching for reproducibility tests: By default, perform full ISAC to 4-way matching. Value indep_run=2 will restrict ISAC to 2-way matching and 3 to 3-way matching. Note the number of used MPI processes requested in mpirun must be a multiplicity of indep_run. (default 4)"
)
parser.add_option("--thld_grp",
type="int",
default=10,
help="minimum size of reproducible class (default 10)")
parser.add_option(
"--n_generations",
type="int",
default=10,
help=
"maximum number of generations: program stops when reaching this total number of generations: (default 10)"
)
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option(
"--rand_seed",
type="int",
help=
"random seed set before calculations: useful for testing purposes. By default, total randomness (type int)"
)
parser.add_option("--new",
action="store_true",
default=False,
help="use new code: (default False)")
parser.add_option("--debug",
action="store_true",
default=False,
help="debug info printout: (default False)")
# must be switched off in production
parser.add_option(
"--use_latest_master_directory",
action="store_true",
default=False,
help=
"use latest master directory: when active, the program looks for the latest directory that starts with the word 'master', so the user does not need to provide a directory name. (default False)"
)
parser.add_option(
"--restart_section",
type="string",
default=' ',
help=
"restart section: each generation (iteration) contains three sections: 'restart', 'candidate_class_averages', and 'reproducible_class_averages'. To restart from a particular step, for example, generation 4 and section 'candidate_class_averages' the following option is needed: '--restart_section=candidate_class_averages,4'. The option requires no white space before or after the comma. The default behavior is to restart execution from where it stopped intentionally or unintentionally. For default restart, it is assumed that the name of the directory is provided as argument. Alternatively, the '--use_latest_master_directory' option can be used. (default ' ')"
)
parser.add_option(
"--stop_after_candidates",
action="store_true",
default=False,
help=
"stop after candidates: stops after the 'candidate_class_averages' section. (default False)"
)
##### XXXXXXXXXXXXXXXXXXXXXX option does not exist in docs XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
parser.add_option("--return_options",
action="store_true",
dest="return_options",
default=False,
help=SUPPRESS_HELP)
parser.add_option(
"--skip_prealignment",
action="store_true",
default=False,
help=
"skip pre-alignment step: to be used if images are already centered. 2dalignment directory will still be generated but the parameters will be zero. (default False)"
)
required_option_list = ['radius']
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
from isac import iter_isac
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
command_line_provided_stack_filename = args[0]
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if (myid == main_node):
print "****************************************************************"
Util.version()
print "****************************************************************"
sys.stdout.flush()
mpi_barrier(MPI_COMM_WORLD)
# Making sure all required options appeared.
for required_option in required_option_list:
if not options.__dict__[required_option]:
print "\n ==%s== mandatory option is missing.\n" % required_option
print "Please run '" + progname + " -h' for detailed options"
return 1
radi = options.radius
target_radius = options.target_radius
target_nx = options.target_nx
center_method = options.center_method
if (radi < 1):
ERROR("Particle radius has to be provided!", "sxisac", 1, myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
# program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(
qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re
r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs) > 0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if (myid == main_node):
if (masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM,
localtime())
masterdir = "master" + timestring
cmd = "{} {}".format("mkdir", masterdir)
junk = cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
junk = cmdexecute(cmd)
if (args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(
masterdir, filename)
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_then_all_processes_exit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != " ":
if os.path.exists(os.path.join(masterdir,
NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(
",")
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % parsed_restart_section_option[0],
stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(
generation_str_format)
stored_state[-1][
"isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % options.restart_section,
stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE),
stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()),
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(
stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# previous code 2016-05-05--20-14-12-153
# # PARAMETERS OF THE PROCEDURE
# if( options.xr == -1 ):
# # Default values
# # target_nx = 76
# # target_radius = 29
# target_xr = 1
# else: # nx//2
# # Check below!
# target_xr = options.xr
# # target_nx = 76 + target_xr - 1 # subtract one, which is default
# target_nx += target_xr - 1 # subtract one, which is default
# # target_radius = 29
target_xr = options.xr
target_nx += target_xr - 1 # subtract one, which is default
if (options.yr == -1):
yr = options.xr
else:
yr = options.yr
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if (myid == main_node):
print "command_line_provided_stack_filename", command_line_provided_stack_filename
number_of_images_in_stack = EMUtil.get_image_count(
command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
source_node=main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir, "2dalignment")
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
if not os.path.exists(
os.path.join(init2dir, "Finished_initial_2d_alignment.txt")):
if (myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p " + log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if (myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node=main_node)
image_start, image_end = MPI_start_end(number_of_images_in_stack,
nproc, myid)
if options.skip_prealignment:
params2d = [[0.0, 0.0, 0.0, 0]
for i in xrange(image_start, image_end)]
else:
original_images = EMData.read_images(
command_line_provided_stack_filename,
range(image_start, image_end))
# We assume the target radius will be 29, and xr = 1.
shrink_ratio = float(target_radius) / float(radi)
for im in xrange(len(original_images)):
if (shrink_ratio != 1.0):
original_images[im] = resample(original_images[im],
shrink_ratio)
nx = original_images[0].get_xsize()
# nx = int(nx*shrink_ratio + 0.5)
txrm = (nx - 2 * (target_radius + 1)) // 2
if (txrm < 0):
ERROR(
"ERROR!! Radius of the structure larger than the window data size permits %d"
% (radi), "sxisac", 1, myid)
if (txrm / nxrsteps > 0):
tss = ""
txr = ""
while (txrm / nxrsteps > 0):
tts = txrm / nxrsteps
tss += " %d" % tts
txr += " %d" % (tts * nxrsteps)
txrm = txrm // 2
else:
tss = "1"
txr = "%d" % txrm
# print "nx, txr, txrm, tss", nx, txr, txrm, tss
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
# section ali2d_base
params2d = ali2d_base(original_images, init2dir, None, 1, target_radius, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
del original_images
for i in xrange(len(params2d)):
alpha, sx, sy, mirror = combine_params2(
0, params2d[i][1], params2d[i][2], 0, -params2d[i][0], 0,
0, 0)
sx /= shrink_ratio
sy /= shrink_ratio
params2d[i][0] = 0.0
params2d[i][1] = sx
params2d[i][2] = sy
params2d[i][3] = 0
#set_params2D(aligned_images[i],[0.0, sx,sy,0.,1.0])
mpi_barrier(MPI_COMM_WORLD)
tmp = params2d[:]
tmp = wrap_mpi_gatherv(tmp, main_node, MPI_COMM_WORLD)
if (myid == main_node):
if options.skip_prealignment:
print "========================================="
print "Even though there is no alignment step, '%s' params are set to zero for later use." % os.path.join(
init2dir, "initial2Dparams.txt")
print "========================================="
write_text_row(tmp, os.path.join(init2dir, "initial2Dparams.txt"))
del tmp
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius) / float(radi)
aligned_images = EMData.read_images(
command_line_provided_stack_filename,
range(image_start, image_end))
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx * shrink_ratio + 0.5)
while not os.path.exists(os.path.join(init2dir,
"initial2Dparams.txt")):
import time
time.sleep(1)
mpi_barrier(MPI_COMM_WORLD)
params = read_text_row(os.path.join(init2dir, "initial2Dparams.txt"))
params = params[image_start:image_end]
msk = model_circle(radi, nx, nx)
for im in xrange(nima):
st = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= st[0]
if options.CTF:
aligned_images[im] = filt_ctf(
aligned_images[im],
aligned_images[im].get_attr("ctf"),
binary=True)
if (shrink_ratio < 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
#aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im],
target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(newx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio == 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = Util.window(aligned_images[im],
target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(newx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
#aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio > 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im],
target_nx, target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(newx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
#alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
#alpha, sx, sy, mirror = combine_params2(0, sx,sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0,
params[im][1],
params[im][2], 0)
aligned_images[im] = resample(aligned_images[im],
shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack,
aligned_images, myid)
number_of_images_in_stack = bcast_number_to_all(
number_of_images_in_stack, source_node=main_node)
if (myid == main_node):
for i in range(number_of_images_in_stack):
aligned_images[i].write_image(
stack_processed_by_ali2d_base__filename, i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
fp = open(os.path.join(masterdir, "README_shrink_ratio.txt"), "w")
output_text = """
Since, for processing purposes, isac changes the image dimensions,
adjustment of pixel size needs to be made in subsequent steps, (e.g.
running sxviper.py). The shrink ratio for this particular isac run is
--------
%.5f
%.5f
--------
To get the pixel size for the isac output the user needs to divide
the original pixel size by the above value. This info is saved in
the following file: README_shrink_ratio.txt
""" % (shrink_ratio, radi)
fp.write(output_text)
fp.flush()
fp.close()
print output_text
fp = open(
os.path.join(init2dir, "Finished_initial_2d_alignment.txt"),
"w")
fp.flush()
fp.close()
else:
if (myid == main_node):
print "Skipping 2d alignment since it was already done!"
mpi_barrier(MPI_COMM_WORLD)
# from mpi import mpi_finalize
# mpi_finalize()
# sys.stdout.flush()
# sys.exit()
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
junk = cmdexecute(
"sxheader.py --consecutive --params=originalid %s" %
stack_processed_by_ali2d_base__filename__without_master_dir)
junk = cmdexecute(
"e2bdb.py %s --makevstack=%s_000" %
(stack_processed_by_ali2d_base__filename__without_master_dir,
stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./",
NAME_OF_MAIN_DIR,
myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line + 1,
main_dir_no + 1):
if i == isac_generation_from_command_line + 1:
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
junk = cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
junk = cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i +
" 000_backup" + "%05d" % backup_dir_no)
junk = cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split(
"___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
junk = cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
junk = cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
junk = cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % isac_generation_from_command_line)
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
junk = cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
junk = cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line)
# junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
junk = cmdexecute("rm -rf " + NAME_OF_MAIN_DIR +
"%04d/ali_params_generation_*" %
isac_generation_from_command_line)
junk = cmdexecute("rm -f " + NAME_OF_MAIN_DIR +
"%04d/class_averages_generation*" %
isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(
isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
write_text_file(
[1],
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_accounted.txt" % isac_generation))
write_text_file(
range(number_of_images_in_stack),
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_unaccounted.txt" %
isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d" % isac_generation
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
junk = cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
# reference the original stack
list_file = os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
"generation_%d_unaccounted.txt" % (isac_generation - 1))
junk = cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if (myid == main_node):
if not os.path.exists(
"class_averages_candidate_generation_%d.hdf" %
isac_generation):
print "This generation (%d) no class average candidates were generated! Finishing." % isac_generation
exit_program = 1
exit_program = int(
mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, yr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if (myid == main_node):
accounted_images = read_text_file(
os.path.join(NAME_OF_MAIN_DIR + "%04d" % (isac_generation),
"generation_%d_accounted.txt" %
(isac_generation)))
number_of_accounted_images = len(accounted_images)
un_accounted_images = read_text_file(
os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation),
"generation_%d_unaccounted.txt" % (isac_generation)))
number_of_un_accounted_images = len(un_accounted_images)
else:
number_of_accounted_images = 0
number_of_un_accounted_images = 0
number_of_accounted_images = int(
mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0,
MPI_COMM_WORLD)[0])
number_of_un_accounted_images = int(
mpi_bcast(number_of_un_accounted_images, 1, MPI_INT, 0,
MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
if (myid == main_node):
print "This generation (%d) there are no accounted images! Finishing." % isac_generation
break
while (myid == main_node):
def files_are_missing(isac_generation):
for i in xrange(1, isac_generation + 1):
if not os.path.exists(
"generation_%04d/class_averages_generation_%d.hdf"
% (i, i)):
print "Error: generation_%04d/class_averages_generation_%d.hdf is missing! Exiting." % (
i, i)
return 1
return 0
if files_are_missing(isac_generation):
break
junk = cmdexecute("rm -f class_averages.hdf")
cpy([
"generation_%04d/class_averages_generation_%d.hdf" % (i, i)
for i in xrange(1, isac_generation + 1)
], "class_averages.hdf")
break
if number_of_un_accounted_images == 0:
if (myid == main_node):
print "This generation (%d) there are no un accounted images! Finishing." % isac_generation
break
program_state_stack(locals(),
getframeinfo(currentframe()),
last_call="__LastCall")
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def main(args):
progname = os.path.basename(sys.argv[0])
usage = (
progname +
" stack_file output_directory --radius=particle_radius --img_per_grp=img_per_grp --CTF --restart_section<The remaining parameters are optional --ir=ir --rs=rs --xr=xr --yr=yr --ts=ts --maxit=maxit --dst=dst --FL=FL --FH=FH --FF=FF --init_iter=init_iter --main_maxit=main_iter"
+
" --iter_reali=iter_reali --match_first=match_first --max_round=max_round --match_second=match_second --stab_ali=stab_ali --thld_err=thld_err --indep_run=indep_run --thld_grp=thld_grp"
+ " --generation=generation --rand_seed=rand_seed>")
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--radius",
type="int",
default=-1,
help="<Particle radius>, it has to be provided.")
parser.add_option(
"--img_per_grp",
type="int",
default=100,
help=
"<number of images per group> in the ideal case (essentially maximum size of class) (100)"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="<CTF flag>, if set the data will be phase-flipped")
parser.add_option(
"--ir",
type="int",
default=1,
help="<inner ring> of the resampling to polar coordinates (1)")
parser.add_option(
"--rs",
type="int",
default=1,
help="<ring step> of the resampling to polar coordinates (1)")
parser.add_option(
"--xr",
type="int",
default=-1,
help=
"<x range> of translational search (By default set by the program) (advanced)"
)
parser.add_option(
"--yr",
type="int",
default=-1,
help="<y range> of translational search (same as xr) (advanced)")
parser.add_option("--ts",
type="float",
default=1.0,
help="<search step> of translational search (1.0)")
parser.add_option(
"--maxit",
type="int",
default=30,
help="number of iterations for reference-free alignment (30)")
#parser.add_option("--snr", type="float", default=1.0, help="signal-to-noise ratio (only meaningful when CTF is enabled, currently not supported)")
parser.add_option(
"--center_method",
type="int",
default=7,
help=
"<Method for centering> of global 2D average during initial prealignment of data (default : 7; 0 : no centering; -1 : average shift method; please see center_2D in utilities.py for methods 1-7)"
)
parser.add_option("--dst",
type="float",
default=90.0,
help="discrete angle used in within group alignment ")
parser.add_option(
"--FL",
type="float",
default=0.2,
help="<lowest stopband> frequency used in the tangent filter (0.2)")
parser.add_option(
"--FH",
type="float",
default=0.3,
help="<highest stopband> frequency used in the tangent filter (0.3)")
parser.add_option("--FF",
type="float",
default=0.2,
help="<fall-off of the tangent> filter (0.2)")
parser.add_option(
"--init_iter",
type="int",
default=3,
help=
"<init_iter> number of iterations of ISAC program in initialization (3)"
)
parser.add_option(
"--main_iter",
type="int",
default=3,
help="<main_iter> number of iterations of ISAC program in main part (3)"
)
parser.add_option(
"--iter_reali",
type="int",
default=1,
help=
"<iter_reali> number of iterations in ISAC before checking stability (1)"
)
parser.add_option(
"--match_first",
type="int",
default=1,
help="number of iterations to run 2-way matching in the first phase (1)"
)
parser.add_option(
"--max_round",
type="int",
default=20,
help=
"maximum rounds of generating candidate averages in the first phase (20)"
)
parser.add_option(
"--match_second",
type="int",
default=5,
help=
"number of iterations to run 2-way (or 3-way) matching in the second phase (5)"
)
parser.add_option("--stab_ali",
type="int",
default=5,
help="number of alignments when checking stability (5)")
parser.add_option(
"--thld_err",
type="float",
default=0.7,
help="the threshold of pixel error when checking stability (0.7)")
parser.add_option(
"--indep_run",
type="int",
default=4,
help=
"number of independent runs for reproducibility (default=4, only values 2, 3 and 4 are supported (4)"
)
parser.add_option("--thld_grp",
type="int",
default=10,
help="minimum size of class (10)")
parser.add_option(
"--n_generations",
type="int",
default=100,
help=
"<n_generations> program stops when reaching this total number of generations (advanced)"
)
#parser.add_option("--candidatesexist",action="store_true", default=False, help="Candidate class averages exist use them (default False)")
parser.add_option(
"--rand_seed",
type="int",
default=None,
help=
"random seed set before calculations, useful for testing purposes (default None - total randomness)"
)
parser.add_option("--new",
action="store_true",
default=False,
help="use new code (default = False)")
parser.add_option("--debug",
action="store_true",
default=False,
help="debug info printout (default = False)")
# must be switched off in production
parser.add_option("--use_latest_master_directory",
action="store_true",
dest="use_latest_master_directory",
default=False)
parser.add_option(
"--restart_section",
type="string",
default="",
help=
"<restart section name> (no spaces) followed immediately by comma, followed immediately by generation to restart, example: \n--restart_section=candidate_class_averages,1 (Sections: restart, candidate_class_averages, reproducible_class_averages)"
)
parser.add_option(
"--stop_after_candidates",
action="store_true",
default=False,
help=
"<stop_after_candidates> stops after the 'candidate_class_averages' section"
)
parser.add_option("--return_options",
action="store_true",
dest="return_options",
default=False,
help=SUPPRESS_HELP)
(options, args) = parser.parse_args(args)
if options.return_options:
return parser
if len(args) > 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
sys.exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from isac import iter_isac
global_def.BATCH = True
global_def.BATCH = True
command_line_provided_stack_filename = args[0]
global_def.BATCH = True
main_node = 0
mpi_init(0, [])
myid = mpi_comm_rank(MPI_COMM_WORLD)
nproc = mpi_comm_size(MPI_COMM_WORLD)
radi = options.radius
center_method = options.center_method
if (radi < 1):
ERROR("Particle radius has to be provided!", "sxisac", 1, myid)
use_latest_master_directory = options.use_latest_master_directory
stop_after_candidates = options.stop_after_candidates
program_state_stack.restart_location_title_from_command_line = options.restart_section
from utilities import qw
program_state_stack.PROGRAM_STATE_VARIABLES = set(
qw("""
isac_generation
"""))
# create or reuse master directory
masterdir = ""
stack_processed_by_ali2d_base__filename = ""
stack_processed_by_ali2d_base__filename__without_master_dir = ""
error_status = 0
if len(args) == 2:
masterdir = args[1]
elif len(args) == 1:
if use_latest_master_directory:
all_dirs = [d for d in os.listdir(".") if os.path.isdir(d)]
import re
r = re.compile("^master.*$")
all_dirs = filter(r.match, all_dirs)
if len(all_dirs) > 0:
# all_dirs = max(all_dirs, key=os.path.getctime)
masterdir = max(all_dirs, key=os.path.getmtime)
#Create folder for all results or check if there is one created already
if (myid == main_node):
if (masterdir == ""):
timestring = strftime("%Y_%m_%d__%H_%M_%S" + DIR_DELIM,
localtime())
masterdir = "master" + timestring
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
elif not os.path.exists(masterdir):
# os.path.exists(masterdir) does not exist
masterdir = args[1]
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
if (args[0][:4] == "bdb:"): filename = args[0][4:]
else: filename = args[0][:-4]
filename = os.path.basename(filename)
stack_processed_by_ali2d_base__filename = "bdb:" + os.path.join(
masterdir, filename)
stack_processed_by_ali2d_base__filename__without_master_dir = "bdb:" + filename
if_error_all_processes_quit_program(error_status)
# send masterdir to all processes
masterdir = send_string_to_all(masterdir)
if myid == 0:
if options.restart_section != "":
if os.path.exists(os.path.join(masterdir,
NAME_OF_JSON_STATE_FILE)):
stored_stack, stored_state = restore_program_stack_and_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
import re
if "," in options.restart_section:
parsed_restart_section_option = options.restart_section.split(
",")
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % parsed_restart_section_option[0],
stored_state[-1]["location_in_program"])
generation_str_format = parsed_restart_section_option[1]
if generation_str_format != "":
isac_generation_from_command_line = int(
generation_str_format)
stored_state[-1][
"isac_generation"] = isac_generation_from_command_line
else:
isac_generation_from_command_line = 1
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
else:
isac_generation_from_command_line = -1
stored_state[-1]["location_in_program"] = re.sub(
r"___.*$", "___%s" % options.restart_section,
stored_state[-1]["location_in_program"])
if "isac_generation" in stored_state[-1]:
del stored_state[-1]["isac_generation"]
store_program_state(
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE),
stored_state, stored_stack)
else:
print "Please remove the restart_section option from the command line. The program must be started from the beginning."
mpi_finalize()
sys.exit()
else:
isac_generation_from_command_line = -1
program_state_stack(locals(), getframeinfo(currentframe()),
os.path.join(masterdir, NAME_OF_JSON_STATE_FILE))
stack_processed_by_ali2d_base__filename = send_string_to_all(
stack_processed_by_ali2d_base__filename)
stack_processed_by_ali2d_base__filename__without_master_dir = \
send_string_to_all(stack_processed_by_ali2d_base__filename__without_master_dir)
# PARAMETERS OF THE PROCEDURE
if (options.xr == -1):
# Default values
target_nx = 76
target_radius = 29
target_xr = 1
else: # nx//2
# Check below!
target_xr = options.xr
target_nx = 76 + target_xr - 1 # subtract one, which is default
target_radius = 29
mpi_barrier(MPI_COMM_WORLD)
# Initialization of stacks
if (myid == main_node):
number_of_images_in_stack = EMUtil.get_image_count(
command_line_provided_stack_filename)
else:
number_of_images_in_stack = 0
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
source_node=main_node)
nxrsteps = 4
init2dir = os.path.join(masterdir, "2dalignment")
if (myid == 0):
import subprocess
from logger import Logger, BaseLogger_Files
# Create output directory
log2d = Logger(BaseLogger_Files())
log2d.prefix = os.path.join(init2dir)
cmd = "mkdir -p " + log2d.prefix
outcome = subprocess.call(cmd, shell=True)
log2d.prefix += "/"
# outcome = subprocess.call("sxheader.py "+command_line_provided_stack_filename+" --params=xform.align2d --zero", shell=True)
else:
outcome = 0
log2d = None
if (myid == main_node):
a = get_im(command_line_provided_stack_filename)
nnxo = a.get_xsize()
else:
nnxo = 0
nnxo = bcast_number_to_all(nnxo, source_node=main_node)
txrm = (nnxo - 2 * (radi + 1)) // 2
if (txrm < 0):
ERROR(
"ERROR!! Radius of the structure larger than the window data size permits %d"
% (radi), "sxisac", 1, myid)
if (txrm / nxrsteps > 0):
tss = ""
txr = ""
while (txrm / nxrsteps > 0):
tts = txrm / nxrsteps
tss += " %d" % tts
txr += " %d" % (tts * nxrsteps)
txrm = txrm // 2
else:
tss = "1"
txr = "%d" % txrm
# section ali2d_base
# centering method is set to #7
params2d, aligned_images = ali2d_base(command_line_provided_stack_filename, init2dir, None, 1, radi, 1, txr, txr, tss, \
False, 90.0, center_method, 14, options.CTF, 1.0, False, \
"ref_ali2d", "", log2d, nproc, myid, main_node, MPI_COMM_WORLD, write_headers = False)
if (myid == main_node):
write_text_row(params2d, os.path.join(init2dir, "initial2Dparams.txt"))
del params2d
mpi_barrier(MPI_COMM_WORLD)
# We assume the target image size will be target_nx, radius will be 29, and xr = 1.
# Note images can be also padded, in which case shrink_ratio > 1.
shrink_ratio = float(target_radius) / float(radi)
nx = aligned_images[0].get_xsize()
nima = len(aligned_images)
newx = int(nx * shrink_ratio + 0.5)
from fundamentals import rot_shift2D, resample
from utilities import pad, combine_params2
if (shrink_ratio < 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
aligned_images[im] = Util.window(aligned_images[im], target_nx,
target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(nx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio == 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = Util.window(aligned_images[im], target_nx,
target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(nx // 2 - 2, newx, newx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = resample(aligned_images[im], shrink_ratio)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
elif (shrink_ratio > 1.0):
if newx > target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
aligned_images[im] = Util.window(aligned_images[im], target_nx,
target_nx, 1)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx == target_nx:
msk = model_circle(target_radius, target_nx, target_nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
elif newx < target_nx:
msk = model_circle(target_radius, nx, nx)
for im in xrange(nima):
# Here we should use only shifts
alpha, sx, sy, mirror, scale = get_params2D(aligned_images[im])
alpha, sx, sy, mirror = combine_params2(
0, sx, sy, 0, -alpha, 0, 0, 0)
aligned_images[im] = rot_shift2D(aligned_images[im], 0, sx, sy,
0)
p = Util.infomask(aligned_images[im], msk, False)
aligned_images[im] -= p[0]
p = Util.infomask(aligned_images[im], msk, True)
aligned_images[im] /= p[1]
aligned_images[im] = pad(aligned_images[im], target_nx,
target_nx, 1, 0.0)
del msk
gather_compacted_EMData_to_root(number_of_images_in_stack, aligned_images,
myid)
number_of_images_in_stack = bcast_number_to_all(number_of_images_in_stack,
source_node=main_node)
if (myid == main_node):
for i in range(number_of_images_in_stack):
aligned_images[i].write_image(
stack_processed_by_ali2d_base__filename, i)
# It has to be explicitly closed
from EMAN2db import db_open_dict
DB = db_open_dict(stack_processed_by_ali2d_base__filename)
DB.close()
mpi_barrier(MPI_COMM_WORLD)
global_def.BATCH = True
os.chdir(masterdir)
if program_state_stack(locals(), getframeinfo(currentframe())):
# if 1:
pass
if (myid == main_node):
cmdexecute(
"sxheader.py --consecutive --params=originalid %s" %
stack_processed_by_ali2d_base__filename__without_master_dir)
cmdexecute(
"e2bdb.py %s --makevstack=%s_000" %
(stack_processed_by_ali2d_base__filename__without_master_dir,
stack_processed_by_ali2d_base__filename__without_master_dir))
if (myid == main_node):
main_dir_no = get_latest_directory_increment_value("./",
NAME_OF_MAIN_DIR,
myformat="%04d")
print "isac_generation_from_command_line", isac_generation_from_command_line, main_dir_no
if isac_generation_from_command_line < 0:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = -1
if isac_generation_from_command_line >= 0 and isac_generation_from_command_line <= main_dir_no:
for i in xrange(isac_generation_from_command_line + 1,
main_dir_no + 1):
if i == isac_generation_from_command_line + 1:
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR + "%04d" % i +
" 000_backup" + "%05d" % backup_dir_no)
cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % i)
# it includes both command line and json file
my_restart_section = stored_state[-1]["location_in_program"].split(
"___")[-1]
if "restart" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
cmdexecute(
"rm " + "EMAN2DB/" +
stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d.bdb" % isac_generation_from_command_line)
elif "candidate_class_averages" in my_restart_section:
if "backup_dir_no" not in locals():
backup_dir_no = get_nonexistent_directory_increment_value(
"./", "000_backup", myformat="%05d", start_value=1)
cmdexecute("mkdir -p " + "000_backup" +
"%05d" % backup_dir_no)
cmdexecute("mv " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line +
" 000_backup" + "%05d" % backup_dir_no)
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation_from_command_line)
# cmdexecute("rm -f " + NAME_OF_MAIN_DIR + "%04d/class_averages_candidate*"%isac_generation_from_command_line)
elif "reproducible_class_averages" in my_restart_section:
cmdexecute("rm -rf " + NAME_OF_MAIN_DIR +
"%04d/ali_params_generation_*" %
isac_generation_from_command_line)
cmdexecute("rm -f " + NAME_OF_MAIN_DIR +
"%04d/class_averages_generation*" %
isac_generation_from_command_line)
else:
if os.path.exists(NAME_OF_JSON_STATE_FILE):
stored_stack, stored_state = restore_program_stack_and_state(
NAME_OF_JSON_STATE_FILE)
if "isac_generation" in stored_state[-1]:
isac_generation_from_command_line = stored_state[-1][
"isac_generation"]
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 1
else:
isac_generation_from_command_line = 0
isac_generation_from_command_line = mpi_bcast(
isac_generation_from_command_line, 1, MPI_INT, 0, MPI_COMM_WORLD)[0]
isac_generation = isac_generation_from_command_line - 1
if (myid == main_node):
if isac_generation == 0:
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
write_text_file(
[1],
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_accounted.txt" % isac_generation))
write_text_file(
range(number_of_images_in_stack),
os.path.join(NAME_OF_MAIN_DIR + "%04d" % isac_generation,
"generation_%d_unaccounted.txt" %
isac_generation))
# Stopping criterion should be inside the program.
while True:
isac_generation += 1
if isac_generation > options.n_generations:
break
data64_stack_current = "bdb:../" + stack_processed_by_ali2d_base__filename__without_master_dir[
4:] + "_%03d" % isac_generation
if (myid == main_node):
accounted_images = read_text_file(
os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
"generation_%d_accounted.txt" % (isac_generation - 1)))
number_of_accounted_images = len(accounted_images)
# unaccounted_images = read_text_file(os.path.join(NAME_OF_MAIN_DIR + "%04d"%(isac_generation - 1),"generation_%d_unaccounted.txt"%(isac_generation - 1)))
# number_of_unaccounted_images = len(unaccounted_images)
else:
number_of_accounted_images = 0
number_of_accounted_images = int(
mpi_bcast(number_of_accounted_images, 1, MPI_INT, 0,
MPI_COMM_WORLD)[0])
if number_of_accounted_images == 0:
os.chdir("..")
break
program_state_stack.restart_location_title = "restart"
if program_state_stack(locals(), getframeinfo(currentframe())):
if (myid == main_node):
cmdexecute("mkdir -p " + NAME_OF_MAIN_DIR +
"%04d" % isac_generation)
# reference the original stack
list_file = os.path.join(
NAME_OF_MAIN_DIR + "%04d" % (isac_generation - 1),
"generation_%d_unaccounted.txt" % (isac_generation - 1))
cmdexecute("e2bdb.py %s --makevstack=%s --list=%s"%(stack_processed_by_ali2d_base__filename__without_master_dir,\
stack_processed_by_ali2d_base__filename__without_master_dir + "_%03d"%isac_generation, list_file))
mpi_barrier(MPI_COMM_WORLD)
os.chdir(NAME_OF_MAIN_DIR + "%04d" % isac_generation)
program_state_stack.restart_location_title = "candidate_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, False, random_seed=options.rand_seed, new=False)#options.new)
# program_state_stack.restart_location_title = "stopped_program1"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack.restart_location_title = "stop_after_candidates"
program_state_stack(locals(), getframeinfo(currentframe()))
if stop_after_candidates:
mpi_finalize()
sys.exit()
exit_program = 0
if (myid == main_node):
if not os.path.exists(
"class_averages_candidate_generation_%d.hdf" %
isac_generation):
print "This generation (%d) no class averages were generated!" % isac_generation
exit_program = 1
exit_program = int(
mpi_bcast(exit_program, 1, MPI_INT, 0, MPI_COMM_WORLD)[0])
if exit_program:
os.chdir("..")
break
program_state_stack.restart_location_title = "reproducible_class_averages"
if program_state_stack(locals(), getframeinfo(currentframe())):
iter_isac(data64_stack_current, options.ir, target_radius, options.rs, target_xr, target_xr, options.ts, options.maxit, False, 1.0,\
options.dst, options.FL, options.FH, options.FF, options.init_iter, options.main_iter, options.iter_reali, options.match_first, \
options.max_round, options.match_second, options.stab_ali, options.thld_err, options.indep_run, options.thld_grp, \
options.img_per_grp, isac_generation, True, random_seed=options.rand_seed, new=False)#options.new)
pass
os.chdir("..")
if (myid == main_node):
cmdexecute("rm -f class_averages.hdf")
cpy([
"generation_%04d/class_averages_generation_%d.hdf" % (i, i)
for i in xrange(1, isac_generation)
], "class_averages.hdf")
# program_state_stack.restart_location_title = "stopped_program2"
# program_state_stack(locals(), getframeinfo(currentframe()))
program_state_stack(locals(),
getframeinfo(currentframe()),
last_call="__LastCall")
mpi_finalize()
def main():
import sys
import os
import math
import random
import pyemtbx.options
import time
from random import random, seed, randint
from optparse import OptionParser
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Forms chains of 2D images based on their similarities.
Functionality:
4. Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent).
Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers.
The ways to use the program:
4.1 Use option initial to specify which image will be used as an initial seed to form the chain.
sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25
4.2 If options initial is omitted, the program will determine which image best serves as initial seed to form the chain
sxprocess.py input_stack.hdf output_stack.hdf --radius=25
4.3 Use option circular to form a circular chain.
sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25
4.4 New circular code based on pairwise alignments
sxprocess.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt
4.5 Circular ordering based on pairwise alignments
sxprocess.py vols.hdf chain.hdf mask.hdf --dd --radius=25
"""
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False)
parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False)
parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False)
parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering")
# import params for 2D alignment
parser.add_option("--ou", type="int", default=-1, help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)")
parser.add_option("--xr", type="int", default=0, help="range for translation search in x direction, search is +/xr (0)")
parser.add_option("--yr", type="int", default=0, help="range for translation search in y direction, search is +/yr (0)")
#parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)")
parser.add_option("--pairwiseccc", type="string", default= None, help="Input/output pairwise ccc file")
(options, args) = parser.parse_args()
global_def.BATCH = True
if options.dd:
nargs = len(args)
if nargs != 3:
print "must provide name of input and two output files!"
return
stack = args[0]
new_stack = args[1]
from utilities import model_circle
from statistics import ccc
from statistics import mono
lend = EMUtil.get_image_count(stack)
lccc = [None]*(lend*(lend-1)/2)
for i in xrange(lend-1):
v1 = get_im( stack, i )
if( i == 0 and nargs == 2):
nx = v1.get_xsize()
ny = v1.get_ysize()
nz = v1.get_ysize()
if options.ou < 1 : radius = nx//2-2
else: radius = options.ou
mask = model_circle(radius, nx, ny, nz)
else:
mask = get_im(args[2])
for j in xrange(i+1, lend):
lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0]
order = tsp(lccc)
if(len(order) != lend):
print " problem with data length"
from sys import exit
exit()
print "Total sum of cccs :",TotalDistance(order, lccc)
print "ordering :",order
for i in xrange(lend): get_im(stack, order[i]).write_image( new_stack, i )
elif options.align:
nargs = len(args)
if nargs != 3:
print "must provide name of input and two output files!"
return
from utilities import get_params2D, model_circle
from fundamentals import rot_shift2D
from statistics import ccc
from time import time
from alignment import align2d, align2d_scf
from multi_shc import mult_transform
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
"""
# will align anyway
try:
ttt = d[0].get_attr('xform.params2d')
for i in xrange(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
"""
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.ou < 1 : radius = nx//2-2
else: radius = options.ou
mask = model_circle(radius, nx, ny)
if(options.xr < 0): xrng = 0
else: xrng = options.xr
if(options.yr < 0): yrng = xrng
else: yrng = options.yr
initial = max(options.initial, 0)
from statistics import mono
lend = len(d)
lccc = [None]*(lend*(lend-1)/2)
from utilities import read_text_row
if options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) :
st = time()
for i in xrange(lend-1):
for j in xrange(i+1, lend):
# j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger)
#alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F")
alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius)
lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir]
#print " %4d %10.1f"%(i,time()-st)
if(not os.path.exists(options.pairwiseccc)):
from utilities import write_text_row
write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc)
elif(os.path.exists(options.pairwiseccc)):
lccc = read_text_row(options.pairwiseccc)
initial = int(lccc[0][0] + 0.1)
del lccc[0]
for i in xrange(len(lccc)):
T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)})
lccc[i] = [lccc[i][0],T]
tdummy = Transform({"type":"2D"})
maxsum = -1.023
for m in xrange(0,lend):#initial, initial+1):
indc = range( lend )
lsnake = [[m, tdummy, 0.0]]
del indc[m]
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in xrange(len(indc)):
cuc = lccc[mono(indc[i], lsnake[-1][0])][0]
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
# Here we need transformation from the current to the previous,
# meaning indc[i] -> lsnake[-1][0]
T = lccc[mono(indc[i], lsnake[-1][0])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if( indc[i] > lsnake[-1][0] ): T = T.inverse()
lsnake.append([qi,T, maxcit])
lsum += maxcit
del indc[indc.index(qi)]
T = lccc[mono(indc[-1], lsnake[-1][0])][1]
if( indc[-1] > lsnake[-1][0]): T = T.inverse()
lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]])
print " initial image and lsum ",m,lsum
#print lsnake
if(lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in xrange(lend)]
print " Initial image selected : ",init,maxsum," ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc)
#for q in snake: print q
from copy import deepcopy
trans=deepcopy([snake[i][1] for i in xrange(len(snake))])
print [snake[i][0] for i in xrange(len(snake))]
"""
for m in xrange(lend):
prms = trans[m].get_params("2D")
print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
"""
for k in xrange(lend-2,0,-1):
T = snake[k][1]
for i in xrange(k+1, lend):
trans[i] = T*trans[i]
# To add - apply all transformations and do the overall centering.
for m in xrange(lend):
prms = trans[m].get_params("2D")
#print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
#rot_shift2D(d[snake[m][0]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image(new_stack, m)
rot_shift2D(d[snake[m][0]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(new_stack, m)
order = tsp(lccc)
if(len(order) != lend):
print " problem with data length"
from sys import exit
exit()
print TotalDistance(order, lccc)
print order
ibeg = order.index(init)
order = [order[(i+ibeg)%lend] for i in xrange(lend)]
print TotalDistance(order, lccc)
print order
snake = [tdummy]
for i in xrange(1,lend):
# Here we need transformation from the current to the previous,
# meaning order[i] -> order[i-1]]
T = lccc[mono(order[i], order[i-1])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if( order[i] > order[i-1] ): T = T.inverse()
snake.append(T)
assert(len(snake) == lend)
from copy import deepcopy
trans = deepcopy(snake)
for k in xrange(lend-2,0,-1):
T = snake[k]
for i in xrange(k+1, lend):
trans[i] = T*trans[i]
# Try to smooth the angles - complicated, I am afraid one would have to use angles forward and backwards
# and find their average??
# In addition, one would have to recenter them
"""
trms = []
for m in xrange(lend):
prms = trans[m].get_params("2D")
trms.append([prms["alpha"], prms["mirror"]])
for i in xrange(3):
for m in xrange(lend):
mb = (m-1)%lend
me = (m+1)%lend
# angles order mb,m,me
# calculate predicted angles mb->m
"""
for m in xrange(lend):
prms = trans[m].get_params("2D")
#rot_shift2D(d[order[m]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image("metro.hdf", m)
rot_shift2D(d[order[m]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(args[2], m)
"""
# This was an effort to get number of loops, inconclusive, to say the least
from numpy import outer, zeros, float32, sqrt
lend = len(d)
cor = zeros(lend,float32)
cor = outer(cor, cor)
for i in xrange(lend): cor[i][i] = 1.0
for i in xrange(lend-1):
for j in xrange(i+1, lend):
cor[i,j] = lccc[mono(i,j)][0]
cor[j,i] = cor[i,j]
lmbd, eigvec = pca(cor)
from utilities import write_text_file
nvec=20
print [lmbd[j] for j in xrange(nvec)]
print " G"
mm = [-1]*lend
for i in xrange(lend): # row
mi = -1.0e23
for j in xrange(nvec):
qt = eigvec[j][i]
if(abs(qt)>mi):
mi = abs(qt)
mm[i] = j
for j in xrange(nvec):
qt = eigvec[j][i]
print round(qt,3), # eigenvector
print mm[i]
print
for j in xrange(nvec):
qt = []
for i in xrange(lend):
if(mm[i] == j): qt.append(i)
if(len(qt)>0): write_text_file(qt,"loop%02d.txt"%j)
"""
"""
print [lmbd[j] for j in xrange(nvec)]
print " B"
mm = [-1]*lend
for i in xrange(lend): # row
mi = -1.0e23
for j in xrange(nvec):
qt = eigvec[j][i]/sqrt(lmbd[j])
if(abs(qt)>mi):
mi = abs(qt)
mm[i] = j
for j in xrange(nvec):
qt = eigvec[j][i]/sqrt(lmbd[j])
print round(qt,3), # eigenvector
print mm[i]
print
"""
"""
lend=3
cor = zeros(lend,float32)
cor = outer(cor, cor)
cor[0][0] =136.77
cor[0][1] = 79.15
cor[0][2] = 37.13
cor[1][0] = 79.15
cor[2][0] = 37.13
cor[1][1] = 50.04
cor[1][2] = 21.65
cor[2][1] = 21.65
cor[2][2] = 13.26
lmbd, eigvec = pca(cor)
print lmbd
print eigvec
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i], # eigenvector
print
print " B"
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i]/sqrt(lmbd[j]), # eigenvector
print
print " G"
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i]*sqrt(lmbd[j]), # eigenvector
print
"""
else:
nargs = len(args)
if nargs != 2:
print "must provide name of input and output file!"
return
from utilities import get_params2D, model_circle
from fundamentals import rot_shift2D
from statistics import ccc
from time import time
from alignment import align2d
from multi_shc import mult_transform
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
try:
ttt = d[0].get_attr('xform.params2d')
for i in xrange(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.radius < 1 : radius = nx//2-2
else: radius = options.radius
mask = model_circle(radius, nx, ny)
init = options.initial
if init > -1 :
print " initial image: %d" % init
temp = d[init].copy()
temp.write_image(new_stack, 0)
del d[init]
k = 1
lsum = 0.0
while len(d) > 1:
maxcit = -111.
for i in xrange(len(d)):
cuc = ccc(d[i], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = i
# print k, maxcit
lsum += maxcit
temp = d[qi].copy()
del d[qi]
temp.write_image(new_stack, k)
k += 1
print lsum
d[0].write_image(new_stack, k)
else:
if options.circular :
print "Using options.circular"
# figure the "best circular" starting image
maxsum = -1.023
for m in xrange(len(d)):
indc = range(len(d) )
lsnake = [-1]*(len(d)+1)
lsnake[0] = m
lsnake[-1] = m
del indc[m]
temp = d[m].copy()
lsum = 0.0
direction = +1
k = 1
while len(indc) > 1:
maxcit = -111.
for i in xrange(len(indc)):
cuc = ccc(d[indc[i]], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
lsnake[k] = qi
lsum += maxcit
del indc[indc.index(qi)]
direction = -direction
for i in xrange( 1,len(d) ):
if( direction > 0 ):
if(lsnake[i] == -1):
temp = d[lsnake[i-1]].copy()
#print " forw ",lsnake[i-1]
k = i
break
else:
if(lsnake[len(d) - i] == -1):
temp = d[lsnake[len(d) - i +1]].copy()
#print " back ",lsnake[len(d) - i +1]
k = len(d) - i
break
lsnake[lsnake.index(-1)] = indc[-1]
#print " initial image and lsum ",m,lsum
#print lsnake
if(lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in xrange(len(d))]
print " Initial image selected : ",init,maxsum
print lsnake
for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m)
else:
# figure the "best" starting image
maxsum = -1.023
for m in xrange(len(d)):
indc = range(len(d) )
lsnake = [m]
del indc[m]
temp = d[m].copy()
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in xrange(len(indc)):
cuc = ccc(d[indc[i]], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
lsnake.append(qi)
lsum += maxcit
temp = d[qi].copy()
del indc[indc.index(qi)]
lsnake.append(indc[-1])
#print " initial image and lsum ",m,lsum
#print lsnake
if(lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in xrange(len(d))]
print " Initial image selected : ",init,maxsum
print lsnake
for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m)
def main():
from utilities import get_input_from_string
progname = os.path.basename(sys.argv[0])
usage = progname + " stack output_average --radius=particle_radius --xr=xr --yr=yr --ts=ts --thld_err=thld_err --num_ali=num_ali --fl=fl --aa=aa --CTF --verbose --stables"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--radius", type="int", default=-1, help=" particle radius for alignment")
parser.add_option("--xr", type="string" , default="2 1", help="range for translation search in x direction, search is +/xr (default 2,1)")
parser.add_option("--yr", type="string" , default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)")
parser.add_option("--ts", type="string" , default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default: 1,0.5)")
parser.add_option("--thld_err", type="float", default=0.75, help="threshld of pixel error (default = 0.75)")
parser.add_option("--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)")
parser.add_option("--maxit", type="int", default=30, help="number of iterations for each xr (default = 30)")
parser.add_option("--fl", type="float" , default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter (default = 0.3)")
parser.add_option("--aa", type="float" , default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter (default = 0.2)")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction during the alignment ")
parser.add_option("--verbose", action="store_true", default=False, help="print individual pixel error (default = False)")
parser.add_option("--stables", action="store_true", default=False, help="output the stable particles number in file (default = False)")
parser.add_option("--method", type="string" , default=" ", help="SHC (standard method is default when flag is ommitted)")
(options, args) = parser.parse_args()
if len(args) != 1 and len(args) != 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
else:
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from applications import within_group_refinement, ali2d_ras
from pixel_error import multi_align_stability
from utilities import write_text_file, write_text_row
global_def.BATCH = True
xrng = get_input_from_string(options.xr)
if options.yr == "-1": yrng = xrng
else : yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
class_data = EMData.read_images(args[0])
nx = class_data[0].get_xsize()
ou = options.radius
num_ali = options.num_ali
if ou == -1: ou = nx/2-2
from utilities import model_circle, get_params2D, set_params2D
mask = model_circle(ou, nx, nx)
if options.CTF :
from filter import filt_ctf
for im in xrange(len(class_data)):
# Flip phases
class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
im.set_attr("previousmax", -1.0e10)
try:
t = im.get_attr("xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
all_ali_params = []
for ii in xrange(num_ali):
ali_params = []
if options.verbose:
ALPHA = []
SX = []
SY = []
MIRROR = []
if( xrng[0] == 0.0 and yrng[0] == 0.0 ):
avet = ali2d_ras(class_data, randomize = True, ir = 1, ou = ou, rs = 1, step = 1.0, dst = 90.0, \
maxit = options.maxit, check_mirror = True, FH=options.fl, FF=options.aa)
else:
avet = within_group_refinement(class_data, mask, True, 1, ou, 1, xrng, yrng, step, 90.0, \
maxit = options.maxit, FH=options.fl, FF=options.aa, method = options.method)
from utilities import info
#print " avet ",info(avet)
for im in class_data:
alpha, sx, sy, mirror, scale = get_params2D(im)
ali_params.extend([alpha, sx, sy, mirror])
if options.verbose:
ALPHA.append(alpha)
SX.append(sx)
SY.append(sy)
MIRROR.append(mirror)
all_ali_params.append(ali_params)
if options.verbose:
write_text_file([ALPHA, SX, SY, MIRROR], "ali_params_run_%d"%ii)
"""
avet = class_data[0]
from utilities import read_text_file
all_ali_params = []
for ii in xrange(5):
temp = read_text_file( "ali_params_run_%d"%ii,-1)
uuu = []
for k in xrange(len(temp[0])):
uuu.extend([temp[0][k],temp[1][k],temp[2][k],temp[3][k]])
all_ali_params.append(uuu)
"""
stable_set, mir_stab_rate, pix_err = multi_align_stability(all_ali_params, 0.0, 10000.0, options.thld_err, options.verbose, 2*ou+1)
print "%4s %20s %20s %20s %30s %6.2f"%("", "Size of set", "Size of stable set", "Mirror stab rate", "Pixel error prior to pruning the set above threshold of",options.thld_err)
print "Average stat: %10d %20d %20.2f %15.2f"%( len(class_data), len(stable_set), mir_stab_rate, pix_err)
if( len(stable_set) > 0):
if options.stables:
stab_mem = [[0,0.0,0] for j in xrange(len(stable_set))]
for j in xrange(len(stable_set)): stab_mem[j] = [int(stable_set[j][1]), stable_set[j][0], j]
write_text_row(stab_mem, "stable_particles.txt")
stable_set_id = []
particle_pixerr = []
for s in stable_set:
stable_set_id.append(s[1])
particle_pixerr.append(s[0])
from fundamentals import rot_shift2D
avet.to_zero()
l = -1
print "average parameters: angle, x-shift, y-shift, mirror"
for j in stable_set_id:
l += 1
print " %4d %4d %12.2f %12.2f %12.2f %1d"%(l,j, stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], int(stable_set[l][2][3]))
avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] )
avet /= (l+1)
avet.set_attr('members', stable_set_id)
avet.set_attr('pix_err', pix_err)
avet.set_attr('pixerr', particle_pixerr)
avet.write_image(args[1])
global_def.BATCH = False
def compare(compare_ref_free, outfile_repro, ref_free_output, yrng, xrng,
rstep, nx, apix, ref_free_cutoff, nproc, myid, main_node):
from alignment import Numrinit, ringwe, Applyws
from random import seed, randint
from utilities import get_params2D, set_params2D, model_circle, inverse_transform2, combine_params2
from fundamentals import rot_shift2D
from mpi import MPI_COMM_WORLD, mpi_barrier, mpi_bcast, MPI_INT
from statistics import fsc_mask
from filter import fit_tanh
from numpy import array
fout = "%s.hdf" % ref_free_output
frc_out = "%s_frc" % ref_free_output
res_out = "%s_res" % ref_free_output
nima = EMUtil.get_image_count(compare_ref_free)
image_start, image_end = MPI_start_end(nima, nproc, myid)
ima = EMData()
ima.read_image(compare_ref_free, image_start)
last_ring = nx / 2 - 2
first_ring = 1
mask = model_circle(last_ring, nx, nx)
refi = []
numref = EMUtil.get_image_count(outfile_repro)
cnx = nx / 2 + 1
cny = cnx
mode = "F"
numr = Numrinit(first_ring, last_ring, rstep, mode)
wr = ringwe(numr, mode)
ima.to_zero()
for j in xrange(numref):
temp = EMData()
temp.read_image(outfile_repro, j)
# even, odd, numer of even, number of images. After frc, totav
refi.append(temp)
# for each node read its share of data
data = EMData.read_images(compare_ref_free, range(image_start, image_end))
for im in xrange(image_start, image_end):
data[im - image_start].set_attr('ID', im)
set_params2D(data[im - image_start], [0, 0, 0, 0, 1])
ringref = []
for j in xrange(numref):
refi[j].process_inplace("normalize.mask", {
"mask": mask,
"no_sigma": 1
}) # normalize reference images to N(0,1)
cimage = Util.Polar2Dm(refi[j], cnx, cny, numr, mode)
Util.Frngs(cimage, numr)
Applyws(cimage, numr, wr)
ringref.append(cimage)
if myid == main_node: seed(1000)
data_shift = []
frc = []
res = []
for im in xrange(image_start, image_end):
alpha, sx, sy, mirror, scale = get_params2D(data[im - image_start])
alphai, sxi, syi, scalei = inverse_transform2(alpha, sx, sy, 1.0)
# normalize
data[im - image_start].process_inplace("normalize.mask", {
"mask": mask,
"no_sigma": 1
}) # subtract average under the mask
# align current image to the reference
[angt, sxst, syst, mirrort, xiref,
peakt] = Util.multiref_polar_ali_2d(data[im - image_start], ringref,
xrng, yrng, 1, mode, numr,
cnx + sxi, cny + syi)
iref = int(xiref)
[alphan, sxn, syn, mn] = combine_params2(0.0, -sxi, -syi, 0, angt,
sxst, syst, (int)(mirrort))
set_params2D(data[im - image_start],
[alphan, sxn, syn, int(mn), scale])
temp = rot_shift2D(data[im - image_start], alphan, sxn, syn, mn)
temp.set_attr('assign', iref)
tfrc = fsc_mask(temp, refi[iref], mask=mask)
temp.set_attr('frc', tfrc[1])
res = fit_tanh(tfrc)
temp.set_attr('res', res)
data_shift.append(temp)
for node in xrange(nproc):
if myid == node:
for image in data_shift:
image.write_image(fout, -1)
refindex = image.get_attr('assign')
refi[refindex].write_image(fout, -1)
mpi_barrier(MPI_COMM_WORLD)
rejects = []
if myid == main_node:
a = EMData()
index = 0
frc = []
res = []
temp = []
classes = []
for im in xrange(nima):
a.read_image(fout, index)
frc.append(a.get_attr("frc"))
if ref_free_cutoff != -1:
classes.append(a.get_attr("class_ptcl_idxs"))
tmp = a.get_attr("res")
temp.append(tmp[0])
res.append("%12f" % (apix / tmp[0]))
res.append("\n")
index = index + 2
res_num = array(temp)
mean_score = res_num.mean(axis=0)
std_score = res_num.std(axis=0)
std = std_score / 2
if ref_free_cutoff != -1:
cutoff = mean_score - std * ref_free_cutoff
reject = res_num < cutoff
index = 0
for i in reject:
if i: rejects.extend(classes[index])
index = index + 1
rejects.sort()
length = mpi_bcast(len(rejects), 1, MPI_INT, main_node,
MPI_COMM_WORLD)
rejects = mpi_bcast(rejects, length, MPI_INT, main_node,
MPI_COMM_WORLD)
del a
fout_frc = open(frc_out, 'w')
fout_res = open(res_out, 'w')
fout_res.write("".join(res))
temp = zip(*frc)
datstrings = []
for i in temp:
for j in i:
datstrings.append(" %12f" % (j))
datstrings.append("\n")
fout_frc.write("".join(datstrings))
fout_frc.close()
del refi
del ringref
return rejects
def helicalshiftali_MPI(stack,
maskfile=None,
maxit=100,
CTF=False,
snr=1.0,
Fourvar=False,
search_rng=-1):
from applications import MPI_start_end
from utilities import model_circle, model_blank, get_image, peak_search, get_im, pad
from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
from pap_statistics import varf2d_MPI
from fundamentals import fft, ccf, rot_shift3D, rot_shift2D, fshift
from utilities import get_params2D, set_params2D, chunks_distribution
from utilities import print_msg, print_begin_msg, print_end_msg
import os
import sys
from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
from time import time
from pixel_error import ordersegments
from math import sqrt, atan2, tan, pi
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("helical-shiftali_MPI")
max_iter = int(maxit)
if (myid == main_node):
infils = EMUtil.get_all_attributes(stack, "filament")
ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord')
filaments = ordersegments(infils, ptlcoords)
total_nfils = len(filaments)
inidl = [0] * total_nfils
for i in range(total_nfils):
inidl[i] = len(filaments[i])
linidl = sum(inidl)
nima = linidl
tfilaments = []
for i in range(total_nfils):
tfilaments += filaments[i]
del filaments
else:
total_nfils = 0
linidl = 0
total_nfils = bcast_number_to_all(total_nfils, source_node=main_node)
if myid != main_node:
inidl = [-1] * total_nfils
inidl = bcast_list_to_all(inidl, myid, source_node=main_node)
linidl = bcast_number_to_all(linidl, source_node=main_node)
if myid != main_node:
tfilaments = [-1] * linidl
tfilaments = bcast_list_to_all(tfilaments, myid, source_node=main_node)
filaments = []
iendi = 0
for i in range(total_nfils):
isti = iendi
iendi = isti + inidl[i]
filaments.append(tfilaments[isti:iendi])
del tfilaments, inidl
if myid == main_node:
print_msg("total number of filaments: %d" % total_nfils)
if total_nfils < nproc:
ERROR(
'number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'
% (nproc, total_nfils), "ehelix_MPI", 1, myid)
# balanced load
temp = chunks_distribution([[len(filaments[i]), i]
for i in range(len(filaments))],
nproc)[myid:myid + 1][0]
filaments = [filaments[temp[i][1]] for i in range(len(temp))]
nfils = len(filaments)
#filaments = [[0,1]]
#print "filaments",filaments
list_of_particles = []
indcs = []
k = 0
for i in range(nfils):
list_of_particles += filaments[i]
k1 = k + len(filaments[i])
indcs.append([k, k1])
k = k1
data = EMData.read_images(stack, list_of_particles)
ldata = len(data)
print("ldata=", ldata)
nx = data[0].get_xsize()
ny = data[0].get_ysize()
if maskfile == None:
mrad = min(nx, ny) // 2 - 2
mask = pad(model_blank(2 * mrad + 1, ny, 1, 1.0), nx, ny, 1, 0.0)
else:
mask = get_im(maskfile)
# apply initial xform.align2d parameters stored in header
init_params = []
for im in range(ldata):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'],
p['mirror'], p['scale'])
if CTF:
from filter import filt_ctf
from morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
ctf_abs_sum = None
from utilities import info
for im in range(ldata):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
qctf = data[im].get_attr("ctf_applied")
if qctf == 0:
data[im] = filt_ctf(fft(data[im]), ctf_params)
data[im].set_attr('ctf_applied', 1)
elif qctf != 1:
ERROR('Incorrectly set qctf flag', "helicalshiftali_MPI", 1,
myid)
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
else:
data[im] = fft(data[im])
del list_of_particles
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
tsnr = 1. / snr
for i in range(0, nx + 2, 2):
for j in range(ny):
temp.set_value_at(i, j, tsnr)
temp.set_value_at(i + 1, j, 0.0)
#info(ctf_2_sum)
Util.add_img(ctf_2_sum, temp)
#info(ctf_2_sum)
del temp
total_iter = 0
shift_x = [0.0] * ldata
for Iter in range(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n" % (total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in range(ldata):
Util.add_img(avg, fshift(data[im], shift_x[im]))
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF: tavg = Util.divn_filter(avg, ctf_2_sum)
else: tavg = Util.mult_scalar(avg, 1.0 / float(nima))
else:
tavg = model_blank(nx, ny)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask(tavg, mask, False)
tavg -= stat[0]
Util.mul_img(tavg, mask)
tavg.write_image("tavg.hdf", Iter)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
tavg = fft(tavg)
sx_sum = 0.0
nxc = nx // 2
for ifil in range(nfils):
"""
# Calculate filament average
avg = EMData(nx, ny, 1, False)
filnima = 0
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
Util.add_img(avg, data[im])
filnima += 1
tavg = Util.mult_scalar(avg, 1.0/float(filnima))
"""
# Calculate 1D ccf between each segment and filament average
nsegms = indcs[ifil][1] - indcs[ifil][0]
ctx = [None] * nsegms
pcoords = [None] * nsegms
for im in range(indcs[ifil][0], indcs[ifil][1]):
ctx[im - indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx,
1)
pcoords[im - indcs[ifil][0]] = data[im].get_attr(
'ptcl_source_coord')
#ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0])
#print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True)
# search for best x-shift
cents = nsegms // 2
dst = sqrt(
max((pcoords[cents][0] - pcoords[0][0])**2 +
(pcoords[cents][1] - pcoords[0][1])**2,
(pcoords[cents][0] - pcoords[-1][0])**2 +
(pcoords[cents][1] - pcoords[-1][1])**2))
maxincline = atan2(ny // 2 - 2 - float(search_rng), dst)
kang = int(dst * tan(maxincline) + 0.5)
#print " settings ",nsegms,cents,dst,search_rng,maxincline,kang
# ## C code for alignment. @ming
results = [0.0] * 3
results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline,
kang, search_rng, nxc)
sib = int(results[0])
bang = results[1]
qm = results[2]
#print qm, sib, bang
# qm = -1.e23
#
# for six in xrange(-search_rng, search_rng+1,1):
# q0 = ctx[cents].get_value_at(six+nxc)
# for incline in xrange(kang+1):
# qt = q0
# qu = q0
# if(kang>0): tang = tan(maxincline/kang*incline)
# else: tang = 0.0
# for kim in xrange(cents+1,nsegms):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# #print " A ", ifil,six,incline,kim,xl,ixl,dxl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# for kim in xrange(cents):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# if( qt > qm ):
# qm = qt
# sib = six
# bang = tang
# if( qu > qm ):
# qm = qu
# sib = six
# bang = -tang
#if incline == 0: print "incline = 0 ",six,tang,qt,qu
#print qm,six,sib,bang
#print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True)
for im in range(indcs[ifil][0], indcs[ifil][1]):
kim = im - indcs[ifil][0]
dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 +
(pcoords[cents][1] - pcoords[kim][1])**2)
if (kim < cents): xl = -dst * bang + sib
else: xl = dst * bang + sib
shift_x[im] = xl
# Average shift
sx_sum += shift_x[indcs[ifil][0] + cents]
# #print myid,sx_sum,total_nfils
sx_sum = mpi_reduce(sx_sum, 1, MPI_FLOAT, MPI_SUM, main_node,
MPI_COMM_WORLD)
if myid == main_node:
sx_sum = float(sx_sum[0]) / total_nfils
print_msg("Average shift %6.2f\n" % (sx_sum))
else:
sx_sum = 0.0
sx_sum = 0.0
sx_sum = bcast_number_to_all(sx_sum, source_node=main_node)
for im in range(ldata):
shift_x[im] -= sx_sum
#print " %3d %6.3f"%(im,shift_x[im])
#exit()
# combine shifts found with the original parameters
for im in range(ldata):
t1 = Transform()
##import random
##shix=random.randint(-10, 10)
##t1.set_params({"type":"2D","tx":shix})
t1.set_params({"type": "2D", "tx": shift_x[im]})
# combine t0 and t1
tt = t1 * init_params[im]
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from utilities import file_type
if (file_type(stack) == "bdb"):
from utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata,
nproc)
else:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc)
else:
send_attr_dict(main_node, data, par_str, 0, ldata)
if myid == main_node: print_end_msg("helical-shiftali_MPI")
def main():
import global_def
from optparse import OptionParser
from EMAN2 import EMUtil
import os
import sys
from time import time
progname = os.path.basename(sys.argv[0])
usage = progname + " proj_stack output_averages --MPI"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--img_per_group",type="int" , default=100 , help="number of images per group" )
parser.add_option("--radius", type="int" , default=-1 , help="radius for alignment" )
parser.add_option("--xr", type="string" , default="2 1", help="range for translation search in x direction, search is +/xr")
parser.add_option("--yr", type="string" , default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)")
parser.add_option("--ts", type="string" , default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional")
parser.add_option("--iter", type="int" , default=30, help="number of iterations within alignment (default = 30)" )
parser.add_option("--num_ali", type="int" , default=5, help="number of alignments performed for stability (default = 5)" )
parser.add_option("--thld_err", type="float" , default=1.0, help="threshold of pixel error (default = 1.732)" )
parser.add_option("--grouping" , type="string" , default="GRP", help="do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size")
parser.add_option("--delta", type="float" , default=-1.0, help="angular step for reference projections (required for GEV method)")
parser.add_option("--fl", type="float" , default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--aa", type="float" , default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ")
parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
(options,args) = parser.parse_args()
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD, MPI_TAG_UB
from mpi import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT
from applications import MPI_start_end, within_group_refinement, ali2d_ras
from pixel_error import multi_align_stability
from utilities import send_EMData, recv_EMData
from utilities import get_image, bcast_number_to_all, set_params2D, get_params2D
from utilities import group_proj_by_phitheta, model_circle, get_input_from_string
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 2:
stack = args[0]
outdir = args[1]
else:
ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid)
exit()
if not options.MPI:
ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid)
exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
#if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid)
#mpi_barrier(MPI_COMM_WORLD)
img_per_grp = options.img_per_group
radius = options.radius
ite = options.iter
num_ali = options.num_ali
thld_err = options.thld_err
xrng = get_input_from_string(options.xr)
if options.yr == "-1": yrng = xrng
else : yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
if radius == -1: radius = nx/2-2
mask = model_circle(radius, nx, nx)
st = time()
if options.grouping == "GRP":
if myid == main_node:
print " A ",myid," ",time()-st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
# Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta,
# So I will briefly explain it here
# proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers
# except for the last one. Depending on the number of particles left, they will either form a
# group or append themselves to the last group
# angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi,
# theta) is the projection angle of the center of the group, delta is the range of this group
# mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates
# whether it should take mirror position.
# In this program angle_list and mirror list are not of interest.
proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(proj_params, img_per_grp=img_per_grp)
del proj_params
print " B number of groups ",myid," ",len(proj_list_all),time()-st
mpi_barrier(MPI_COMM_WORLD)
# Number of groups, actually there could be one or two more groups, since the size of the remaining group varies
# we will simply assign them to main node.
n_grp = nima/img_per_grp-1
# Divide proj_list_all equally to all nodes, and becomes proj_list
proj_list = []
for i in xrange(n_grp):
proc_to_stay = i%number_of_proc
if proc_to_stay == main_node:
if myid == main_node: proj_list.append(proj_list_all[i])
elif myid == main_node:
mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD)
elif myid == proc_to_stay:
img_per_grp = mpi_recv(1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
img_per_grp = int(img_per_grp[0])
temp = mpi_recv(img_per_grp, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
proj_list.append(map(int, temp))
del temp
mpi_barrier(MPI_COMM_WORLD)
print " C ",myid," ",time()-st
if myid == main_node:
# Assign the remaining groups to main_node
for i in xrange(n_grp, len(proj_list_all)):
proj_list.append(proj_list_all[i])
del proj_list_all, angle_list, mirror_list
# Compute stability per projection projection direction, equal number assigned, thus overlaps
elif options.grouping == "GEV":
if options.delta == -1.0: ERROR("Angular step for reference projections is required for GEV method","sxproj_stability",1)
from utilities import even_angles, nearestk_to_refdir, getvec
refproj = even_angles(options.delta)
img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid)
# Now each processor keeps its own share of reference projections
refprojdir = refproj[img_begin: img_end]
del refproj
ref_ang = [0.0]*(len(refprojdir)*2)
for i in xrange(len(refprojdir)):
ref_ang[i*2] = refprojdir[0][0]
ref_ang[i*2+1] = refprojdir[0][1]+i*0.1
print " A ",myid," ",time()-st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
# the solution below is very slow, do not use it unless there is a problem with the i/O
"""
for i in xrange(number_of_proc):
if myid == i:
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
mpi_barrier(MPI_COMM_WORLD)
"""
print " B ",myid," ",time()-st
proj_ang = [0.0]*(nima*2)
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
proj_ang[i*2] = dp["phi"]
proj_ang[i*2+1] = dp["theta"]
print " C ",myid," ",time()-st
asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp)
del proj_ang, ref_ang
proj_list = []
for i in xrange(len(refprojdir)):
proj_list.append(asi[i*img_per_grp:(i+1)*img_per_grp])
del asi
print " D ",myid," ",time()-st
#from sys import exit
#exit()
# Compute stability per projection
elif options.grouping == "PPR":
print " A ",myid," ",time()-st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
print " B ",myid," ",time()-st
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
print " C ",myid," ",time()-st
from utilities import nearest_proj
proj_list, mirror_list = nearest_proj(proj_params, img_per_grp, range(img_begin, img_begin+1))#range(img_begin, img_end))
refprojdir = proj_params[img_begin: img_end]
del proj_params, mirror_list
print " D ",myid," ",time()-st
else: ERROR("Incorrect projection grouping option","sxproj_stability",1)
"""
from utilities import write_text_file
for i in xrange(len(proj_list)):
write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid))
"""
###########################################################################################################
# Begin stability test
from utilities import get_params_proj, read_text_file
#if myid == 0:
# from utilities import read_text_file
# proj_list[0] = map(int, read_text_file("lggrpp0.txt"))
from utilities import model_blank
aveList = [model_blank(nx,ny)]*len(proj_list)
if options.grouping == "GRP": refprojdir = [[0.0,0.0,-1.0]]*len(proj_list)
for i in xrange(len(proj_list)):
print " E ",myid," ",time()-st
class_data = EMData.read_images(stack, proj_list[i])
#print " R ",myid," ",time()-st
if options.CTF :
from filter import filt_ctf
for im in xrange(len(class_data)): # MEM LEAK!!
atemp = class_data[im].copy()
btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1)
class_data[im] = btemp
#class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
try:
t = im.get_attr("xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0,-d["tx"],-d["ty"],0,1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
#print " F ",myid," ",time()-st
# Here, we perform realignment num_ali times
all_ali_params = []
for j in xrange(num_ali):
if( xrng[0] == 0.0 and yrng[0] == 0.0 ):
avet = ali2d_ras(class_data, randomize = True, ir = 1, ou = radius, rs = 1, step = 1.0, dst = 90.0, maxit = ite, check_mirror = True, FH=options.fl, FF=options.aa)
else:
avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa)
ali_params = []
for im in xrange(len(class_data)):
alpha, sx, sy, mirror, scale = get_params2D(class_data[im])
ali_params.extend( [alpha, sx, sy, mirror] )
all_ali_params.append(ali_params)
#aveList[i] = avet
#print " G ",myid," ",time()-st
del ali_params
# We determine the stability of this group here.
# stable_set contains all particles deemed stable, it is a list of list
# each list has two elements, the first is the pixel error, the second is the image number
# stable_set is sorted based on pixel error
#from utilities import write_text_file
#write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid)
stable_set, mir_stab_rate, average_pix_err = multi_align_stability(all_ali_params, 0.0, 10000.0, thld_err, False, 2*radius+1)
#print " H ",myid," ",time()-st
if(len(stable_set) > 5):
stable_set_id = []
members = []
pix_err = []
# First put the stable members into attr 'members' and 'pix_err'
for s in stable_set:
# s[1] - number in this subset
stable_set_id.append(s[1])
# the original image number
members.append(proj_list[i][s[1]])
pix_err.append(s[0])
# Then put the unstable members into attr 'members' and 'pix_err'
from fundamentals import rot_shift2D
avet.to_zero()
if options.grouping == "GRP":
aphi = 0.0
atht = 0.0
vphi = 0.0
vtht = 0.0
l = -1
for j in xrange(len(proj_list[i])):
# Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses
if j in stable_set_id:
l += 1
avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] )
if options.grouping == "GRP":
phi, theta, psi, sxs, sys = get_params_proj(class_data[j])
if( theta > 90.0):
phi = (phi+540.0)%360.0
theta = 180.0 - theta
aphi += phi
atht += theta
vphi += phi*phi
vtht += theta*theta
else:
members.append(proj_list[i][j])
pix_err.append(99999.99)
aveList[i] = avet.copy()
if l>1 :
l += 1
aveList[i] /= l
if options.grouping == "GRP":
aphi /= l
atht /= l
vphi = (vphi - l*aphi*aphi)/l
vtht = (vtht - l*atht*atht)/l
from math import sqrt
refprojdir[i] = [aphi, atht, (sqrt(max(vphi,0.0))+sqrt(max(vtht,0.0)))/2.0]
# Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION
aveList[i].set_attr('members', members)
aveList[i].set_attr('refprojdir',refprojdir[i])
aveList[i].set_attr('pixerr', pix_err)
else:
print " empty group ",i, refprojdir[i]
aveList[i].set_attr('members',[-1])
aveList[i].set_attr('refprojdir',refprojdir[i])
aveList[i].set_attr('pixerr', [99999.])
del class_data
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(aveList)):
aveList[im].write_image(args[1], km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('pixerr', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
ave.write_image(args[1], km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node,im+myid+70000)
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
members = aveList[im].get_attr('pixerr')
mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
global_def.BATCH = False
mpi_barrier(MPI_COMM_WORLD)
from mpi import mpi_finalize
mpi_finalize()
def main():
import global_def
from optparse import OptionParser
from EMAN2 import EMUtil
import os
import sys
from time import time
progname = os.path.basename(sys.argv[0])
usage = progname + " proj_stack output_averages --MPI"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--img_per_group",
type="int",
default=100,
help="number of images per group")
parser.add_option("--radius",
type="int",
default=-1,
help="radius for alignment")
parser.add_option(
"--xr",
type="string",
default="2 1",
help="range for translation search in x direction, search is +/xr")
parser.add_option(
"--yr",
type="string",
default="-1",
help=
"range for translation search in y direction, search is +/yr (default = same as xr)"
)
parser.add_option(
"--ts",
type="string",
default="1 0.5",
help=
"step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional"
)
parser.add_option(
"--iter",
type="int",
default=30,
help="number of iterations within alignment (default = 30)")
parser.add_option(
"--num_ali",
type="int",
default=5,
help="number of alignments performed for stability (default = 5)")
parser.add_option("--thld_err",
type="float",
default=1.0,
help="threshold of pixel error (default = 1.732)")
parser.add_option(
"--grouping",
type="string",
default="GRP",
help=
"do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size"
)
parser.add_option(
"--delta",
type="float",
default=-1.0,
help="angular step for reference projections (required for GEV method)"
)
parser.add_option(
"--fl",
type="float",
default=0.3,
help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
parser.add_option(
"--aa",
type="float",
default=0.2,
help="fall-off of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--CTF",
action="store_true",
default=False,
help="Consider CTF correction during the alignment ")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version")
(options, args) = parser.parse_args()
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT
from applications import MPI_start_end, within_group_refinement, ali2d_ras
from pixel_error import multi_align_stability
from utilities import send_EMData, recv_EMData
from utilities import get_image, bcast_number_to_all, set_params2D, get_params2D
from utilities import group_proj_by_phitheta, model_circle, get_input_from_string
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 2:
stack = args[0]
outdir = args[1]
else:
ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid)
exit()
if not options.MPI:
ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid)
exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
#if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid)
#mpi_barrier(MPI_COMM_WORLD)
img_per_grp = options.img_per_group
radius = options.radius
ite = options.iter
num_ali = options.num_ali
thld_err = options.thld_err
xrng = get_input_from_string(options.xr)
if options.yr == "-1": yrng = xrng
else: yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
if radius == -1: radius = nx / 2 - 2
mask = model_circle(radius, nx, nx)
st = time()
if options.grouping == "GRP":
if myid == main_node:
print " A ", myid, " ", time() - st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
"psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
# Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta,
# So I will briefly explain it here
# proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers
# except for the last one. Depending on the number of particles left, they will either form a
# group or append themselves to the last group
# angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi,
# theta) is the projection angle of the center of the group, delta is the range of this group
# mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates
# whether it should take mirror position.
# In this program angle_list and mirror list are not of interest.
proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(
proj_params, img_per_grp=img_per_grp)
del proj_params
print " B number of groups ", myid, " ", len(
proj_list_all), time() - st
mpi_barrier(MPI_COMM_WORLD)
# Number of groups, actually there could be one or two more groups, since the size of the remaining group varies
# we will simply assign them to main node.
n_grp = nima / img_per_grp - 1
# Divide proj_list_all equally to all nodes, and becomes proj_list
proj_list = []
for i in xrange(n_grp):
proc_to_stay = i % number_of_proc
if proc_to_stay == main_node:
if myid == main_node: proj_list.append(proj_list_all[i])
elif myid == main_node:
mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT,
proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
elif myid == proc_to_stay:
img_per_grp = mpi_recv(1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
img_per_grp = int(img_per_grp[0])
temp = mpi_recv(img_per_grp, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
proj_list.append(map(int, temp))
del temp
mpi_barrier(MPI_COMM_WORLD)
print " C ", myid, " ", time() - st
if myid == main_node:
# Assign the remaining groups to main_node
for i in xrange(n_grp, len(proj_list_all)):
proj_list.append(proj_list_all[i])
del proj_list_all, angle_list, mirror_list
# Compute stability per projection projection direction, equal number assigned, thus overlaps
elif options.grouping == "GEV":
if options.delta == -1.0:
ERROR(
"Angular step for reference projections is required for GEV method",
"sxproj_stability", 1)
from utilities import even_angles, nearestk_to_refdir, getvec
refproj = even_angles(options.delta)
img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid)
# Now each processor keeps its own share of reference projections
refprojdir = refproj[img_begin:img_end]
del refproj
ref_ang = [0.0] * (len(refprojdir) * 2)
for i in xrange(len(refprojdir)):
ref_ang[i * 2] = refprojdir[0][0]
ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1
print " A ", myid, " ", time() - st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
# the solution below is very slow, do not use it unless there is a problem with the i/O
"""
for i in xrange(number_of_proc):
if myid == i:
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
mpi_barrier(MPI_COMM_WORLD)
"""
print " B ", myid, " ", time() - st
proj_ang = [0.0] * (nima * 2)
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
proj_ang[i * 2] = dp["phi"]
proj_ang[i * 2 + 1] = dp["theta"]
print " C ", myid, " ", time() - st
asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp)
del proj_ang, ref_ang
proj_list = []
for i in xrange(len(refprojdir)):
proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp])
del asi
print " D ", myid, " ", time() - st
#from sys import exit
#exit()
# Compute stability per projection
elif options.grouping == "PPR":
print " A ", myid, " ", time() - st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
print " B ", myid, " ", time() - st
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
"psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
print " C ", myid, " ", time() - st
from utilities import nearest_proj
proj_list, mirror_list = nearest_proj(
proj_params, img_per_grp,
range(img_begin, img_begin + 1)) #range(img_begin, img_end))
refprojdir = proj_params[img_begin:img_end]
del proj_params, mirror_list
print " D ", myid, " ", time() - st
else:
ERROR("Incorrect projection grouping option", "sxproj_stability", 1)
"""
from utilities import write_text_file
for i in xrange(len(proj_list)):
write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid))
"""
###########################################################################################################
# Begin stability test
from utilities import get_params_proj, read_text_file
#if myid == 0:
# from utilities import read_text_file
# proj_list[0] = map(int, read_text_file("lggrpp0.txt"))
from utilities import model_blank
aveList = [model_blank(nx, ny)] * len(proj_list)
if options.grouping == "GRP":
refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list)
for i in xrange(len(proj_list)):
print " E ", myid, " ", time() - st
class_data = EMData.read_images(stack, proj_list[i])
#print " R ",myid," ",time()-st
if options.CTF:
from filter import filt_ctf
for im in xrange(len(class_data)): # MEM LEAK!!
atemp = class_data[im].copy()
btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1)
class_data[im] = btemp
#class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
try:
t = im.get_attr(
"xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
#print " F ",myid," ",time()-st
# Here, we perform realignment num_ali times
all_ali_params = []
for j in xrange(num_ali):
if (xrng[0] == 0.0 and yrng[0] == 0.0):
avet = ali2d_ras(class_data,
randomize=True,
ir=1,
ou=radius,
rs=1,
step=1.0,
dst=90.0,
maxit=ite,
check_mirror=True,
FH=options.fl,
FF=options.aa)
else:
avet = within_group_refinement(class_data, mask, True, 1,
radius, 1, xrng, yrng, step,
90.0, ite, options.fl,
options.aa)
ali_params = []
for im in xrange(len(class_data)):
alpha, sx, sy, mirror, scale = get_params2D(class_data[im])
ali_params.extend([alpha, sx, sy, mirror])
all_ali_params.append(ali_params)
#aveList[i] = avet
#print " G ",myid," ",time()-st
del ali_params
# We determine the stability of this group here.
# stable_set contains all particles deemed stable, it is a list of list
# each list has two elements, the first is the pixel error, the second is the image number
# stable_set is sorted based on pixel error
#from utilities import write_text_file
#write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid)
stable_set, mir_stab_rate, average_pix_err = multi_align_stability(
all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1)
#print " H ",myid," ",time()-st
if (len(stable_set) > 5):
stable_set_id = []
members = []
pix_err = []
# First put the stable members into attr 'members' and 'pix_err'
for s in stable_set:
# s[1] - number in this subset
stable_set_id.append(s[1])
# the original image number
members.append(proj_list[i][s[1]])
pix_err.append(s[0])
# Then put the unstable members into attr 'members' and 'pix_err'
from fundamentals import rot_shift2D
avet.to_zero()
if options.grouping == "GRP":
aphi = 0.0
atht = 0.0
vphi = 0.0
vtht = 0.0
l = -1
for j in xrange(len(proj_list[i])):
# Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses
if j in stable_set_id:
l += 1
avet += rot_shift2D(class_data[j], stable_set[l][2][0],
stable_set[l][2][1],
stable_set[l][2][2],
stable_set[l][2][3])
if options.grouping == "GRP":
phi, theta, psi, sxs, sys = get_params_proj(
class_data[j])
if (theta > 90.0):
phi = (phi + 540.0) % 360.0
theta = 180.0 - theta
aphi += phi
atht += theta
vphi += phi * phi
vtht += theta * theta
else:
members.append(proj_list[i][j])
pix_err.append(99999.99)
aveList[i] = avet.copy()
if l > 1:
l += 1
aveList[i] /= l
if options.grouping == "GRP":
aphi /= l
atht /= l
vphi = (vphi - l * aphi * aphi) / l
vtht = (vtht - l * atht * atht) / l
from math import sqrt
refprojdir[i] = [
aphi, atht,
(sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0
]
# Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION
aveList[i].set_attr('members', members)
aveList[i].set_attr('refprojdir', refprojdir[i])
aveList[i].set_attr('pixerr', pix_err)
else:
print " empty group ", i, refprojdir[i]
aveList[i].set_attr('members', [-1])
aveList[i].set_attr('refprojdir', refprojdir[i])
aveList[i].set_attr('pixerr', [99999.])
del class_data
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(aveList)):
aveList[im].write_image(args[1], km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pixerr', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
ave.write_image(args[1], km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pixerr')
mpi_send(members, len(members), MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
global_def.BATCH = False
mpi_barrier(MPI_COMM_WORLD)
from mpi import mpi_finalize
mpi_finalize()
def main():
from utilities import get_input_from_string
progname = os.path.basename(sys.argv[0])
usage = (
progname
+ " stack output_average --radius=particle_radius --xr=xr --yr=yr --ts=ts --thld_err=thld_err --num_ali=num_ali --fl=fl --aa=aa --CTF --verbose --stables"
)
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--radius", type="int", default=-1, help=" particle radius for alignment")
parser.add_option(
"--xr",
type="string",
default="2 1",
help="range for translation search in x direction, search is +/xr (default 2,1)",
)
parser.add_option(
"--yr",
type="string",
default="-1",
help="range for translation search in y direction, search is +/yr (default = same as xr)",
)
parser.add_option(
"--ts",
type="string",
default="1 0.5",
help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default: 1,0.5)",
)
parser.add_option("--thld_err", type="float", default=0.75, help="threshld of pixel error (default = 0.75)")
parser.add_option(
"--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)"
)
parser.add_option("--maxit", type="int", default=30, help="number of iterations for each xr (default = 30)")
parser.add_option(
"--fl",
type="float",
default=0.3,
help="cut-off frequency of hyperbolic tangent low-pass Fourier filter (default = 0.3)",
)
parser.add_option(
"--aa", type="float", default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter (default = 0.2)"
)
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction during the alignment ")
parser.add_option(
"--verbose", action="store_true", default=False, help="print individual pixel error (default = False)"
)
parser.add_option(
"--stables",
action="store_true",
default=False,
help="output the stable particles number in file (default = False)",
)
parser.add_option(
"--method", type="string", default=" ", help="SHC (standard method is default when flag is ommitted)"
)
(options, args) = parser.parse_args()
if len(args) != 1 and len(args) != 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
else:
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from applications import within_group_refinement, ali2d_ras
from pixel_error import multi_align_stability
from utilities import write_text_file, write_text_row
global_def.BATCH = True
xrng = get_input_from_string(options.xr)
if options.yr == "-1":
yrng = xrng
else:
yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
class_data = EMData.read_images(args[0])
nx = class_data[0].get_xsize()
ou = options.radius
num_ali = options.num_ali
if ou == -1:
ou = nx / 2 - 2
from utilities import model_circle, get_params2D, set_params2D
mask = model_circle(ou, nx, nx)
if options.CTF:
from filter import filt_ctf
for im in xrange(len(class_data)):
# Flip phases
class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
im.set_attr("previousmax", -1.0e10)
try:
t = im.get_attr("xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
all_ali_params = []
for ii in xrange(num_ali):
ali_params = []
if options.verbose:
ALPHA = []
SX = []
SY = []
MIRROR = []
if xrng[0] == 0.0 and yrng[0] == 0.0:
avet = ali2d_ras(
class_data,
randomize=True,
ir=1,
ou=ou,
rs=1,
step=1.0,
dst=90.0,
maxit=options.maxit,
check_mirror=True,
FH=options.fl,
FF=options.aa,
)
else:
avet = within_group_refinement(
class_data,
mask,
True,
1,
ou,
1,
xrng,
yrng,
step,
90.0,
maxit=options.maxit,
FH=options.fl,
FF=options.aa,
method=options.method,
)
from utilities import info
# print " avet ",info(avet)
for im in class_data:
alpha, sx, sy, mirror, scale = get_params2D(im)
ali_params.extend([alpha, sx, sy, mirror])
if options.verbose:
ALPHA.append(alpha)
SX.append(sx)
SY.append(sy)
MIRROR.append(mirror)
all_ali_params.append(ali_params)
if options.verbose:
write_text_file([ALPHA, SX, SY, MIRROR], "ali_params_run_%d" % ii)
"""
avet = class_data[0]
from utilities import read_text_file
all_ali_params = []
for ii in xrange(5):
temp = read_text_file( "ali_params_run_%d"%ii,-1)
uuu = []
for k in xrange(len(temp[0])):
uuu.extend([temp[0][k],temp[1][k],temp[2][k],temp[3][k]])
all_ali_params.append(uuu)
"""
stable_set, mir_stab_rate, pix_err = multi_align_stability(
all_ali_params, 0.0, 10000.0, options.thld_err, options.verbose, 2 * ou + 1
)
print "%4s %20s %20s %20s %30s %6.2f" % (
"",
"Size of set",
"Size of stable set",
"Mirror stab rate",
"Pixel error prior to pruning the set above threshold of",
options.thld_err,
)
print "Average stat: %10d %20d %20.2f %15.2f" % (len(class_data), len(stable_set), mir_stab_rate, pix_err)
if len(stable_set) > 0:
if options.stables:
stab_mem = [[0, 0.0, 0] for j in xrange(len(stable_set))]
for j in xrange(len(stable_set)):
stab_mem[j] = [int(stable_set[j][1]), stable_set[j][0], j]
write_text_row(stab_mem, "stable_particles.txt")
stable_set_id = []
particle_pixerr = []
for s in stable_set:
stable_set_id.append(s[1])
particle_pixerr.append(s[0])
from fundamentals import rot_shift2D
avet.to_zero()
l = -1
print "average parameters: angle, x-shift, y-shift, mirror"
for j in stable_set_id:
l += 1
print " %4d %4d %12.2f %12.2f %12.2f %1d" % (
l,
j,
stable_set[l][2][0],
stable_set[l][2][1],
stable_set[l][2][2],
int(stable_set[l][2][3]),
)
avet += rot_shift2D(
class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3]
)
avet /= l + 1
avet.set_attr("members", stable_set_id)
avet.set_attr("pix_err", pix_err)
avet.set_attr("pixerr", particle_pixerr)
avet.write_image(args[1])
global_def.BATCH = False
