def ave_ali_err(data1, data2=None, r=25, suffix="_ideal"): ''' This function determines the relative angle, shifts and mirrorness between the two lists of data. It also calculates the mirror consistent rate and average pixel error between two sets of parameters. ''' from utilities import get_params2D, combine_params2 from math import sqrt, sin, pi # Determine relative angle, shifts and mirror alphai, sxi, syi, mirror = align_diff(data1, data2, suffix) # Determine the average pixel error err = 0.0 nima = len(data1) mirror_same = 0 for i in xrange(nima): alpha1, sx1, sy1, mirror1, scale1 = get_params2D(data1[i]) if data2 != None: alpha2, sx2, sy2, mirror2, scale2 = get_params2D(data2[i]) else: alpha2, sx2, sy2, mirror2, scale2 = get_params2D(data1[i], "xform.align2d"+suffix) if abs(mirror1-mirror2) == mirror: mirror_same += 1 alpha12, sx12, sy12, mirror12 = combine_params2(alpha1, sx1, sy1, int(mirror1), alphai, sxi, syi, 0) err += max_2D_pixel_error([alpha12, sx12, sy12], [alpha2, sx2, sy2], r) return alphai, sxi, syi, mirror, float(mirror_same)/nima, err/mirror_same
def align_diff(data1, data2=None, suffix="_ideal"): ''' This function determines the relative angle, shifts and mirrorness between two list of data ''' from utilities import get_params2D nima = len(data1) if data2 != None: nima2 = len(data2) if nima2 != nima: print "Error: Number of images don't agree!" return 0.0, 0.0, 0.0, 0 else: del nima2 # Read the alignment parameters and determine the relative mirrorness ali_params1 = [] ali_params2 = [] for i in xrange(nima): alpha1, sx1, sy1, mirror1, scale1 = get_params2D(data1[i]) if data2 != None: alpha2, sx2, sy2, mirror2, scale2 = get_params2D(data2[i]) else: alpha2, sx2, sy2, mirror2, scale2 = get_params2D(data1[i], "xform.align2d"+suffix) ali_params1.extend([alpha1, sx1, sy1, mirror1]) ali_params2.extend([alpha2, sx2, sy2, mirror2]) return align_diff_params(ali_params1, ali_params2)
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ Input Output [options]
Generate three micrographs, each micrograph contains one projection of a long filament.
Input: Reference Volume, output directory
Output: Three micrographs stored in output directory
sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84
Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
Generate run through example script for helicon
sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
"""
parser = OptionParser(usage, version=SPARXVERSION)
# helicise the Atom coordinates
# generate micrographs of helical filament
parser.add_option(
"--generate_micrograph",
action="store_true",
default=False,
help=
"Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="Use CTF correction")
parser.add_option("--apix",
type="float",
default=-1,
help="pixel size in Angstroms")
parser.add_option(
"--rand_seed",
type="int",
default=14567,
help=
"the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs."
)
parser.add_option("--Cs",
type="float",
default=2.0,
help="Microscope Cs (spherical aberation)")
parser.add_option("--voltage",
type="float",
default=200.0,
help="Microscope voltage in KV")
parser.add_option("--ac",
type="float",
default=10.0,
help="Amplitude contrast (percentage, default=10)")
parser.add_option("--nonoise",
action="store_true",
default=False,
help="Do not add noise to the micrograph.")
# generate initial volume
parser.add_option("--generate_noisycyl",
action="store_true",
default=False,
help="Generate initial volume of noisy cylinder.")
parser.add_option(
"--boxsize",
type="string",
default="100,100,200",
help=
"String containing x , y, z dimensions (separated by comma) in pixels")
parser.add_option("--rad",
type="int",
default=35,
help="Radius of initial volume in pixels")
# generate 2D mask
parser.add_option("--generate_mask",
action="store_true",
default=False,
help="Generate 2D rectangular mask.")
parser.add_option(
"--masksize",
type="string",
default="200,200",
help=
"String containing x and y dimensions (separated by comma) in pixels")
parser.add_option("--maskwidth",
type="int",
default=60,
help="Width of rectangular mask")
# Apply 2D alignment parameters to input stack and output new images to output stack
parser.add_option(
"--applyparams",
action="store_true",
default=False,
help=
"Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack"
)
# Generate run script
parser.add_option("--generate_script",
action="store_true",
default=False,
help="Generate script for helicon run through example")
parser.add_option("--filename",
type="string",
default="runhelicon",
help="Name of run script to generate")
parser.add_option("--seg_ny",
type="int",
default=180,
help="y-dimension of segment used for refinement")
parser.add_option(
"--ptcl_dist",
type="int",
default=15,
help=
"Distance in pixels between adjacent segments windowed from same filament"
)
parser.add_option(
"--fract",
type="float",
default=0.35,
help="Fraction of the volume used for applying helical symmetry.")
(options, args) = parser.parse_args()
if len(args) > 3:
print("usage: " + usage)
print("Please run '" + progname + " -h' for detailed options")
else:
if options.generate_script:
generate_runscript(options.filename, options.seg_ny,
options.ptcl_dist, options.fract)
if options.generate_micrograph:
if options.apix <= 0:
print("Please enter pixel size.")
sys.exit()
generate_helimic(args[0], args[1], options.apix, options.CTF,
options.Cs, options.voltage, options.ac,
options.nonoise, options.rand_seed)
if options.generate_noisycyl:
from utilities import model_cylinder, model_gauss_noise
outvol = args[0]
boxdims = options.boxsize.split(',')
if len(boxdims) < 1 or len(boxdims) > 3:
print(
"Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'"
)
sys.exit()
nx = int(boxdims[0])
if len(boxdims) == 1:
ny = nx
nz = nx
else:
ny = int(boxdims[1])
if len(boxdims) == 3:
nz = int(boxdims[2])
(model_cylinder(options.rad, nx, ny, nz) *
model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol)
if options.generate_mask:
from utilities import model_blank, pad
outvol = args[0]
maskdims = options.masksize.split(',')
if len(maskdims) < 1 or len(maskdims) > 2:
print(
"Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'"
)
sys.exit()
nx = int(maskdims[0])
if len(maskdims) == 1:
ny = nx
else:
ny = int(maskdims[1])
mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1,
0.0)
mask.write_image(outvol)
if options.applyparams:
from utilities import get_im, get_params2D, set_params2D
from fundamentals import cyclic_shift
stack = args[0]
newstack = args[1]
mask = get_im(args[2])
nima = EMUtil.get_image_count(stack)
for im in range(nima):
prj = get_im(stack, im)
alpha, sx, sy, mirror, scale = get_params2D(prj)
prj = cyclic_shift(prj, int(sx))
set_params2D(prj, [0.0, 0., 0.0, 0, 1])
stat = Util.infomask(prj, mask, False)
prj = old_div((prj - stat[0]), stat[1])
ctf_params = prj.get_attr("ctf")
prj.set_attr('ctf_applied', 0)
prj.write_image(newstack, im)
def main():
import global_def
from optparse import OptionParser
from EMAN2 import EMUtil
import os
import sys
from time import time
progname = os.path.basename(sys.argv[0])
usage = progname + " proj_stack output_averages --MPI"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--img_per_group",
type="int",
default=100,
help="number of images per group")
parser.add_option("--radius",
type="int",
default=-1,
help="radius for alignment")
parser.add_option(
"--xr",
type="string",
default="2 1",
help="range for translation search in x direction, search is +/xr")
parser.add_option(
"--yr",
type="string",
default="-1",
help=
"range for translation search in y direction, search is +/yr (default = same as xr)"
)
parser.add_option(
"--ts",
type="string",
default="1 0.5",
help=
"step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional"
)
parser.add_option(
"--iter",
type="int",
default=30,
help="number of iterations within alignment (default = 30)")
parser.add_option(
"--num_ali",
type="int",
default=5,
help="number of alignments performed for stability (default = 5)")
parser.add_option("--thld_err",
type="float",
default=1.0,
help="threshold of pixel error (default = 1.732)")
parser.add_option(
"--grouping",
type="string",
default="GRP",
help=
"do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size"
)
parser.add_option(
"--delta",
type="float",
default=-1.0,
help="angular step for reference projections (required for GEV method)"
)
parser.add_option(
"--fl",
type="float",
default=0.3,
help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
parser.add_option(
"--aa",
type="float",
default=0.2,
help="fall-off of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--CTF",
action="store_true",
default=False,
help="Consider CTF correction during the alignment ")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version")
(options, args) = parser.parse_args()
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT
from applications import MPI_start_end, within_group_refinement, ali2d_ras
from pixel_error import multi_align_stability
from utilities import send_EMData, recv_EMData
from utilities import get_image, bcast_number_to_all, set_params2D, get_params2D
from utilities import group_proj_by_phitheta, model_circle, get_input_from_string
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 2:
stack = args[0]
outdir = args[1]
else:
ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid)
exit()
if not options.MPI:
ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid)
exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
#if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid)
#mpi_barrier(MPI_COMM_WORLD)
img_per_grp = options.img_per_group
radius = options.radius
ite = options.iter
num_ali = options.num_ali
thld_err = options.thld_err
xrng = get_input_from_string(options.xr)
if options.yr == "-1": yrng = xrng
else: yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
if radius == -1: radius = nx / 2 - 2
mask = model_circle(radius, nx, nx)
st = time()
if options.grouping == "GRP":
if myid == main_node:
print " A ", myid, " ", time() - st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
"psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
# Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta,
# So I will briefly explain it here
# proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers
# except for the last one. Depending on the number of particles left, they will either form a
# group or append themselves to the last group
# angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi,
# theta) is the projection angle of the center of the group, delta is the range of this group
# mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates
# whether it should take mirror position.
# In this program angle_list and mirror list are not of interest.
proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(
proj_params, img_per_grp=img_per_grp)
del proj_params
print " B number of groups ", myid, " ", len(
proj_list_all), time() - st
mpi_barrier(MPI_COMM_WORLD)
# Number of groups, actually there could be one or two more groups, since the size of the remaining group varies
# we will simply assign them to main node.
n_grp = nima / img_per_grp - 1
# Divide proj_list_all equally to all nodes, and becomes proj_list
proj_list = []
for i in xrange(n_grp):
proc_to_stay = i % number_of_proc
if proc_to_stay == main_node:
if myid == main_node: proj_list.append(proj_list_all[i])
elif myid == main_node:
mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT,
proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
elif myid == proc_to_stay:
img_per_grp = mpi_recv(1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
img_per_grp = int(img_per_grp[0])
temp = mpi_recv(img_per_grp, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
proj_list.append(map(int, temp))
del temp
mpi_barrier(MPI_COMM_WORLD)
print " C ", myid, " ", time() - st
if myid == main_node:
# Assign the remaining groups to main_node
for i in xrange(n_grp, len(proj_list_all)):
proj_list.append(proj_list_all[i])
del proj_list_all, angle_list, mirror_list
# Compute stability per projection projection direction, equal number assigned, thus overlaps
elif options.grouping == "GEV":
if options.delta == -1.0:
ERROR(
"Angular step for reference projections is required for GEV method",
"sxproj_stability", 1)
from utilities import even_angles, nearestk_to_refdir, getvec
refproj = even_angles(options.delta)
img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid)
# Now each processor keeps its own share of reference projections
refprojdir = refproj[img_begin:img_end]
del refproj
ref_ang = [0.0] * (len(refprojdir) * 2)
for i in xrange(len(refprojdir)):
ref_ang[i * 2] = refprojdir[0][0]
ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1
print " A ", myid, " ", time() - st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
# the solution below is very slow, do not use it unless there is a problem with the i/O
"""
for i in xrange(number_of_proc):
if myid == i:
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
mpi_barrier(MPI_COMM_WORLD)
"""
print " B ", myid, " ", time() - st
proj_ang = [0.0] * (nima * 2)
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
proj_ang[i * 2] = dp["phi"]
proj_ang[i * 2 + 1] = dp["theta"]
print " C ", myid, " ", time() - st
asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp)
del proj_ang, ref_ang
proj_list = []
for i in xrange(len(refprojdir)):
proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp])
del asi
print " D ", myid, " ", time() - st
#from sys import exit
#exit()
# Compute stability per projection
elif options.grouping == "PPR":
print " A ", myid, " ", time() - st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
print " B ", myid, " ", time() - st
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
"psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
print " C ", myid, " ", time() - st
from utilities import nearest_proj
proj_list, mirror_list = nearest_proj(
proj_params, img_per_grp,
range(img_begin, img_begin + 1)) #range(img_begin, img_end))
refprojdir = proj_params[img_begin:img_end]
del proj_params, mirror_list
print " D ", myid, " ", time() - st
else:
ERROR("Incorrect projection grouping option", "sxproj_stability", 1)
"""
from utilities import write_text_file
for i in xrange(len(proj_list)):
write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid))
"""
###########################################################################################################
# Begin stability test
from utilities import get_params_proj, read_text_file
#if myid == 0:
# from utilities import read_text_file
# proj_list[0] = map(int, read_text_file("lggrpp0.txt"))
from utilities import model_blank
aveList = [model_blank(nx, ny)] * len(proj_list)
if options.grouping == "GRP":
refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list)
for i in xrange(len(proj_list)):
print " E ", myid, " ", time() - st
class_data = EMData.read_images(stack, proj_list[i])
#print " R ",myid," ",time()-st
if options.CTF:
from filter import filt_ctf
for im in xrange(len(class_data)): # MEM LEAK!!
atemp = class_data[im].copy()
btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1)
class_data[im] = btemp
#class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
try:
t = im.get_attr(
"xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
#print " F ",myid," ",time()-st
# Here, we perform realignment num_ali times
all_ali_params = []
for j in xrange(num_ali):
if (xrng[0] == 0.0 and yrng[0] == 0.0):
avet = ali2d_ras(class_data,
randomize=True,
ir=1,
ou=radius,
rs=1,
step=1.0,
dst=90.0,
maxit=ite,
check_mirror=True,
FH=options.fl,
FF=options.aa)
else:
avet = within_group_refinement(class_data, mask, True, 1,
radius, 1, xrng, yrng, step,
90.0, ite, options.fl,
options.aa)
ali_params = []
for im in xrange(len(class_data)):
alpha, sx, sy, mirror, scale = get_params2D(class_data[im])
ali_params.extend([alpha, sx, sy, mirror])
all_ali_params.append(ali_params)
#aveList[i] = avet
#print " G ",myid," ",time()-st
del ali_params
# We determine the stability of this group here.
# stable_set contains all particles deemed stable, it is a list of list
# each list has two elements, the first is the pixel error, the second is the image number
# stable_set is sorted based on pixel error
#from utilities import write_text_file
#write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid)
stable_set, mir_stab_rate, average_pix_err = multi_align_stability(
all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1)
#print " H ",myid," ",time()-st
if (len(stable_set) > 5):
stable_set_id = []
members = []
pix_err = []
# First put the stable members into attr 'members' and 'pix_err'
for s in stable_set:
# s[1] - number in this subset
stable_set_id.append(s[1])
# the original image number
members.append(proj_list[i][s[1]])
pix_err.append(s[0])
# Then put the unstable members into attr 'members' and 'pix_err'
from fundamentals import rot_shift2D
avet.to_zero()
if options.grouping == "GRP":
aphi = 0.0
atht = 0.0
vphi = 0.0
vtht = 0.0
l = -1
for j in xrange(len(proj_list[i])):
# Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses
if j in stable_set_id:
l += 1
avet += rot_shift2D(class_data[j], stable_set[l][2][0],
stable_set[l][2][1],
stable_set[l][2][2],
stable_set[l][2][3])
if options.grouping == "GRP":
phi, theta, psi, sxs, sys = get_params_proj(
class_data[j])
if (theta > 90.0):
phi = (phi + 540.0) % 360.0
theta = 180.0 - theta
aphi += phi
atht += theta
vphi += phi * phi
vtht += theta * theta
else:
members.append(proj_list[i][j])
pix_err.append(99999.99)
aveList[i] = avet.copy()
if l > 1:
l += 1
aveList[i] /= l
if options.grouping == "GRP":
aphi /= l
atht /= l
vphi = (vphi - l * aphi * aphi) / l
vtht = (vtht - l * atht * atht) / l
from math import sqrt
refprojdir[i] = [
aphi, atht,
(sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0
]
# Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION
aveList[i].set_attr('members', members)
aveList[i].set_attr('refprojdir', refprojdir[i])
aveList[i].set_attr('pixerr', pix_err)
else:
print " empty group ", i, refprojdir[i]
aveList[i].set_attr('members', [-1])
aveList[i].set_attr('refprojdir', refprojdir[i])
aveList[i].set_attr('pixerr', [99999.])
del class_data
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(aveList)):
aveList[im].write_image(args[1], km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pixerr', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
ave.write_image(args[1], km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pixerr')
mpi_send(members, len(members), MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
global_def.BATCH = False
mpi_barrier(MPI_COMM_WORLD)
from mpi import mpi_finalize
mpi_finalize()
def main():
from utilities import get_input_from_string
progname = os.path.basename(sys.argv[0])
usage = progname + " stack output_average --radius=particle_radius --xr=xr --yr=yr --ts=ts --thld_err=thld_err --num_ali=num_ali --fl=fl --aa=aa --CTF --verbose --stables"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--radius", type="int", default=-1, help=" particle radius for alignment")
parser.add_option("--xr", type="string" , default="2 1", help="range for translation search in x direction, search is +/xr (default 2,1)")
parser.add_option("--yr", type="string" , default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)")
parser.add_option("--ts", type="string" , default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default: 1,0.5)")
parser.add_option("--thld_err", type="float", default=0.75, help="threshld of pixel error (default = 0.75)")
parser.add_option("--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)")
parser.add_option("--maxit", type="int", default=30, help="number of iterations for each xr (default = 30)")
parser.add_option("--fl", type="float" , default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter (default = 0.3)")
parser.add_option("--aa", type="float" , default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter (default = 0.2)")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction during the alignment ")
parser.add_option("--verbose", action="store_true", default=False, help="print individual pixel error (default = False)")
parser.add_option("--stables", action="store_true", default=False, help="output the stable particles number in file (default = False)")
parser.add_option("--method", type="string" , default=" ", help="SHC (standard method is default when flag is ommitted)")
(options, args) = parser.parse_args()
if len(args) != 1 and len(args) != 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
else:
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from applications import within_group_refinement, ali2d_ras
from pixel_error import multi_align_stability
from utilities import write_text_file, write_text_row
global_def.BATCH = True
xrng = get_input_from_string(options.xr)
if options.yr == "-1": yrng = xrng
else : yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
class_data = EMData.read_images(args[0])
nx = class_data[0].get_xsize()
ou = options.radius
num_ali = options.num_ali
if ou == -1: ou = nx/2-2
from utilities import model_circle, get_params2D, set_params2D
mask = model_circle(ou, nx, nx)
if options.CTF :
from filter import filt_ctf
for im in xrange(len(class_data)):
# Flip phases
class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
im.set_attr("previousmax", -1.0e10)
try:
t = im.get_attr("xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
all_ali_params = []
for ii in xrange(num_ali):
ali_params = []
if options.verbose:
ALPHA = []
SX = []
SY = []
MIRROR = []
if( xrng[0] == 0.0 and yrng[0] == 0.0 ):
avet = ali2d_ras(class_data, randomize = True, ir = 1, ou = ou, rs = 1, step = 1.0, dst = 90.0, \
maxit = options.maxit, check_mirror = True, FH=options.fl, FF=options.aa)
else:
avet = within_group_refinement(class_data, mask, True, 1, ou, 1, xrng, yrng, step, 90.0, \
maxit = options.maxit, FH=options.fl, FF=options.aa, method = options.method)
from utilities import info
#print " avet ",info(avet)
for im in class_data:
alpha, sx, sy, mirror, scale = get_params2D(im)
ali_params.extend([alpha, sx, sy, mirror])
if options.verbose:
ALPHA.append(alpha)
SX.append(sx)
SY.append(sy)
MIRROR.append(mirror)
all_ali_params.append(ali_params)
if options.verbose:
write_text_file([ALPHA, SX, SY, MIRROR], "ali_params_run_%d"%ii)
"""
avet = class_data[0]
from utilities import read_text_file
all_ali_params = []
for ii in xrange(5):
temp = read_text_file( "ali_params_run_%d"%ii,-1)
uuu = []
for k in xrange(len(temp[0])):
uuu.extend([temp[0][k],temp[1][k],temp[2][k],temp[3][k]])
all_ali_params.append(uuu)
"""
stable_set, mir_stab_rate, pix_err = multi_align_stability(all_ali_params, 0.0, 10000.0, options.thld_err, options.verbose, 2*ou+1)
print "%4s %20s %20s %20s %30s %6.2f"%("", "Size of set", "Size of stable set", "Mirror stab rate", "Pixel error prior to pruning the set above threshold of",options.thld_err)
print "Average stat: %10d %20d %20.2f %15.2f"%( len(class_data), len(stable_set), mir_stab_rate, pix_err)
if( len(stable_set) > 0):
if options.stables:
stab_mem = [[0,0.0,0] for j in xrange(len(stable_set))]
for j in xrange(len(stable_set)): stab_mem[j] = [int(stable_set[j][1]), stable_set[j][0], j]
write_text_row(stab_mem, "stable_particles.txt")
stable_set_id = []
particle_pixerr = []
for s in stable_set:
stable_set_id.append(s[1])
particle_pixerr.append(s[0])
from fundamentals import rot_shift2D
avet.to_zero()
l = -1
print "average parameters: angle, x-shift, y-shift, mirror"
for j in stable_set_id:
l += 1
print " %4d %4d %12.2f %12.2f %12.2f %1d"%(l,j, stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], int(stable_set[l][2][3]))
avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] )
avet /= (l+1)
avet.set_attr('members', stable_set_id)
avet.set_attr('pix_err', pix_err)
avet.set_attr('pixerr', particle_pixerr)
avet.write_image(args[1])
global_def.BATCH = False
def main():
import sys
import os
import math
import random
import pyemtbx.options
import time
from random import random, seed, randint
from optparse import OptionParser
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Forms chains of 2D images based on their similarities.
Functionality:
4. Order a 2-D stack of image based on pair-wise similarity (computed as a cross-correlation coefficent).
Options 1-3 require image stack to be aligned. The program will apply orientation parameters if present in headers.
The ways to use the program:
4.1 Use option initial to specify which image will be used as an initial seed to form the chain.
sxprocess.py input_stack.hdf output_stack.hdf --initial=23 --radius=25
4.2 If options initial is omitted, the program will determine which image best serves as initial seed to form the chain
sxprocess.py input_stack.hdf output_stack.hdf --radius=25
4.3 Use option circular to form a circular chain.
sxprocess.py input_stack.hdf output_stack.hdf --circular--radius=25
4.4 New circular code based on pairwise alignments
sxprocess.py aclf.hdf chain.hdf circle.hdf --align --radius=25 --xr=2 --pairwiseccc=lcc.txt
4.5 Circular ordering based on pairwise alignments
sxprocess.py vols.hdf chain.hdf mask.hdf --dd --radius=25
"""
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--dd", action="store_true", help="Circular ordering without adjustment of orientations", default=False)
parser.add_option("--circular", action="store_true", help="Select circular ordering (first image has to be similar to the last)", default=False)
parser.add_option("--align", action="store_true", help="Compute all pairwise alignments and for the table of their similarities find the best chain", default=False)
parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering")
# import params for 2D alignment
parser.add_option("--ou", type="int", default=-1, help="outer radius for 2D alignment < nx/2-1 (set to the radius of the particle)")
parser.add_option("--xr", type="int", default=0, help="range for translation search in x direction, search is +/xr (0)")
parser.add_option("--yr", type="int", default=0, help="range for translation search in y direction, search is +/yr (0)")
#parser.add_option("--nomirror", action="store_true", default=False, help="Disable checking mirror orientations of images (default False)")
parser.add_option("--pairwiseccc", type="string", default= None, help="Input/output pairwise ccc file")
(options, args) = parser.parse_args()
global_def.BATCH = True
if options.dd:
nargs = len(args)
if nargs != 3:
print "must provide name of input and two output files!"
return
stack = args[0]
new_stack = args[1]
from utilities import model_circle
from statistics import ccc
from statistics import mono
lend = EMUtil.get_image_count(stack)
lccc = [None]*(lend*(lend-1)/2)
for i in xrange(lend-1):
v1 = get_im( stack, i )
if( i == 0 and nargs == 2):
nx = v1.get_xsize()
ny = v1.get_ysize()
nz = v1.get_ysize()
if options.ou < 1 : radius = nx//2-2
else: radius = options.ou
mask = model_circle(radius, nx, ny, nz)
else:
mask = get_im(args[2])
for j in xrange(i+1, lend):
lccc[mono(i,j)] = [ccc(v1, get_im( stack, j ), mask), 0]
order = tsp(lccc)
if(len(order) != lend):
print " problem with data length"
from sys import exit
exit()
print "Total sum of cccs :",TotalDistance(order, lccc)
print "ordering :",order
for i in xrange(lend): get_im(stack, order[i]).write_image( new_stack, i )
elif options.align:
nargs = len(args)
if nargs != 3:
print "must provide name of input and two output files!"
return
from utilities import get_params2D, model_circle
from fundamentals import rot_shift2D
from statistics import ccc
from time import time
from alignment import align2d, align2d_scf
from multi_shc import mult_transform
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
"""
# will align anyway
try:
ttt = d[0].get_attr('xform.params2d')
for i in xrange(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
"""
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.ou < 1 : radius = nx//2-2
else: radius = options.ou
mask = model_circle(radius, nx, ny)
if(options.xr < 0): xrng = 0
else: xrng = options.xr
if(options.yr < 0): yrng = xrng
else: yrng = options.yr
initial = max(options.initial, 0)
from statistics import mono
lend = len(d)
lccc = [None]*(lend*(lend-1)/2)
from utilities import read_text_row
if options.pairwiseccc == None or not os.path.exists(options.pairwiseccc) :
st = time()
for i in xrange(lend-1):
for j in xrange(i+1, lend):
# j>i meaning mono entry (i,j) or (j,i) indicates T i->j (from smaller index to larger)
#alpha, sx, sy, mir, peak = align2d(d[i],d[j], xrng, yrng, step=options.ts, first_ring=options.ir, last_ring=radius, mode = "F")
alpha, sx, sy, mir, peak = align2d_scf(d[i],d[j], xrng, yrng, ou=radius)
lccc[mono(i,j)] = [ccc(d[j], rot_shift2D(d[i], alpha, sx, sy, mir, 1.0), mask), alpha, sx, sy, mir]
#print " %4d %10.1f"%(i,time()-st)
if(not os.path.exists(options.pairwiseccc)):
from utilities import write_text_row
write_text_row([[initial,0,0,0,0]]+lccc,options.pairwiseccc)
elif(os.path.exists(options.pairwiseccc)):
lccc = read_text_row(options.pairwiseccc)
initial = int(lccc[0][0] + 0.1)
del lccc[0]
for i in xrange(len(lccc)):
T = Transform({"type":"2D","alpha":lccc[i][1],"tx":lccc[i][2],"ty":lccc[i][3],"mirror":int(lccc[i][4]+0.1)})
lccc[i] = [lccc[i][0],T]
tdummy = Transform({"type":"2D"})
maxsum = -1.023
for m in xrange(0,lend):#initial, initial+1):
indc = range( lend )
lsnake = [[m, tdummy, 0.0]]
del indc[m]
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in xrange(len(indc)):
cuc = lccc[mono(indc[i], lsnake[-1][0])][0]
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
# Here we need transformation from the current to the previous,
# meaning indc[i] -> lsnake[-1][0]
T = lccc[mono(indc[i], lsnake[-1][0])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if( indc[i] > lsnake[-1][0] ): T = T.inverse()
lsnake.append([qi,T, maxcit])
lsum += maxcit
del indc[indc.index(qi)]
T = lccc[mono(indc[-1], lsnake[-1][0])][1]
if( indc[-1] > lsnake[-1][0]): T = T.inverse()
lsnake.append([indc[-1], T, lccc[mono(indc[-1], lsnake[-1][0])][0]])
print " initial image and lsum ",m,lsum
#print lsnake
if(lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in xrange(lend)]
print " Initial image selected : ",init,maxsum," ",TotalDistance([snake[m][0] for m in xrange(lend)], lccc)
#for q in snake: print q
from copy import deepcopy
trans=deepcopy([snake[i][1] for i in xrange(len(snake))])
print [snake[i][0] for i in xrange(len(snake))]
"""
for m in xrange(lend):
prms = trans[m].get_params("2D")
print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
"""
for k in xrange(lend-2,0,-1):
T = snake[k][1]
for i in xrange(k+1, lend):
trans[i] = T*trans[i]
# To add - apply all transformations and do the overall centering.
for m in xrange(lend):
prms = trans[m].get_params("2D")
#print " %3d %7.1f %7.1f %7.1f %2d %6.2f"%(snake[m][0], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"], snake[m][2])
#rot_shift2D(d[snake[m][0]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image(new_stack, m)
rot_shift2D(d[snake[m][0]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(new_stack, m)
order = tsp(lccc)
if(len(order) != lend):
print " problem with data length"
from sys import exit
exit()
print TotalDistance(order, lccc)
print order
ibeg = order.index(init)
order = [order[(i+ibeg)%lend] for i in xrange(lend)]
print TotalDistance(order, lccc)
print order
snake = [tdummy]
for i in xrange(1,lend):
# Here we need transformation from the current to the previous,
# meaning order[i] -> order[i-1]]
T = lccc[mono(order[i], order[i-1])][1]
# If direction is from larger to smaller index, the transformation has to be inverted
if( order[i] > order[i-1] ): T = T.inverse()
snake.append(T)
assert(len(snake) == lend)
from copy import deepcopy
trans = deepcopy(snake)
for k in xrange(lend-2,0,-1):
T = snake[k]
for i in xrange(k+1, lend):
trans[i] = T*trans[i]
# Try to smooth the angles - complicated, I am afraid one would have to use angles forward and backwards
# and find their average??
# In addition, one would have to recenter them
"""
trms = []
for m in xrange(lend):
prms = trans[m].get_params("2D")
trms.append([prms["alpha"], prms["mirror"]])
for i in xrange(3):
for m in xrange(lend):
mb = (m-1)%lend
me = (m+1)%lend
# angles order mb,m,me
# calculate predicted angles mb->m
"""
for m in xrange(lend):
prms = trans[m].get_params("2D")
#rot_shift2D(d[order[m]], prms["alpha"], prms["tx"], prms["ty"], prms["mirror"]).write_image("metro.hdf", m)
rot_shift2D(d[order[m]], prms["alpha"], 0.0,0.0, prms["mirror"]).write_image(args[2], m)
"""
# This was an effort to get number of loops, inconclusive, to say the least
from numpy import outer, zeros, float32, sqrt
lend = len(d)
cor = zeros(lend,float32)
cor = outer(cor, cor)
for i in xrange(lend): cor[i][i] = 1.0
for i in xrange(lend-1):
for j in xrange(i+1, lend):
cor[i,j] = lccc[mono(i,j)][0]
cor[j,i] = cor[i,j]
lmbd, eigvec = pca(cor)
from utilities import write_text_file
nvec=20
print [lmbd[j] for j in xrange(nvec)]
print " G"
mm = [-1]*lend
for i in xrange(lend): # row
mi = -1.0e23
for j in xrange(nvec):
qt = eigvec[j][i]
if(abs(qt)>mi):
mi = abs(qt)
mm[i] = j
for j in xrange(nvec):
qt = eigvec[j][i]
print round(qt,3), # eigenvector
print mm[i]
print
for j in xrange(nvec):
qt = []
for i in xrange(lend):
if(mm[i] == j): qt.append(i)
if(len(qt)>0): write_text_file(qt,"loop%02d.txt"%j)
"""
"""
print [lmbd[j] for j in xrange(nvec)]
print " B"
mm = [-1]*lend
for i in xrange(lend): # row
mi = -1.0e23
for j in xrange(nvec):
qt = eigvec[j][i]/sqrt(lmbd[j])
if(abs(qt)>mi):
mi = abs(qt)
mm[i] = j
for j in xrange(nvec):
qt = eigvec[j][i]/sqrt(lmbd[j])
print round(qt,3), # eigenvector
print mm[i]
print
"""
"""
lend=3
cor = zeros(lend,float32)
cor = outer(cor, cor)
cor[0][0] =136.77
cor[0][1] = 79.15
cor[0][2] = 37.13
cor[1][0] = 79.15
cor[2][0] = 37.13
cor[1][1] = 50.04
cor[1][2] = 21.65
cor[2][1] = 21.65
cor[2][2] = 13.26
lmbd, eigvec = pca(cor)
print lmbd
print eigvec
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i], # eigenvector
print
print " B"
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i]/sqrt(lmbd[j]), # eigenvector
print
print " G"
for i in xrange(lend): # row
for j in xrange(lend): print eigvec[j][i]*sqrt(lmbd[j]), # eigenvector
print
"""
else:
nargs = len(args)
if nargs != 2:
print "must provide name of input and output file!"
return
from utilities import get_params2D, model_circle
from fundamentals import rot_shift2D
from statistics import ccc
from time import time
from alignment import align2d
from multi_shc import mult_transform
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
try:
ttt = d[0].get_attr('xform.params2d')
for i in xrange(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.radius < 1 : radius = nx//2-2
else: radius = options.radius
mask = model_circle(radius, nx, ny)
init = options.initial
if init > -1 :
print " initial image: %d" % init
temp = d[init].copy()
temp.write_image(new_stack, 0)
del d[init]
k = 1
lsum = 0.0
while len(d) > 1:
maxcit = -111.
for i in xrange(len(d)):
cuc = ccc(d[i], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = i
# print k, maxcit
lsum += maxcit
temp = d[qi].copy()
del d[qi]
temp.write_image(new_stack, k)
k += 1
print lsum
d[0].write_image(new_stack, k)
else:
if options.circular :
print "Using options.circular"
# figure the "best circular" starting image
maxsum = -1.023
for m in xrange(len(d)):
indc = range(len(d) )
lsnake = [-1]*(len(d)+1)
lsnake[0] = m
lsnake[-1] = m
del indc[m]
temp = d[m].copy()
lsum = 0.0
direction = +1
k = 1
while len(indc) > 1:
maxcit = -111.
for i in xrange(len(indc)):
cuc = ccc(d[indc[i]], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
lsnake[k] = qi
lsum += maxcit
del indc[indc.index(qi)]
direction = -direction
for i in xrange( 1,len(d) ):
if( direction > 0 ):
if(lsnake[i] == -1):
temp = d[lsnake[i-1]].copy()
#print " forw ",lsnake[i-1]
k = i
break
else:
if(lsnake[len(d) - i] == -1):
temp = d[lsnake[len(d) - i +1]].copy()
#print " back ",lsnake[len(d) - i +1]
k = len(d) - i
break
lsnake[lsnake.index(-1)] = indc[-1]
#print " initial image and lsum ",m,lsum
#print lsnake
if(lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in xrange(len(d))]
print " Initial image selected : ",init,maxsum
print lsnake
for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m)
else:
# figure the "best" starting image
maxsum = -1.023
for m in xrange(len(d)):
indc = range(len(d) )
lsnake = [m]
del indc[m]
temp = d[m].copy()
lsum = 0.0
while len(indc) > 1:
maxcit = -111.
for i in xrange(len(indc)):
cuc = ccc(d[indc[i]], temp, mask)
if cuc > maxcit:
maxcit = cuc
qi = indc[i]
lsnake.append(qi)
lsum += maxcit
temp = d[qi].copy()
del indc[indc.index(qi)]
lsnake.append(indc[-1])
#print " initial image and lsum ",m,lsum
#print lsnake
if(lsum > maxsum):
maxsum = lsum
init = m
snake = [lsnake[i] for i in xrange(len(d))]
print " Initial image selected : ",init,maxsum
print lsnake
for m in xrange(len(d)): d[snake[m]].write_image(new_stack, m)
from utilities import get_params2D, model_circle
from fundamentals import rot_shift2D
from statistics import ccc
from time import time
from alignment import align2d
from multi_shc import mult_transform
stack = args[0]
new_stack = args[1]
d = EMData.read_images(stack)
try:
ttt = d[0].get_attr('xform.params2d')
for i in xrange(len(d)):
alpha, sx, sy, mirror, scale = get_params2D(d[i])
d[i] = rot_shift2D(d[i], alpha, sx, sy, mirror)
except:
pass
nx = d[0].get_xsize()
ny = d[0].get_ysize()
if options.radius < 1 : radius = nx//2-2
else: radius = options.radius
mask = model_circle(radius, nx, ny)
init = options.initial
if init > -1 :
print " initial image: %d" % init
temp = d[init].copy()
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ Input Output [options]
Generate three micrographs, each micrograph contains one projection of a long filament.
Input: Reference Volume, output directory
Output: Three micrographs stored in output directory
sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84
Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
Generate run through example script for helicon
sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
"""
parser = OptionParser(usage,version=SPARXVERSION)
# helicise the Atom coordinates
# generate micrographs of helical filament
parser.add_option("--generate_micrograph", action="store_true", default=False, help="Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction")
parser.add_option("--apix", type="float", default= -1, help="pixel size in Angstroms")
parser.add_option("--rand_seed", type="int", default=14567, help="the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs.")
parser.add_option("--Cs", type="float", default= 2.0, help="Microscope Cs (spherical aberation)")
parser.add_option("--voltage", type="float", default=200.0, help="Microscope voltage in KV")
parser.add_option("--ac", type="float", default=10.0, help="Amplitude contrast (percentage, default=10)")
parser.add_option("--nonoise", action="store_true", default=False, help="Do not add noise to the micrograph.")
# generate initial volume
parser.add_option("--generate_noisycyl", action="store_true", default=False, help="Generate initial volume of noisy cylinder.")
parser.add_option("--boxsize", type="string", default="100,100,200", help="String containing x , y, z dimensions (separated by comma) in pixels")
parser.add_option("--rad", type="int", default=35, help="Radius of initial volume in pixels")
# generate 2D mask
parser.add_option("--generate_mask", action="store_true", default=False, help="Generate 2D rectangular mask.")
parser.add_option("--masksize", type="string", default="200,200", help="String containing x and y dimensions (separated by comma) in pixels")
parser.add_option("--maskwidth", type="int", default=60, help="Width of rectangular mask")
# Apply 2D alignment parameters to input stack and output new images to output stack
parser.add_option("--applyparams", action="store_true", default=False, help="Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack")
# Generate run script
parser.add_option("--generate_script", action="store_true", default=False, help="Generate script for helicon run through example")
parser.add_option("--filename", type="string", default="runhelicon", help="Name of run script to generate")
parser.add_option("--seg_ny", type="int", default=180, help="y-dimension of segment used for refinement")
parser.add_option("--ptcl_dist", type="int", default=15, help="Distance in pixels between adjacent segments windowed from same filament")
parser.add_option("--fract", type="float", default=0.35, help="Fraction of the volume used for applying helical symmetry.")
(options, args) = parser.parse_args()
if len(args) > 3:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
else:
if options.generate_script:
generate_runscript(options.filename, options.seg_ny, options.ptcl_dist, options.fract)
if options.generate_micrograph:
if options.apix <= 0:
print "Please enter pixel size."
sys.exit()
generate_helimic(args[0], args[1], options.apix, options.CTF, options.Cs, options.voltage, options.ac, options.nonoise, options.rand_seed)
if options.generate_noisycyl:
from utilities import model_cylinder, model_gauss_noise
outvol = args[0]
boxdims = options.boxsize.split(',')
if len(boxdims) < 1 or len(boxdims) > 3:
print "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'"
sys.exit()
nx= int(boxdims[0])
if len(boxdims) == 1:
ny = nx
nz = nx
else:
ny = int(boxdims[1])
if len(boxdims) == 3:
nz = int(boxdims[2])
(model_cylinder(options.rad,nx, ny, nz)*model_gauss_noise(1.0, nx, ny, nz) ).write_image(outvol)
if options.generate_mask:
from utilities import model_blank, pad
outvol = args[0]
maskdims = options.masksize.split(',')
if len(maskdims) < 1 or len(maskdims) > 2:
print "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'"
sys.exit()
nx= int(maskdims[0])
if len(maskdims) == 1:
ny = nx
else:
ny = int(maskdims[1])
mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1, 0.0)
mask.write_image(outvol)
if options.applyparams:
from utilities import get_im, get_params2D, set_params2D
from fundamentals import cyclic_shift
stack = args[0]
newstack = args[1]
mask = get_im(args[2])
nima = EMUtil.get_image_count(stack)
for im in xrange(nima):
prj = get_im(stack,im)
alpha, sx, sy, mirror, scale = get_params2D(prj)
prj = cyclic_shift(prj, int(sx))
set_params2D(prj, [0.0,0.,0.0,0,1])
stat = Util.infomask(prj , mask, False )
prj= (prj-stat[0])/stat[1]
ctf_params = prj.get_attr("ctf")
prj.set_attr('ctf_applied', 0)
prj.write_image(newstack, im)
def compare(compare_ref_free, outfile_repro,ref_free_output,yrng, xrng, rstep,nx,apix,ref_free_cutoff, nproc, myid, main_node):
from alignment import Numrinit, ringwe, Applyws
from random import seed, randint
from utilities import get_params2D, set_params2D, model_circle, inverse_transform2, combine_params2
from fundamentals import rot_shift2D
from mpi import MPI_COMM_WORLD, mpi_barrier, mpi_bcast, MPI_INT
from statistics import fsc_mask
from filter import fit_tanh
from numpy import array
fout = "%s.hdf" % ref_free_output
frc_out = "%s_frc" % ref_free_output
res_out = "%s_res" % ref_free_output
nima = EMUtil.get_image_count(compare_ref_free)
image_start, image_end = MPI_start_end(nima, nproc, myid)
ima = EMData()
ima.read_image(compare_ref_free, image_start)
last_ring = nx/2-2
first_ring = 1
mask = model_circle(last_ring, nx, nx)
refi = []
numref = EMUtil.get_image_count(outfile_repro)
cnx = nx/2 +1
cny = cnx
mode = "F"
numr = Numrinit(first_ring, last_ring, rstep, mode)
wr = ringwe(numr, mode)
ima.to_zero()
for j in xrange(numref):
temp = EMData()
temp.read_image(outfile_repro, j)
# even, odd, numer of even, number of images. After frc, totav
refi.append(temp)
# for each node read its share of data
data = EMData.read_images(compare_ref_free, range(image_start, image_end))
for im in xrange(image_start, image_end):
data[im-image_start].set_attr('ID', im)
set_params2D(data[im-image_start],[0,0,0,0,1])
ringref = []
for j in xrange(numref):
refi[j].process_inplace("normalize.mask", {"mask":mask, "no_sigma":1}) # normalize reference images to N(0,1)
cimage = Util.Polar2Dm(refi[j], cnx, cny, numr, mode)
Util.Frngs(cimage, numr)
Applyws(cimage, numr, wr)
ringref.append(cimage)
if myid == main_node: seed(1000)
data_shift = []
frc = []
res = []
for im in xrange(image_start, image_end):
alpha, sx, sy, mirror, scale = get_params2D(data[im-image_start])
alphai, sxi, syi, scalei = inverse_transform2(alpha, sx, sy, 1.0)
# normalize
data[im-image_start].process_inplace("normalize.mask", {"mask":mask, "no_sigma":1}) # subtract average under the mask
# align current image to the reference
[angt, sxst, syst, mirrort, xiref, peakt] = Util.multiref_polar_ali_2d(data[im-image_start], ringref, xrng, yrng, 1, mode, numr, cnx+sxi, cny+syi)
iref = int(xiref)
[alphan, sxn, syn, mn] = combine_params2(0.0, -sxi, -syi, 0, angt, sxst, syst, (int)(mirrort))
set_params2D(data[im-image_start], [alphan, sxn, syn, int(mn), scale])
temp = rot_shift2D(data[im-image_start], alphan, sxn, syn, mn)
temp.set_attr('assign',iref)
tfrc = fsc_mask(temp,refi[iref],mask = mask)
temp.set_attr('frc',tfrc[1])
res = fit_tanh(tfrc)
temp.set_attr('res',res)
data_shift.append(temp)
for node in xrange(nproc):
if myid == node:
for image in data_shift:
image.write_image(fout,-1)
refindex = image.get_attr('assign')
refi[refindex].write_image(fout,-1)
mpi_barrier(MPI_COMM_WORLD)
rejects = []
if myid == main_node:
a = EMData()
index = 0
frc = []
res = []
temp = []
classes = []
for im in xrange(nima):
a.read_image(fout, index)
frc.append(a.get_attr("frc"))
if ref_free_cutoff != -1: classes.append(a.get_attr("class_ptcl_idxs"))
tmp = a.get_attr("res")
temp.append(tmp[0])
res.append("%12f" %(apix/tmp[0]))
res.append("\n")
index = index + 2
res_num = array(temp)
mean_score = res_num.mean(axis=0)
std_score = res_num.std(axis=0)
std = std_score / 2
if ref_free_cutoff !=-1:
cutoff = mean_score - std * ref_free_cutoff
reject = res_num < cutoff
index = 0
for i in reject:
if i: rejects.extend(classes[index])
index = index + 1
rejects.sort()
length = mpi_bcast(len(rejects),1,MPI_INT,main_node, MPI_COMM_WORLD)
rejects = mpi_bcast(rejects,length , MPI_INT, main_node, MPI_COMM_WORLD)
del a
fout_frc = open(frc_out,'w')
fout_res = open(res_out,'w')
fout_res.write("".join(res))
temp = zip(*frc)
datstrings = []
for i in temp:
for j in i:
datstrings.append(" %12f" % (j))
datstrings.append("\n")
fout_frc.write("".join(datstrings))
fout_frc.close()
del refi
del ringref
return rejects
def main():
import global_def
from optparse import OptionParser
from EMAN2 import EMUtil
import os
import sys
from time import time
progname = os.path.basename(sys.argv[0])
usage = progname + " proj_stack output_averages --MPI"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--img_per_group",type="int" , default=100 , help="number of images per group" )
parser.add_option("--radius", type="int" , default=-1 , help="radius for alignment" )
parser.add_option("--xr", type="string" , default="2 1", help="range for translation search in x direction, search is +/xr")
parser.add_option("--yr", type="string" , default="-1", help="range for translation search in y direction, search is +/yr (default = same as xr)")
parser.add_option("--ts", type="string" , default="1 0.5", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional")
parser.add_option("--iter", type="int" , default=30, help="number of iterations within alignment (default = 30)" )
parser.add_option("--num_ali", type="int" , default=5, help="number of alignments performed for stability (default = 5)" )
parser.add_option("--thld_err", type="float" , default=1.0, help="threshold of pixel error (default = 1.732)" )
parser.add_option("--grouping" , type="string" , default="GRP", help="do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size")
parser.add_option("--delta", type="float" , default=-1.0, help="angular step for reference projections (required for GEV method)")
parser.add_option("--fl", type="float" , default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--aa", type="float" , default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ")
parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
(options,args) = parser.parse_args()
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD, MPI_TAG_UB
from mpi import mpi_barrier, mpi_send, mpi_recv, mpi_bcast, MPI_INT, mpi_finalize, MPI_FLOAT
from applications import MPI_start_end, within_group_refinement, ali2d_ras
from pixel_error import multi_align_stability
from utilities import send_EMData, recv_EMData
from utilities import get_image, bcast_number_to_all, set_params2D, get_params2D
from utilities import group_proj_by_phitheta, model_circle, get_input_from_string
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 2:
stack = args[0]
outdir = args[1]
else:
ERROR("incomplete list of arguments", "sxproj_stability", 1, myid=myid)
exit()
if not options.MPI:
ERROR("Non-MPI not supported!", "sxproj_stability", myid=myid)
exit()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
#if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxproj_stability", 1, myid)
#mpi_barrier(MPI_COMM_WORLD)
img_per_grp = options.img_per_group
radius = options.radius
ite = options.iter
num_ali = options.num_ali
thld_err = options.thld_err
xrng = get_input_from_string(options.xr)
if options.yr == "-1": yrng = xrng
else : yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
if radius == -1: radius = nx/2-2
mask = model_circle(radius, nx, nx)
st = time()
if options.grouping == "GRP":
if myid == main_node:
print " A ",myid," ",time()-st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
# Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta,
# So I will briefly explain it here
# proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers
# except for the last one. Depending on the number of particles left, they will either form a
# group or append themselves to the last group
# angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi,
# theta) is the projection angle of the center of the group, delta is the range of this group
# mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates
# whether it should take mirror position.
# In this program angle_list and mirror list are not of interest.
proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(proj_params, img_per_grp=img_per_grp)
del proj_params
print " B number of groups ",myid," ",len(proj_list_all),time()-st
mpi_barrier(MPI_COMM_WORLD)
# Number of groups, actually there could be one or two more groups, since the size of the remaining group varies
# we will simply assign them to main node.
n_grp = nima/img_per_grp-1
# Divide proj_list_all equally to all nodes, and becomes proj_list
proj_list = []
for i in xrange(n_grp):
proc_to_stay = i%number_of_proc
if proc_to_stay == main_node:
if myid == main_node: proj_list.append(proj_list_all[i])
elif myid == main_node:
mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, MPI_TAG_UB, MPI_COMM_WORLD)
elif myid == proc_to_stay:
img_per_grp = mpi_recv(1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
img_per_grp = int(img_per_grp[0])
temp = mpi_recv(img_per_grp, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
proj_list.append(map(int, temp))
del temp
mpi_barrier(MPI_COMM_WORLD)
print " C ",myid," ",time()-st
if myid == main_node:
# Assign the remaining groups to main_node
for i in xrange(n_grp, len(proj_list_all)):
proj_list.append(proj_list_all[i])
del proj_list_all, angle_list, mirror_list
# Compute stability per projection projection direction, equal number assigned, thus overlaps
elif options.grouping == "GEV":
if options.delta == -1.0: ERROR("Angular step for reference projections is required for GEV method","sxproj_stability",1)
from utilities import even_angles, nearestk_to_refdir, getvec
refproj = even_angles(options.delta)
img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid)
# Now each processor keeps its own share of reference projections
refprojdir = refproj[img_begin: img_end]
del refproj
ref_ang = [0.0]*(len(refprojdir)*2)
for i in xrange(len(refprojdir)):
ref_ang[i*2] = refprojdir[0][0]
ref_ang[i*2+1] = refprojdir[0][1]+i*0.1
print " A ",myid," ",time()-st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
# the solution below is very slow, do not use it unless there is a problem with the i/O
"""
for i in xrange(number_of_proc):
if myid == i:
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
mpi_barrier(MPI_COMM_WORLD)
"""
print " B ",myid," ",time()-st
proj_ang = [0.0]*(nima*2)
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
proj_ang[i*2] = dp["phi"]
proj_ang[i*2+1] = dp["theta"]
print " C ",myid," ",time()-st
asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp)
del proj_ang, ref_ang
proj_list = []
for i in xrange(len(refprojdir)):
proj_list.append(asi[i*img_per_grp:(i+1)*img_per_grp])
del asi
print " D ",myid," ",time()-st
#from sys import exit
#exit()
# Compute stability per projection
elif options.grouping == "PPR":
print " A ",myid," ",time()-st
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
print " B ",myid," ",time()-st
proj_params = []
for i in xrange(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp["psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
print " C ",myid," ",time()-st
from utilities import nearest_proj
proj_list, mirror_list = nearest_proj(proj_params, img_per_grp, range(img_begin, img_begin+1))#range(img_begin, img_end))
refprojdir = proj_params[img_begin: img_end]
del proj_params, mirror_list
print " D ",myid," ",time()-st
else: ERROR("Incorrect projection grouping option","sxproj_stability",1)
"""
from utilities import write_text_file
for i in xrange(len(proj_list)):
write_text_file(proj_list[i],"projlist%06d_%04d"%(i,myid))
"""
###########################################################################################################
# Begin stability test
from utilities import get_params_proj, read_text_file
#if myid == 0:
# from utilities import read_text_file
# proj_list[0] = map(int, read_text_file("lggrpp0.txt"))
from utilities import model_blank
aveList = [model_blank(nx,ny)]*len(proj_list)
if options.grouping == "GRP": refprojdir = [[0.0,0.0,-1.0]]*len(proj_list)
for i in xrange(len(proj_list)):
print " E ",myid," ",time()-st
class_data = EMData.read_images(stack, proj_list[i])
#print " R ",myid," ",time()-st
if options.CTF :
from filter import filt_ctf
for im in xrange(len(class_data)): # MEM LEAK!!
atemp = class_data[im].copy()
btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1)
class_data[im] = btemp
#class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
try:
t = im.get_attr("xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0,-d["tx"],-d["ty"],0,1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
#print " F ",myid," ",time()-st
# Here, we perform realignment num_ali times
all_ali_params = []
for j in xrange(num_ali):
if( xrng[0] == 0.0 and yrng[0] == 0.0 ):
avet = ali2d_ras(class_data, randomize = True, ir = 1, ou = radius, rs = 1, step = 1.0, dst = 90.0, maxit = ite, check_mirror = True, FH=options.fl, FF=options.aa)
else:
avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa)
ali_params = []
for im in xrange(len(class_data)):
alpha, sx, sy, mirror, scale = get_params2D(class_data[im])
ali_params.extend( [alpha, sx, sy, mirror] )
all_ali_params.append(ali_params)
#aveList[i] = avet
#print " G ",myid," ",time()-st
del ali_params
# We determine the stability of this group here.
# stable_set contains all particles deemed stable, it is a list of list
# each list has two elements, the first is the pixel error, the second is the image number
# stable_set is sorted based on pixel error
#from utilities import write_text_file
#write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid)
stable_set, mir_stab_rate, average_pix_err = multi_align_stability(all_ali_params, 0.0, 10000.0, thld_err, False, 2*radius+1)
#print " H ",myid," ",time()-st
if(len(stable_set) > 5):
stable_set_id = []
members = []
pix_err = []
# First put the stable members into attr 'members' and 'pix_err'
for s in stable_set:
# s[1] - number in this subset
stable_set_id.append(s[1])
# the original image number
members.append(proj_list[i][s[1]])
pix_err.append(s[0])
# Then put the unstable members into attr 'members' and 'pix_err'
from fundamentals import rot_shift2D
avet.to_zero()
if options.grouping == "GRP":
aphi = 0.0
atht = 0.0
vphi = 0.0
vtht = 0.0
l = -1
for j in xrange(len(proj_list[i])):
# Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses
if j in stable_set_id:
l += 1
avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3] )
if options.grouping == "GRP":
phi, theta, psi, sxs, sys = get_params_proj(class_data[j])
if( theta > 90.0):
phi = (phi+540.0)%360.0
theta = 180.0 - theta
aphi += phi
atht += theta
vphi += phi*phi
vtht += theta*theta
else:
members.append(proj_list[i][j])
pix_err.append(99999.99)
aveList[i] = avet.copy()
if l>1 :
l += 1
aveList[i] /= l
if options.grouping == "GRP":
aphi /= l
atht /= l
vphi = (vphi - l*aphi*aphi)/l
vtht = (vtht - l*atht*atht)/l
from math import sqrt
refprojdir[i] = [aphi, atht, (sqrt(max(vphi,0.0))+sqrt(max(vtht,0.0)))/2.0]
# Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION
aveList[i].set_attr('members', members)
aveList[i].set_attr('refprojdir',refprojdir[i])
aveList[i].set_attr('pixerr', pix_err)
else:
print " empty group ",i, refprojdir[i]
aveList[i].set_attr('members',[-1])
aveList[i].set_attr('refprojdir',refprojdir[i])
aveList[i].set_attr('pixerr', [99999.])
del class_data
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(aveList)):
aveList[im].write_image(args[1], km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('pixerr', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
ave.write_image(args[1], km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node,im+myid+70000)
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
members = aveList[im].get_attr('pixerr')
mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
global_def.BATCH = False
mpi_barrier(MPI_COMM_WORLD)
from mpi import mpi_finalize
mpi_finalize()
def compare(compare_ref_free, outfile_repro, ref_free_output, yrng, xrng,
rstep, nx, apix, ref_free_cutoff, nproc, myid, main_node):
from alignment import Numrinit, ringwe, Applyws
from random import seed, randint
from utilities import get_params2D, set_params2D, model_circle, inverse_transform2, combine_params2
from fundamentals import rot_shift2D
from mpi import MPI_COMM_WORLD, mpi_barrier, mpi_bcast, MPI_INT
from statistics import fsc_mask
from filter import fit_tanh
from numpy import array
fout = "%s.hdf" % ref_free_output
frc_out = "%s_frc" % ref_free_output
res_out = "%s_res" % ref_free_output
nima = EMUtil.get_image_count(compare_ref_free)
image_start, image_end = MPI_start_end(nima, nproc, myid)
ima = EMData()
ima.read_image(compare_ref_free, image_start)
last_ring = nx / 2 - 2
first_ring = 1
mask = model_circle(last_ring, nx, nx)
refi = []
numref = EMUtil.get_image_count(outfile_repro)
cnx = nx / 2 + 1
cny = cnx
mode = "F"
numr = Numrinit(first_ring, last_ring, rstep, mode)
wr = ringwe(numr, mode)
ima.to_zero()
for j in xrange(numref):
temp = EMData()
temp.read_image(outfile_repro, j)
# even, odd, numer of even, number of images. After frc, totav
refi.append(temp)
# for each node read its share of data
data = EMData.read_images(compare_ref_free, range(image_start, image_end))
for im in xrange(image_start, image_end):
data[im - image_start].set_attr('ID', im)
set_params2D(data[im - image_start], [0, 0, 0, 0, 1])
ringref = []
for j in xrange(numref):
refi[j].process_inplace("normalize.mask", {
"mask": mask,
"no_sigma": 1
}) # normalize reference images to N(0,1)
cimage = Util.Polar2Dm(refi[j], cnx, cny, numr, mode)
Util.Frngs(cimage, numr)
Applyws(cimage, numr, wr)
ringref.append(cimage)
if myid == main_node: seed(1000)
data_shift = []
frc = []
res = []
for im in xrange(image_start, image_end):
alpha, sx, sy, mirror, scale = get_params2D(data[im - image_start])
alphai, sxi, syi, scalei = inverse_transform2(alpha, sx, sy, 1.0)
# normalize
data[im - image_start].process_inplace("normalize.mask", {
"mask": mask,
"no_sigma": 1
}) # subtract average under the mask
# align current image to the reference
[angt, sxst, syst, mirrort, xiref,
peakt] = Util.multiref_polar_ali_2d(data[im - image_start], ringref,
xrng, yrng, 1, mode, numr,
cnx + sxi, cny + syi)
iref = int(xiref)
[alphan, sxn, syn, mn] = combine_params2(0.0, -sxi, -syi, 0, angt,
sxst, syst, (int)(mirrort))
set_params2D(data[im - image_start],
[alphan, sxn, syn, int(mn), scale])
temp = rot_shift2D(data[im - image_start], alphan, sxn, syn, mn)
temp.set_attr('assign', iref)
tfrc = fsc_mask(temp, refi[iref], mask=mask)
temp.set_attr('frc', tfrc[1])
res = fit_tanh(tfrc)
temp.set_attr('res', res)
data_shift.append(temp)
for node in xrange(nproc):
if myid == node:
for image in data_shift:
image.write_image(fout, -1)
refindex = image.get_attr('assign')
refi[refindex].write_image(fout, -1)
mpi_barrier(MPI_COMM_WORLD)
rejects = []
if myid == main_node:
a = EMData()
index = 0
frc = []
res = []
temp = []
classes = []
for im in xrange(nima):
a.read_image(fout, index)
frc.append(a.get_attr("frc"))
if ref_free_cutoff != -1:
classes.append(a.get_attr("class_ptcl_idxs"))
tmp = a.get_attr("res")
temp.append(tmp[0])
res.append("%12f" % (apix / tmp[0]))
res.append("\n")
index = index + 2
res_num = array(temp)
mean_score = res_num.mean(axis=0)
std_score = res_num.std(axis=0)
std = std_score / 2
if ref_free_cutoff != -1:
cutoff = mean_score - std * ref_free_cutoff
reject = res_num < cutoff
index = 0
for i in reject:
if i: rejects.extend(classes[index])
index = index + 1
rejects.sort()
length = mpi_bcast(len(rejects), 1, MPI_INT, main_node,
MPI_COMM_WORLD)
rejects = mpi_bcast(rejects, length, MPI_INT, main_node,
MPI_COMM_WORLD)
del a
fout_frc = open(frc_out, 'w')
fout_res = open(res_out, 'w')
fout_res.write("".join(res))
temp = zip(*frc)
datstrings = []
for i in temp:
for j in i:
datstrings.append(" %12f" % (j))
datstrings.append("\n")
fout_frc.write("".join(datstrings))
fout_frc.close()
del refi
del ringref
return rejects
def main():
from utilities import get_input_from_string
progname = os.path.basename(sys.argv[0])
usage = (
progname
+ " stack output_average --radius=particle_radius --xr=xr --yr=yr --ts=ts --thld_err=thld_err --num_ali=num_ali --fl=fl --aa=aa --CTF --verbose --stables"
)
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--radius", type="int", default=-1, help=" particle radius for alignment")
parser.add_option(
"--xr",
type="string",
default="2 1",
help="range for translation search in x direction, search is +/xr (default 2,1)",
)
parser.add_option(
"--yr",
type="string",
default="-1",
help="range for translation search in y direction, search is +/yr (default = same as xr)",
)
parser.add_option(
"--ts",
type="string",
default="1 0.5",
help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional (default: 1,0.5)",
)
parser.add_option("--thld_err", type="float", default=0.75, help="threshld of pixel error (default = 0.75)")
parser.add_option(
"--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)"
)
parser.add_option("--maxit", type="int", default=30, help="number of iterations for each xr (default = 30)")
parser.add_option(
"--fl",
type="float",
default=0.3,
help="cut-off frequency of hyperbolic tangent low-pass Fourier filter (default = 0.3)",
)
parser.add_option(
"--aa", type="float", default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter (default = 0.2)"
)
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction during the alignment ")
parser.add_option(
"--verbose", action="store_true", default=False, help="print individual pixel error (default = False)"
)
parser.add_option(
"--stables",
action="store_true",
default=False,
help="output the stable particles number in file (default = False)",
)
parser.add_option(
"--method", type="string", default=" ", help="SHC (standard method is default when flag is ommitted)"
)
(options, args) = parser.parse_args()
if len(args) != 1 and len(args) != 2:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
else:
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from applications import within_group_refinement, ali2d_ras
from pixel_error import multi_align_stability
from utilities import write_text_file, write_text_row
global_def.BATCH = True
xrng = get_input_from_string(options.xr)
if options.yr == "-1":
yrng = xrng
else:
yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
class_data = EMData.read_images(args[0])
nx = class_data[0].get_xsize()
ou = options.radius
num_ali = options.num_ali
if ou == -1:
ou = nx / 2 - 2
from utilities import model_circle, get_params2D, set_params2D
mask = model_circle(ou, nx, nx)
if options.CTF:
from filter import filt_ctf
for im in xrange(len(class_data)):
# Flip phases
class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
im.set_attr("previousmax", -1.0e10)
try:
t = im.get_attr("xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
all_ali_params = []
for ii in xrange(num_ali):
ali_params = []
if options.verbose:
ALPHA = []
SX = []
SY = []
MIRROR = []
if xrng[0] == 0.0 and yrng[0] == 0.0:
avet = ali2d_ras(
class_data,
randomize=True,
ir=1,
ou=ou,
rs=1,
step=1.0,
dst=90.0,
maxit=options.maxit,
check_mirror=True,
FH=options.fl,
FF=options.aa,
)
else:
avet = within_group_refinement(
class_data,
mask,
True,
1,
ou,
1,
xrng,
yrng,
step,
90.0,
maxit=options.maxit,
FH=options.fl,
FF=options.aa,
method=options.method,
)
from utilities import info
# print " avet ",info(avet)
for im in class_data:
alpha, sx, sy, mirror, scale = get_params2D(im)
ali_params.extend([alpha, sx, sy, mirror])
if options.verbose:
ALPHA.append(alpha)
SX.append(sx)
SY.append(sy)
MIRROR.append(mirror)
all_ali_params.append(ali_params)
if options.verbose:
write_text_file([ALPHA, SX, SY, MIRROR], "ali_params_run_%d" % ii)
"""
avet = class_data[0]
from utilities import read_text_file
all_ali_params = []
for ii in xrange(5):
temp = read_text_file( "ali_params_run_%d"%ii,-1)
uuu = []
for k in xrange(len(temp[0])):
uuu.extend([temp[0][k],temp[1][k],temp[2][k],temp[3][k]])
all_ali_params.append(uuu)
"""
stable_set, mir_stab_rate, pix_err = multi_align_stability(
all_ali_params, 0.0, 10000.0, options.thld_err, options.verbose, 2 * ou + 1
)
print "%4s %20s %20s %20s %30s %6.2f" % (
"",
"Size of set",
"Size of stable set",
"Mirror stab rate",
"Pixel error prior to pruning the set above threshold of",
options.thld_err,
)
print "Average stat: %10d %20d %20.2f %15.2f" % (len(class_data), len(stable_set), mir_stab_rate, pix_err)
if len(stable_set) > 0:
if options.stables:
stab_mem = [[0, 0.0, 0] for j in xrange(len(stable_set))]
for j in xrange(len(stable_set)):
stab_mem[j] = [int(stable_set[j][1]), stable_set[j][0], j]
write_text_row(stab_mem, "stable_particles.txt")
stable_set_id = []
particle_pixerr = []
for s in stable_set:
stable_set_id.append(s[1])
particle_pixerr.append(s[0])
from fundamentals import rot_shift2D
avet.to_zero()
l = -1
print "average parameters: angle, x-shift, y-shift, mirror"
for j in stable_set_id:
l += 1
print " %4d %4d %12.2f %12.2f %12.2f %1d" % (
l,
j,
stable_set[l][2][0],
stable_set[l][2][1],
stable_set[l][2][2],
int(stable_set[l][2][3]),
)
avet += rot_shift2D(
class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3]
)
avet /= l + 1
avet.set_attr("members", stable_set_id)
avet.set_attr("pix_err", pix_err)
avet.set_attr("pixerr", particle_pixerr)
avet.write_image(args[1])
global_def.BATCH = False
