def errors_per_image(params, avgtrans, thresherr=1.0, radius = 1.0): #print "errors per image" # We have average parameters and transformed parameters # Find errors per image ll = len(params) nn = len(params[0]) perr = [None]*nn for k in xrange(nn): # nas = [0.0,0.0,0.0] r1,r2,r3 = getfvec(avgtrans[k][0],avgtrans[k][1]) sacos = 0.0 sder = 0.0 ser3 = 0.0 for i in xrange(ll): #print i,fifi[i] d = max_3D_pixel_error(params[i][k], avgtrans[k], r=radius) #if(d>10.): print " LARGE ERROR",k,i,d,fifi[i], avgtrans[k] ser3 += d n1,n2,n3 = getfvec(params[i][k][0],params[i][k][1]) sacos += acos( min(1.0, n1*r1+n2*r2+n3*r3) ) sder += pixel_error_2D(params[i][k][2:], avgtrans[k][2:], r = 1.0) # average deviation in radians sacos /= ll sare = tan(sacos) sder /= ll ser3 /= ll #print k, ser3, sare, sder perr[k] = [ser3, sacos, sare, sder] #write_text_row(perr, 'per.txt') #perr = [perr[k][1] <= thresherr for k in xrange(nn)] return perr
def errors_per_image(params, avgtrans, thresherr=1.0, radius=1.0):
#print "errors per image"
# We have average parameters and transformed parameters
# Find errors per image
ll = len(params)
nn = len(params[0])
perr = [None] * nn
for k in xrange(nn):
#
nas = [0.0, 0.0, 0.0]
r1, r2, r3 = getfvec(avgtrans[k][0], avgtrans[k][1])
sacos = 0.0
sder = 0.0
ser3 = 0.0
for i in xrange(ll):
#print i,fifi[i]
d = max_3D_pixel_error(params[i][k], avgtrans[k], r=radius)
#if(d>10.): print " LARGE ERROR",k,i,d,fifi[i], avgtrans[k]
ser3 += d
n1, n2, n3 = getfvec(params[i][k][0], params[i][k][1])
sacos += lacos(n1 * r1 + n2 * r2 + n3 * r3)
sder += pixel_error_2D(params[i][k][2:], avgtrans[k][2:], r=1.0)
# average deviation in radians
sacos /= ll
sare = tan(sacos)
sder /= ll
ser3 /= ll
#print k, ser3, sare, sder
perr[k] = [ser3, sacos, sare, sder]
#write_text_row(perr, 'per.txt')
#perr = [perr[k][1] <= thresherr for k in xrange(nn)]
return perr
def comparetwoalis(params1, params2, thresherr=1.0, radius = 1.0): # Find errors per image nn = len(params1) perr = 0 for k in xrange(nn): if(max_3D_pixel_error(params1[k], params2[k], r=radius) < thresherr): perr += 1 return perr/float(nn)*100.0
def proj_ali_incore_local(data, refrings, numr, xrng, yrng, step, an, finfo=None):
from utilities import compose_transform2
#from utilities import set_params_proj, get_params_proj
from math import cos, sin, pi
ID = data.get_attr("ID")
mode = "F"
nx = data.get_xsize()
ny = data.get_ysize()
# center is in SPIDER convention
cnx = nx//2 + 1
cny = ny//2 + 1
ant = cos(an*pi/180.0)
#phi, theta, psi, sxo, syo = get_params_proj(data)
t1 = data.get_attr("xform.projection")
dp = t1.get_params("spider")
# get translations from data
tx = dp["tx"]
ty = dp["ty"]
if finfo:
finfo.write("Image id: %6d\n"%(ID))
finfo.write("Old parameters: %9.4f %9.4f %9.4f %9.4f %9.4f\n"%(dp["phi"], dp["theta"], dp["psi"], -tx, -ty))
finfo.flush()
[ang, sxs, sys, mirror, iref, peak] = Util.multiref_polar_ali_2d_local(data, refrings, xrng, yrng, step, ant, mode, numr, cnx+tx, cny+ty)
iref=int(iref)
data.set_attr("assign",iref)
if iref > -1:
# The ormqip returns parameters such that the transformation is applied first, the mirror operation second.
# What that means is that one has to change the the Eulerian angles so they point into mirrored direction: phi+180, 180-theta, 180-psi
angb, sxb, syb, ct = compose_transform2(0.0, sxs, sys, 1, -ang, 0.0, 0.0, 1)
if mirror:
phi = (refrings[iref].get_attr("phi")+540.0)%360.0
theta = 180.0-refrings[iref].get_attr("theta")
psi = (540.0-refrings[iref].get_attr("psi")+angb)%360.0
s2x = sxb - tx
s2y = syb - ty
else:
phi = refrings[iref].get_attr("phi")
theta = refrings[iref].get_attr("theta")
psi = (refrings[iref].get_attr("psi")+angb+360.0)%360.0
s2x = sxb - tx
s2y = syb - ty
t2 = Transform({"type":"spider","phi":phi,"theta":theta,"psi":psi})
t2.set_trans(Vec2f(-s2x, -s2y))
from pixel_error import max_3D_pixel_error
pixel_error = max_3D_pixel_error(t1, t2, numr[-3])
if finfo:
finfo.write( "New parameters: %9.4f %9.4f %9.4f %9.4f %9.4f %10.5f %11.3e\n\n" %(phi, theta, psi, s2x, s2y, peak, pixel_error))
finfo.flush()
return t2, peak, pixel_error
else:
return -1.0e23, 0.0
def errors_per_image_original(nn,
qt,
asi,
avgtrans,
thresherr=1.0,
radius=1.0):
#print "errors per image"
# We have average parameters and transformed parameters
# Find errors per image
# Find outliers, remove them, repeat calculations
perr = [None] * nn
for k in xrange(nn):
lin = k // (nn // 4)
#
fifi = []
for j in xrange(6):
if (asi[lin][j] != -10):
#print " fff ",k,lin,j,asi[lin][j],qt[j][k]
fifi.append(qt[j][k])
assert (len(fifi) == 3)
nas = [0.0, 0.0, 0.0]
r1, r2, r3 = getfvec(avgtrans[k][0], avgtrans[k][1])
sacos = 0.0
sder = 0.0
ser3 = 0.0
for i in xrange(3):
#print i,fifi[i]
d = max_3D_pixel_error(fifi[i], avgtrans[k], r=radius)
#if(d>10.): print " LARGE ERROR",k,i,d,fifi[i], avgtrans[k]
ser3 += d
n1, n2, n3 = getfvec(fifi[i][0], fifi[i][1])
sacos += lacos(n1 * r1 + n2 * r2 + n3 * r3)
sder += pixel_error_2D(fifi[i][2:], avgtrans[k][2:], r=1.0)
# average deviation in radians
sacos /= 3.0
sare = tan(sacos)
sder /= 3.0
ser3 /= 3
#print k, ser3, sare, sder
perr[k] = [k, ser3, sacos, sare, sder]
#write_text_row(perr, 'per.txt')
perr = [perr[k][1] <= thresherr for k in xrange(nn)]
return perr
def errors_per_image_original(nn, qt, asi, avgtrans, thresherr=1.0, radius = 1.0): #print "errors per image" # We have average parameters and transformed parameters # Find errors per image # Find outliers, remove them, repeat calculations perr = [None]*nn for k in xrange(nn): lin = k//(nn//4) # fifi = [] for j in xrange(6): if(asi[lin][j] != -10): #print " fff ",k,lin,j,asi[lin][j],qt[j][k] fifi.append(qt[j][k]) assert(len(fifi) == 3) nas = [0.0,0.0,0.0] r1,r2,r3 = getfvec(avgtrans[k][0],avgtrans[k][1]) sacos = 0.0 sder = 0.0 ser3 = 0.0 for i in xrange(3): #print i,fifi[i] d = max_3D_pixel_error(fifi[i], avgtrans[k], r=radius) #if(d>10.): print " LARGE ERROR",k,i,d,fifi[i], avgtrans[k] ser3 += d n1,n2,n3 = getfvec(fifi[i][0],fifi[i][1]) sacos += acos(min(1.0,n1*r1+n2*r2+n3*r3)) sder += pixel_error_2D(fifi[i][2:], avgtrans[k][2:], r = 1.0) # average deviation in radians sacos /= 3.0 sare = tan(sacos) sder /= 3.0 ser3 /= 3 #print k, ser3, sare, sder perr[k] = [k, ser3, sacos, sare, sder] #write_text_row(perr, 'per.txt') perr = [perr[k][1] <= thresherr for k in xrange(nn)] return perr
def main():
from utilities import write_text_row, drop_image, model_gauss_noise, get_im, set_params_proj, wrap_mpi_bcast, model_circle
import user_functions
from applications import MPI_start_end
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from multi_shc import multi_shc, do_volume
from logger import Logger, BaseLogger_Files
import sys
import os
import time
import socket
progname = os.path.basename(sys.argv[0])
usage = progname + " stack [output_directory] initial_volume --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --center=center_type --fl --aa --ref_a=S --sym=c1"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--ir", type= "int", default= 1, help="inner radius for rotational correlation > 0 (set to 1)")
parser.add_option("--ou", type= "int", default= -1, help="outer radius for rotational correlation < int(nx/2)-1 (set to the radius of the particle)")
parser.add_option("--rs", type= "int", default= 1, help="step between rings in rotational correlation >0 (set to 1)" )
parser.add_option("--xr", type="string", default= "-1", help="range for translation search in x direction, search is +/xr (default 0)")
parser.add_option("--yr", type="string", default= "-1", help="range for translation search in y direction, search is +/yr (default = same as xr)")
parser.add_option("--ts", type="string", default= "1", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional")
parser.add_option("--delta", type="string", default= "-1", help="angular step of reference projections during initialization step (default automatically selected based on radius of the structure.)")
#parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches (phi and theta)")
parser.add_option("--center", type="float", default= -1, help="-1: average shift method; 0: no centering; 1: center of gravity (default=-1)")
parser.add_option("--maxit", type="int", default= 400, help="maximum number of iterations performed for the GA part (set to 400) ")
parser.add_option("--outlier_percentile",type="float", default= 95, help="percentile above which outliers are removed every iteration")
parser.add_option("--iteration_start",type="int", default= 0, help="starting iteration for rviper, 0 means go to the most recent one (default).")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF (Default no CTF correction)")
parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data (default 1.0)")
parser.add_option("--ref_a", type="string", default= "S", help="method for generating the quasi-uniformly distributed projection directions (default S)")
parser.add_option("--sym", type="string", default= "c1", help="symmetry of the refined structure")
parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default=2)")
parser.add_option("--startangles", action="store_true", default=False, help="Use orientation parameters in the input file header to jumpstart the procedure")
#options introduced for the do_volume function
parser.add_option("--fl", type="float", default=0.12, help="cut-off frequency of hyperbolic tangent low-pass Fourier filte (default 0.12)")
parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass Fourier filter (default 0.1)")
parser.add_option("--pwreference", type="string", default="", help="text file with a reference power spectrum (default no power spectrum adjustment)")
parser.add_option("--mask3D", type="string", default=None, help="3D mask file (default a sphere WHAT RADIUS??)")
(options, args) = parser.parse_args(sys.argv[1:])
#print( " args ",args)
if( len(args) == 3):
volinit = args[2]
masterdir = args[1]
elif(len(args) == 2):
volinit = args[1]
masterdir = ""
else:
print( "usage: " + usage)
print( "Please run '" + progname + " -h' for detailed options")
return 1
orgstack = args[0]
#print( orgstack,masterdir,volinit )
# INPUT PARAMETERS
radi = options.ou
global_def.BATCH = True
ali3d_options.ir = options.ir
ali3d_options.rs = options.rs
ali3d_options.ou = options.ou
ali3d_options.xr = options.xr
ali3d_options.yr = options.yr
ali3d_options.ts = options.ts
ali3d_options.an = "-1"
ali3d_options.sym = options.sym
ali3d_options.delta = options.delta
ali3d_options.npad = options.npad
ali3d_options.center = options.center
ali3d_options.CTF = options.CTF
ali3d_options.ref_a = options.ref_a
ali3d_options.snr = options.snr
ali3d_options.mask3D = options.mask3D
ali3d_options.pwreference = options.pwreference
ali3d_options.fl = 0.4
ali3d_options.aa = 0.1
if( ali3d_options.xr == "-1" ): ali3d_options.xr = "2"
"""
print( options)
print( 'ali3d_options', ali3d_options.ir ,\
ali3d_options.rs ,\
ali3d_options.ou ,\
ali3d_options.xr ,\
ali3d_options.yr ,\
ali3d_options.ts ,\
ali3d_options.an ,\
ali3d_options.sym ,\
ali3d_options.delta ,\
ali3d_options.npad ,\
ali3d_options.center ,\
ali3d_options.CTF ,\
ali3d_options.ref_a ,\
ali3d_options.snr ,\
ali3d_options.mask3D ,\
ali3d_options.fl ,\
ali3d_options.aa \
)
#exit()
"""
mpi_init(0, [])
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
#mpi_finalize()
#exit()
nxinit = -1 #int(280*0.3*2)
nsoft = 0
mempernode = 4.0e9
# PARAMETERS OF THE PROCEDURE
# threshold error
thresherr = 0
fq = 0.11 # low-freq limit to which fuse ref volumes. Should it be estimated somehow?
# Get the pixel size, if none set to 1.0, and the original image size
if(myid == main_node):
a = get_im(orgstack)
nnxo = a.get_xsize()
if ali3d_options.CTF:
i = a.get_attr('ctf')
pixel_size = i.apix
else:
pixel_size = 1.0
del a
else:
nnxo = 0
pixel_size = 1.0
pixel_size = bcast_number_to_all(pixel_size, source_node = main_node)
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
if(radi < 1): radi = nnxo//2-2
elif((2*radi+2)>nnxo): ERROR("HERE","particle radius set too large!",1)
ali3d_options.ou = radi
if(nxinit < 0): nxinit = min(32, nnxo)
nxshrink = nxinit
minshrink = 32.0/float(nnxo)
shrink = max(float(nxshrink)/float(nnxo),minshrink)
# MASTER DIRECTORY
if(myid == main_node):
print( " masterdir ",masterdir)
if( masterdir == ""):
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "master"+timestring
li = len(masterdir)
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
keepchecking = 0
else:
li = 0
keepchecking = 1
else:
li = 0
keepchecking = 1
li = mpi_bcast(li,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
if( li > 0 ):
masterdir = mpi_bcast(masterdir,li,MPI_CHAR,main_node,MPI_COMM_WORLD)
masterdir = string.join(masterdir,"")
# create a vstack from input stack to the local stack in masterdir
# Stack name set to default
stack = "bdb:"+masterdir+"/rdata"
# Initialization of stacks
if(myid == main_node):
if keepchecking:
if(os.path.exists(os.path.join(masterdir,"EMAN2DB/rdata.bdb"))): doit = False
else: doit = True
else: doit = True
if doit:
if(orgstack[:4] == "bdb:"): cmd = "{} {} {}".format("e2bdb.py", orgstack,"--makevstack="+stack)
else: cmd = "{} {} {}".format("sxcpy.py", orgstack, stack)
cmdexecute(cmd)
cmd = "{} {}".format("sxheader.py --consecutive --params=originalid", stack)
cmdexecute(cmd)
keepchecking = False
total_stack = EMUtil.get_image_count(stack)
junk = get_im(stack)
nnxo = junk.get_xsize()
del junk
else:
total_stack = 0
nnxo = 0
total_stack = bcast_number_to_all(total_stack, source_node = main_node)
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
# INITIALIZATION
# Run exhaustive projection matching to get initial orientation parameters
# Estimate initial resolution
initdir = os.path.join(masterdir,"main000")
# make sure the initial volume is not set to zero outside of a mask, as if it is it will crach the program
if( myid == main_node and (not options.startangles)):
viv = get_im(volinit)
if(options.mask3D == None): mask33d = model_circle(radi,nnxo,nnxo,nnxo)
else: mask33d = (options.mask3D).copy()
st = Util.infomask(viv, mask33d, False)
if( st[0] == 0.0 ):
viv += (model_blank(nnxo,nnxo,nnxo,1.0) - mask33d)*model_gauss_noise(st[1]/1000.0,nnxo,nnxo,nnxo)
viv.write_image(volinit)
del mask33d, viv
doit, keepchecking = checkstep(initdir, keepchecking, myid, main_node)
if doit:
partids = os.path.join(masterdir, "ids.txt")
partstack = os.path.join(masterdir, "paramszero.txt")
xr = min(8,(nnxo - (2*radi+1))//2)
if(xr > 3): ts = "2"
else: ts = "1"
delta = int(options.delta)
if(delta <= 0.0):
delta = "%f"%round(degrees(atan(1.0/float(radi))), 2)
paramsdict = { "stack":stack,"delta":"2.0", "ts":ts, "xr":"%f"%xr, "an":"-1", "center":options.center, "maxit":1, \
"currentres":0.4, "aa":0.1, "radius":radi, "nsoft":0, "delpreviousmax":True, "shrink":1.0, "saturatecrit":1.0, \
"refvol":volinit, "mask3D":options.mask3D}
if(options.startangles):
if( myid == main_node ):
cmd = "mkdir "+initdir
cmdexecute(cmd)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"INITIALIZATION")
cmd = "{} {}".format("sxheader.py --params=xform.projection --export="+os.path.join(initdir,"params-chunk0.txt"), stack)
cmdexecute(cmd)
print(line,"Executed successfully: ","Imported initial parameters from the input stack")
else:
if( myid == main_node ):
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"INITIALIZATION")
write_text_file(range(total_stack), partids)
write_text_row([[0.0,0.0,0.0,0.0,0.0] for i in xrange(total_stack) ], partstack)
metamove(paramsdict, partids, partstack, initdir, 0, myid, main_node, nproc)
if(myid == main_node):
print(line,"Executed successfully: ","initialization ali3d_base_MPI %d"%nsoft)
# store params
partids = [None]*2
for procid in xrange(2): partids[procid] = os.path.join(initdir,"chunk%01d.txt"%procid)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(initdir,"params-chunk%01d.txt"%procid)
from random import shuffle
if(myid == main_node):
# split randomly
params = read_text_row(os.path.join(initdir,"params-chunk0.txt"))
assert(len(params) == total_stack)
ll = range(total_stack)
shuffle(ll)
l1 = ll[:total_stack//2]
l2 = ll[total_stack//2:]
del ll
l1.sort()
l2.sort()
write_text_file(l1,partids[0])
write_text_file(l2,partids[1])
write_text_row([params[i] for i in l1], partstack[0])
write_text_row([params[i] for i in l2], partstack[1])
del params, l1, l2
mpi_barrier(MPI_COMM_WORLD)
# Now parallel
vol = [None]*2
for procid in xrange(2):
projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc)
if ali3d_options.CTF: vol[procid] = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol[procid] = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if( myid == main_node):
vol[procid].write_image(os.path.join(initdir,"vol%01d.hdf"%procid) )
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Generated inivol #%01d "%procid)
if(myid == main_node):
currentres = get_resolution(vol, radi, nnxo, initdir)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Initial resolution %6.4f"%currentres)
write_text_file([currentres],os.path.join(initdir,"current_resolution.txt"))
else: currentres = 0.0
currentres = bcast_number_to_all(currentres, source_node = main_node)
else:
if(myid == main_node): currentres = read_text_file(os.path.join(initdir,"current_resolution.txt"))[0]
else: currentres = 0.0
currentres = bcast_number_to_all(currentres, source_node = main_node)
# set for the first iteration
nxshrink = min(max(32, int((currentres+paramsdict["aa"]/2.)*2*nnxo + 0.5)), nnxo)
shrink = float(nxshrink)/nnxo
tracker = {"previous-resolution":currentres, "movedup":False,"eliminated-outliers":False,\
"previous-nx":nxshrink, "previous-shrink":shrink, "extension":0, "bestsolution":0}
previousoutputdir = initdir
# MAIN ITERATION
mainiteration = 0
keepgoing = 1
while(keepgoing):
mainiteration += 1
# prepare output directory
mainoutputdir = os.path.join(masterdir,"main%03d"%mainiteration)
if(myid == main_node):
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"MAIN ITERATION #",mainiteration, shrink, nxshrink)
if keepchecking:
if(os.path.exists(mainoutputdir)):
doit = 0
print("Directory ",mainoutputdir," exists!")
else:
doit = 1
keepchecking = False
else:
doit = 1
if doit:
cmd = "{} {}".format("mkdir", mainoutputdir)
cmdexecute(cmd)
# prepare names of input file names, they are in main directory,
# log subdirectories contain outputs from specific refinements
partids = [None]*2
for procid in xrange(2): partids[procid] = os.path.join(previousoutputdir,"chunk%01d.txt"%procid)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(previousoutputdir,"params-chunk%01d.txt"%procid)
mpi_barrier(MPI_COMM_WORLD)
#mpi_finalize()
#exit()
#print("RACING A ",myid)
outvol = [os.path.join(previousoutputdir,"vol%01d.hdf"%procid) for procid in xrange(2)]
for procid in xrange(2):
doit, keepchecking = checkstep(outvol[procid], keepchecking, myid, main_node)
if doit:
from multi_shc import do_volume
projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc)
if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if( myid == main_node):
vol.write_image(outvol[procid])
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Generated inivol #%01d "%procid)
del vol
if(myid == main_node):
if keepchecking:
procid = 1
if(os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid)):
doit = 0
else:
doit = 1
keepchecking = False
else:
doit = 1
if doit:
vol = [get_im(outvol[procid]) for procid in xrange(2) ]
fq = 0.11 # which part to fuse
fuselowf(vol, fq)
for procid in xrange(2): vol[procid].write_image(os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid) )
del vol
else: doit = 0
mpi_barrier(MPI_COMM_WORLD)
doit = bcast_number_to_all(doit, source_node = main_node)
# Refine two groups at a current resolution
lastring = int(shrink*radi + 0.5)
if(lastring < 2):
print( line,"ERROR!! lastring too small ", radi, shrink, lastring)
break
# REFINEMENT
# Part "a" SHC
for procid in xrange(2):
coutdir = os.path.join(mainoutputdir,"loga%01d"%procid)
doit, keepchecking = checkstep(coutdir , keepchecking, myid, main_node)
paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":1500, \
"currentres":currentres, "aa":0.1, "radius":radi, "nsoft":1, "saturatecrit":0.75, "delpreviousmax":True, "shrink":shrink, \
"refvol":os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid),"mask3D":options.mask3D }
if doit:
metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir, "loga%01d"%procid, "params-chunk%01d.txt"%procid)
for procid in xrange(2):
outvol = os.path.join(mainoutputdir,"loga%01d"%procid,"shcvol%01d.hdf"%procid)
doit, keepchecking = checkstep(outvol, keepchecking, myid, main_node)
if doit:
from multi_shc import do_volume
projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc)
if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if( myid == main_node):
vol.write_image(outvol)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Generated shcvol #%01d "%procid)
del vol
if(myid == main_node):
if keepchecking:
procid = 1
if(os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid) ):
doit = 0
else:
doit = 1
keepchecking = False
else:
doit = 1
if doit:
vol = []
for procid in xrange(2): vol.append(get_im(os.path.join(mainoutputdir,"loga%01d"%procid,"shcvol%01d.hdf"%procid) ))
fq = 0.11 # which part to fuse
fuselowf(vol, fq)
for procid in xrange(2): vol[procid].write_image( os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid) )
del vol
else: doit = 0
mpi_barrier(MPI_COMM_WORLD)
doit = bcast_number_to_all(doit, source_node = main_node)
# Part "b" deterministic
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir,"loga%01d"%procid,"params-chunk%01d.txt"%procid)
for procid in xrange(2):
coutdir = os.path.join(mainoutputdir,"logb%01d"%procid)
doit, keepchecking = checkstep(coutdir, keepchecking, myid, main_node)
# Run exhaustive to finish up matching
paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":10, \
"currentres":currentres, "aa":0.1, "radius":radi, "nsoft":0, "saturatecrit":0.95, "delpreviousmax":True, "shrink":shrink, \
"refvol":os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid), "mask3D":options.mask3D }
if doit:
metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir,"logb%01d"%procid,"params-chunk%01d.txt"%procid)
# Compute current resolution, store result in main directory
doit, keepchecking = checkstep(os.path.join(mainoutputdir,"current_resolution.txt"), keepchecking, myid, main_node)
newres = 0.0
if doit:
newres = compute_resolution(stack, mainoutputdir, partids, partstack, radi, nnxo, ali3d_options.CTF, myid, main_node, nproc)
else:
if(myid == main_node): newres = read_text_file( os.path.join(mainoutputdir,"current_resolution.txt") )[0]
newres = bcast_number_to_all(newres, source_node = main_node)
# Here I have code to generate presentable results. IDs and params have to be merged and stored and an overall volume computed.
doit, keepchecking = checkstep(os.path.join(mainoutputdir,"volf.hdf"), keepchecking, myid, main_node)
if doit:
if( myid == main_node ):
pinids = map(int, read_text_file(partids[0]) ) + map(int, read_text_file(partids[1]) )
params = read_text_row(partstack[0]) + read_text_row(partstack[1])
assert(len(pinids) == len(params))
for i in xrange(len(pinids)):
pinids[i] = [ pinids[i], params[i] ]
del params
pinids.sort()
write_text_file([pinids[i][0] for i in xrange(len(pinids))], os.path.join(mainoutputdir,"indexes.txt"))
write_text_row( [pinids[i][1] for i in xrange(len(pinids))], os.path.join(mainoutputdir,"params.txt"))
mpi_barrier(MPI_COMM_WORLD)
ali3d_options.fl = newres
ali3d_options.ou = radi
projdata = getindexdata(stack, os.path.join(mainoutputdir,"indexes.txt"), os.path.join(mainoutputdir,"params.txt"), myid, nproc)
volf = do_volume(projdata, ali3d_options, mainiteration, mpi_comm = MPI_COMM_WORLD)
if(myid == main_node): volf.write_image(os.path.join(mainoutputdir,"volf.hdf"))
mpi_barrier(MPI_COMM_WORLD)
#print("RACING X ",myid)
if(newres == currentres):
for procid in xrange(2):
coutdir = os.path.join(mainoutputdir,"logc%01d"%procid)
doit, keepchecking = checkstep(coutdir, keepchecking, myid, main_node)
if doit:
# Do cross-check of the results
paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":1, \
"currentres":newres, "aa":0.1, "radius":radi, "nsoft":0, "saturatecrit":0.95, "delpreviousmax":True, "shrink":shrink, \
"refvol":os.path.join(mainoutputdir,"vol%01d.hdf"%(1-procid)), "mask3D":options.mask3D }
# The cross-check uses parameters from step "b" to make sure shifts are correct.
# As the check is exhaustive, angles are ignored
metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc)
# identify bad apples
doit, keepchecking = checkstep(os.path.join(mainoutputdir,"badapples.txt"), keepchecking, myid, main_node)
if doit:
if(myid == main_node):
from utilities import get_symt
from pixel_error import max_3D_pixel_error
ts = get_symt(ali3d_options.sym)
badapples = []
deltaerror = 2.0
total_images_now = 0
for procid in xrange(2):
bad = []
ids = map(int,read_text_file( partids[procid] ))
total_images_now += len(ids)
oldp = read_text_row(partstack[procid])
newp = read_text_row(os.path.join(mainoutputdir,"logc%01d"%procid,"params-chunk%01d.txt"%procid))
for i in xrange(len(ids)):
t1 = Transform({"type":"spider","phi":oldp[i][0],"theta":oldp[i][1],"psi":oldp[i][2]})
t1.set_trans(Vec2f(-oldp[i][3]*shrink, -oldp[i][4]*shrink))
t2 = Transform({"type":"spider","phi":newp[i][0],"theta":newp[i][1],"psi":newp[i][2]})
t2.set_trans(Vec2f(-newp[i][3]*shrink, -newp[i][4]*shrink))
if(len(ts) > 1):
# only do it if it is not c1
pixel_error = +1.0e23
for kts in ts:
ut = t2*kts
# we do not care which position minimizes the error
pixel_error = min(max_3D_pixel_error(t1, ut, lastring), pixel_error)
else:
pixel_error = max_3D_pixel_error(t1, t2, lastring)
if(pixel_error > deltaerror):
bad.append(i)
if(len(bad)>0):
badapples += [ids[bad[i]] for i in xrange(len(bad))]
for i in xrange(len(bad)-1,-1,-1):
del oldp[bad[i]],ids[bad[i]]
if(len(ids) == 0):
ERROR("sxpetite","program diverged, all images have large angular errors, most likely the initial model is badly off",1)
else:
# This generate new parameters, hopefully to be used as starting ones in the new iteration
write_text_file(ids,os.path.join(mainoutputdir,"chunk%01d.txt"%procid))
write_text_row(oldp,os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))
if(len(badapples)>0):
badapples.sort()
write_text_file(badapples,os.path.join(mainoutputdir,"badapples.txt"))
eli = 100*float(len(badapples))/float(total_images_now)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"Elimination of outliers: %5.1f percent"%eli )
else: eli = 0.0
del badapples, oldp,ids,bad,newp,ts
else: eli =0.0
eli = bcast_number_to_all(eli, source_node = main_node)
# This part under MPI
if(eli > 0.0):
# Compute current resolution
depres = compute_resolution(stack, mainoutputdir, \
[os.path.join(mainoutputdir,"chunk%01d.txt"%procid) for procid in xrange(2)], \
[os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid) for procid in xrange(2)], \
radi, nnxo, ali3d_options.CTF, myid, main_node, nproc)
depres = bcast_number_to_all(depres, source_node = main_node)
if(depres < newres):
# elimination of outliers decreased resolution, ignore the effort
eliminated_outliers = False
else:
eliminated_outliers = True
newres = depres
"""
# It does not seem to be needed, as data is there, we just point to the directory
for procid in xrange(2):
# set pointers to current parameters in main, which are for the reduced set stored above
partids[procid] = os.path.join(mainoutputdir,"chunk%01d.txt"%procid
partstack[procid] = os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid)
"""
else:
eliminated_outliers = False
else:
eliminated_outliers = False
if(myid == main_node and not eliminated_outliers):
for procid in xrange(2):
# This is standard path, copy parameters to be used to the main
cmd = "{} {} {}".format("cp -p ", partids[procid] , os.path.join(mainoutputdir,"chunk%01d.txt"%procid))
cmdexecute(cmd)
cmd = "{} {} {}".format("cp -p ", partstack[procid], os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))
cmdexecute(cmd)
keepgoing = 0
if( newres > currentres or (eliminated_outliers and not tracker["eliminated-outliers"])):
if(myid == main_node): print(" Resolution improved, full steam ahead!")
if( newres > currentres ): tracker["movedup"] = True
else: tracker["movedup"] = False
shrink = max(min(2*newres + paramsdict["aa"], 1.0),minshrink)
tracker["extension"] = 4
nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo)
tracker["previous-resolution"] = newres
currentres = newres
tracker["bestsolution"] = mainiteration
bestoutputdir = mainoutputdir
tracker["eliminated-outliers"] = eliminated_outliers
keepgoing = 1
elif(newres < currentres):
if(not tracker["movedup"] and tracker["extension"] < 2 and mainiteration > 1):
keepgoing = 0
if(myid == main_node): print(" Cannot improve resolution, the best result is in the directory main%03d"%tracker["bestsolution"])
else:
if(not tracker["movedup"] and tracker["extension"] > 1 and mainiteration > 1):
if(myid == main_node): print(" Resolution decreased. Will decrease target resolution and will fall back on the best so far: main%03d"%tracker["bestsolution"])
bestoutputdir = os.path.join(masterdir,"main%03d"%tracker["bestsolution"])
elif( tracker["movedup"] and tracker["extension"] > 1 and mainiteration > 1):
if(myid == main_node): print(" Resolution decreased. Will decrease target resolution and will try starting from previous stage: main%03d"%(mainiteration - 1))
bestoutputdir = os.path.join(masterdir,"main%03d"%(mainiteration-1))
elif( mainiteration == 1):
if(myid == main_node): print(" Resolution decreased in the first iteration. It is expected, not to worry")
bestoutputdir = mainoutputdir
tracker["extension"] += 1
else: # missing something here?
if(myid == main_node): print(" Should not be here, ERROR 175!")
break
mpi_finalize()
exit()
if( bestoutputdir != mainoutputdir ):
# This is the key, we just reset the main to previous, so it will be eventually used as a starting in the next iteration
mainoutputdir = bestoutputdir
"""
# Set data from the main previous best to the current.
for procid in xrange(2):
partids[procid] = os.path.join(bestoutputdir,"chunk%01d.txt"%procid)
partstack[procid] = os.path.join(bestoutputdir,"params-chunk%01d.txt"%procid)
"""
if(myid == main_node):
currentres = read_text_file( os.path.join(bestoutputdir,"current_resolution.txt") )[0]
currentres = bcast_number_to_all(currentres, source_node = main_node)
shrink = max(min(2*currentres + paramsdict["aa"], 1.0), minshrink)
tracker["extension"] -= 1
nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo)
tracker["previous-resolution"] = newres
tracker["eliminated-outliers"] = eliminated_outliers
tracker["movedup"] = False
keepgoing = 1
elif(newres == currentres):
if( tracker["extension"] > 0 ):
if(myid == main_node): print("The resolution did not improve. This is look ahead move. Let's try to relax slightly and hope for the best")
tracker["extension"] -= 1
tracker["movedup"] = False
shrink = max(min(2*currentres + paramsdict["aa"], 1.0), minshrink)
nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo)
if( tracker["previous-nx"] == nnxo ):
keepgoing = 0
else:
tracker["previous-resolution"] = newres
currentres = newres
tracker["eliminated-outliers"] = eliminated_outliers
tracker["movedup"] = False
keepgoing = 1
else:
if(myid == main_node): print("The resolution did not improve.")
keepgoing = 0
if( keepgoing == 1 ):
if(myid == main_node):
print(" New shrink and image dimension :",shrink,nxshrink)
"""
# It does not look like it is necessary, we just have to point to the directory as the files should be there.
# Will continue, so update the params files
for procid in xrange(2):
# partids ads partstack contain parameters to be used as starting in the next iteration
if(not os.path.exists(os.path.join(mainoutputdir,"chunk%01d.txt"%procid))):
cmd = "{} {} {}".format("cp -p ", partids[procid] , os.path.join(mainoutputdir,"chunk%01d.txt"%procid))
cmdexecute(cmd)
if(not os.path.exists(os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))):
cmd = "{} {} {}".format("cp -p ", partstack[procid], os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))
cmdexecute(cmd)
"""
previousoutputdir = mainoutputdir
tracker["previous-shrink"] = shrink
tracker["previous-nx"] = nxshrink
else:
if(myid == main_node):
print(" Terminating, the best solution is in the directory main%03d"%tracker["bestsolution"])
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = progname + " stack outdir --phase=1 --ou=outer_radius|sxconsistency.py --phase=3 newlocal/main000 --ou=133 --thresherr=3.0 --params=paramsa outgrouparms"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--phase",
type="int",
default=1,
help=
"Phase =1 prepares resampled stacks, =2 analyzes consistency of orientation parameters"
)
parser.add_option("--ou",
type="int",
default=-1,
help="outer radius for calculation of pixel error")
parser.add_option("--sym",
type="string",
default="c1",
help="symmetry of the refined structure")
parser.add_option(
"--thresherr",
type="float",
default=1.0,
help="Threshold for accpetable orientation errors (in pixels)")
parser.add_option(
"--ndigits",
type="int",
default=1,
help="Accuracy for checking whether parameters are identical")
parser.add_option("--chunk",
type="string",
default="",
help="Root of of four chunk files with indeces")
parser.add_option(
"--params",
type="string",
default="",
help="Root of of six parameter file names with refinement results")
(options, args) = parser.parse_args(arglist[1:])
global_def.BATCH = True
if options.phase == 1 and len(args) == 2:
inputbdb = args[0]
outdir = args[1]
nn = EMUtil.get_image_count(inputbdb)
t = range(nn)
shuffle(t)
chunks = []
for i in xrange(4):
# I use the MPI function here just to easily get the balanced load
j, k = MPI_start_end(nn, 4, i)
chunks.append(t[j:k])
chunks[i].sort()
write_text_file(chunks[i], os.path.join(outdir,
'chunk%01d.txt' % i))
del t
write_text_file([len(chunks[i]) for i in xrange(4)],
os.path.join(outdir, 'chunklengths.txt'))
"""
pt = [[None]]*6
ll=0
for i in xrange(3):
for j in xrange(i+1,4):
pt[ll] = chunks[i]+chunks[j]
ll+=1
for i in xrange(6):
listfile = os.path.join(outdir,'lili%01d.txt'%i)
write_text_file(pt[i],listfile)
outbdb = "bdb:"+ os.path.join(outdir,"X%01d"%i)
cmd = '{} {} {} {}'.format('e2bdb.py', inputbdb, '--makevstack='+outbdb, '--list='+listfile)
subprocess.call(cmd, shell=True)
"""
# Run 6 programs
elif options.phase == 2 and len(args) == 2:
outdir = args[0]
howmanythesame = args[1]
ndigits = options.ndigits #for chc5 1, for ribo 4#4.0
prms = []
for i in xrange(6):
prms.append(
read_text_row(
os.path.join(outdir, options.params + "%01d.txt" % i)))
for j in xrange(len(prms[-1])):
for k in xrange(5):
prms[-1][j][k] = round(prms[-1][j][k], ndigits)
nn = 2 * len(prms[0])
n4 = nn // 4
qt = [[[-1.0]] * nn for i in xrange(6)]
ll = 0
for i in xrange(3):
for j in xrange(i + 1, 4):
qt[ll][i * n4:(i + 1) * n4] = prms[ll][:n4]
qt[ll][j * n4:(j + 1) * n4] = prms[ll][n4:]
ll += 1
thesame = 0
for ll in xrange(len(qt[0])):
rw = []
for j in xrange(6):
if (len(qt[j][ll]) > 1): rw.append(qt[j][ll])
isame = True
for j in xrange(3):
if (rw[0][j] != rw[1][j]):
isame = False
#print ll,rw[0][j], rw[1][j]
break
if (rw[0][j] != rw[2][j]):
isame = False
#print ll,rw[0][j], rw[2][j]
break
if isame: thesame += 1
qt = float(thesame) / nn
print("Proportion of the same orientations ", qt)
write_text_file([qt], os.path.join(outdir, howmanythesame))
######### PHASE 3
elif options.phase == 3 and len(args) == 2:
outdir = args[0]
outgrouparms = args[1]
radius = options.ou
thresherr = options.thresherr #for chc5 1, for ribo 4#4.0
sym = int(options.sym[1:])
qsym = 360.0 / sym
blocks = ['A', 'B', 'C', 'D']
params = {}
ll = 0
for i in xrange(3):
for j in xrange(i + 1, 4):
params[chr(65 + i) + chr(48 + ll)] = []
params[chr(65 + j) + chr(48 + ll)] = []
ll += 1
chunks = {}
chunklengths = {}
for i in xrange(4):
chunks[chr(65 + i)] = map(
int,
read_text_file(
os.path.join(outdir, options.chunk + "%01d.txt" % i)))
chunklengths[chr(65 + i)] = len(chunks[chr(65 + i)])
for i in xrange(6):
prms = read_text_row(
os.path.join(outdir, options.params + "%01d.txt" % i))
for q in blocks:
if q + chr(48 + i) in params:
params[q + chr(48 + i)] = prms[:chunklengths[q]]
del prms[:chunklengths[q]]
pairs = [["A0", "A1"], ["B3", "B4"], ["C1", "C5"], ["D2", "D4"]]
lefts = ["A2", "B0", "C3", "D5"]
# Compute average projection params and pixel errors
avgtrans = {}
pixer = {}
for i, q in enumerate(pairs):
avgtrans[q[0][0]] = [0.0] * chunklengths[q[0][0]]
pixer[q[0][0]] = [0.0] * chunklengths[q[0][0]]
for j in xrange(chunklengths[q[0][0]]):
fifi = [params[q[0]][j], params[q[1]][j]]
nas = [0.0, 0.0, 0.0]
if (sym == 1):
pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0],
fifi[1],
r=radius)
for i in xrange(2):
n1, n2, n3 = getfvec(fifi[i][0], fifi[i][1])
nas[0] += n1
nas[1] += n2
nas[2] += n3
else:
m1, m2, m3 = getfvec(fifi[0][0], fifi[0][1])
nas[0] = m1
nas[1] = m2
nas[2] = m3
#if(k == 2):
# print "XXXX"
# print fifi[0],nas
for i in xrange(1, 2):
qnom = -1.e10
for j in xrange(-1, 2, 1):
t1, t2, t3 = getfvec(fifi[i][0] + j * qsym,
fifi[i][1])
nom = t1 * m1 + t2 * m2 + t3 * m3
if (nom > qnom):
qnom = nom
n1 = t1
n2 = t2
n3 = t3
#if(k == 2):
# print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom
nas[0] += n1
nas[1] += n2
nas[2] += n3
print(qnom, n1, n2, n3, nas)
# To get the correct pixer phi angle has to be taken from the above!!
pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0],
fifi[1],
r=radius)
nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2)
if (nom < 1.e-6):
nphi = 0.0
ntheta = 0.0
else:
ntheta = degrees(acos(nas[2] / nom)) % 360.0
if (sym > 1 and ntheta > 90.0):
nphi = (degrees(atan2(nas[1], nas[0])) -
180.0) % qsym + 180.0
else:
nphi = degrees(atan2(nas[1], nas[0])) % qsym
#print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta)
twod = average2dtransform([fifi[0][2:], fifi[1][2:]])
avgtrans[q[0][0]][j] = [
nphi, ntheta, twod[0], twod[1], twod[2]
]
perr = {}
tgood = 0
for q in blocks:
perr[q] = [True] * chunklengths[q]
for k in xrange(chunklengths[q]):
if (pixer[q][k] > thresherr): perr[q][k] = False
if perr[q][k]: tgood += 1
if (tgood < 4):
print(
" No good images within the pixel error threshold specified"
)
exit()
print(" tgood ", tgood)
hi = hist_list(
[pixer[q][k] for q in blocks for k in xrange(chunklengths[q])], 16)
for i in xrange(len(hi[0])):
print("%4d %12.3f %12.0f " % (i, hi[0][i], hi[1][i]))
# Finished, store average orientation params and table of good images
# store lists of good images for each group
# blocks is indexed by first letter
good = [[] for i in xrange(4)]
bad = [[] for i in xrange(4)]
for i, q in enumerate(blocks):
for k in xrange(chunklengths[q]):
#deprt = max_3D_pixel_error(params[lefts[i]][k],avgtrans[q][k],r=radius)
if perr[q][k]:
good[i].append(chunks[q][k])
#[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k] ])
else:
bad[i].append(chunks[q][k])
#[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k]])
write_text_file(good[i], os.path.join(outdir,
"newgood%01d.txt" % i))
write_text_file(bad[i], os.path.join(outdir, "newbad%01d.txt" % i))
# write out parameters, for those in pairs write out average, for leftouts leave them as they were
# These parameters refer to the original X files.
ll = 0
for m in xrange(6):
if q + chr(48 + m) in params:
prmsgood = []
prmsbad = []
try:
j = lefts.index(q + chr(48 + m))
for k in xrange(chunklengths[q]):
if perr[q][k]:
prmsgood.append(params[q + chr(48 + m)][k])
else:
prmsbad.append(params[q + chr(48 + m)][k])
except:
for k in xrange(chunklengths[q]):
if perr[q][k]:
prmsgood.append(avgtrans[q][k])
else:
prmsbad.append(avgtrans[q][k])
write_text_row(
prmsgood,
os.path.join(outdir,
"params-newgood%01d%01d.txt" % (i, ll)))
write_text_row(
prmsbad,
os.path.join(outdir,
"params-newbad%01d%01d.txt" % (i, ll)))
ll += 1
# Generate newlili files from newgood, these contain original numbering of the total single file
ll = 0
for i in xrange(3):
for j in xrange(i + 1, 4):
write_text_file(good[i] + good[j],
os.path.join(outdir, "newlili%01d.txt" % ll))
ll += 1
# write out parameters, for those in pairs write out average, for leftouts leave them as they were
# These parameters refer to the original X files.
for i in xrange(6):
prms = []
for q in blocks:
if q + chr(48 + i) in params:
try:
j = lefts.index(q + chr(48 + i))
prms += params[q + chr(48 + i)]
except:
prms += avgtrans[q]
write_text_row(prms,
os.path.join(outdir, outgrouparms + "%01d.txt" % i))
# Write chunklengths
chunklengths = [len(good[i]) for i in xrange(4)]
write_text_file(chunklengths, os.path.join(outdir, "chunklengths.txt"))
"""
# We do not use consecutive numbering anymore
# Generate newx files from newgood, these contain consecutive (with gaps) numbering that allows to generate truncated X files from the previous X files
ll = 0
for i in xrange(3):
firstblock = []
l = len(perr[blocks[i]])
for k in xrange(l):
if perr[blocks[i]][k]:
firstblock.append(k)
for j in xrange(i+1,4):
secondblock = []
for k in xrange(len(perr[blocks[j]])):
if perr[blocks[j]][k]:
secondblock.append(k+l)
write_text_file( firstblock + secondblock, os.path.join(outdir,"goodX%01d.txt"%ll) )
ll += 1
del good,bad,firstblock,secondblock,perr
"""
"""
0 1 2 3 4 5
A A A = = =
B = = B B =
= C = C = C
= = D = D D
"""
elif options.phase == 4 and len(args) == 1:
outdir = args[0]
bp = 'badparams'
#outgrouparms= args[1]
radius = options.ou
thresherr = options.thresherr
sym = int(options.sym[1:])
qsym = 360.0 / sym
#params = [[None for i in xrange(3)] for j in xrange(4)]
ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general.
for jj in xrange(4):
params = [None for ii in xrange(ll)]
newbad = map(int, read_text_file(options.params + "%01d.txt" % jj))
nn = len(newbad)
for ii in xrange(ll):
params[ii] = read_text_row(
os.path.join(outdir, bp + "%01d%01d.txt" % (jj, ii)))
assert (nn == len(params[ii]))
# Compute average projection params and pixel errors
avgtrans = [None] * nn
pixer = [0.0] * nn
for j in xrange(nn):
nas = [0.0, 0.0, 0.0]
if (sym == 1):
#pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius)
for i in xrange(ll):
n1, n2, n3 = getfvec(params[i][j][0], params[i][j][1])
nas[0] += n1
nas[1] += n2
nas[2] += n3
else:
m1, m2, m3 = getfvec(params[0][j][0], params[0][j][1])
nas[0] = m1
nas[1] = m2
nas[2] = m3
for i in xrange(1, ll):
qnom = -1.e10
for j in xrange(-1, 2, 1):
t1, t2, t3 = getfvec(params[i][j][0] + j * qsym,
params[i][j][1])
nom = t1 * m1 + t2 * m2 + t3 * m3
if (nom > qnom):
qnom = nom
n1 = t1
n2 = t2
n3 = t3
#if(k == 2):
# print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom
nas[0] += n1
nas[1] += n2
nas[2] += n3
print(qnom, n1, n2, n3, nas)
# To get the correct pixer phi angle has to be taken from the above!!
nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2)
if (nom < 1.e-6):
nphi = 0.0
ntheta = 0.0
else:
ntheta = degrees(acos(nas[2] / nom)) % 360.0
if (sym > 1 and ntheta > 90.0):
nphi = (degrees(atan2(nas[1], nas[0])) -
180.0) % qsym + 180.0
else:
nphi = degrees(atan2(nas[1], nas[0])) % qsym
#print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta)
twod = average2dtransform(
[params[ii][j][2:] for ii in xrange(ll)])
avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]]
perr = errors_per_image(params, avgtrans, thresherr, radius)
rescued = []
rejects = []
for j in xrange(nn):
#print chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)]
if (perr[j][0] <= thresherr): rescued.append([newbad[j], j])
else: rejects.append([newbad[j], j])
if (len(rescued) == 0):
write_text_row([-1, -1],
os.path.join(outdir, "rescued%01d.txt" % jj))
else:
write_text_row(rescued,
os.path.join(outdir, "rescued%01d.txt" % jj))
# We also have to write params.
if (len(rescued) != 0):
for ii in xrange(ll):
write_text_row(
[
params[ii][rescued[k][1]]
for k in xrange(len(rescued))
],
os.path.join(outdir,
"params-rescued%01d%01d.txt" % (jj, ii)))
if (len(rejects) == 0):
write_text_row([-1, -1],
os.path.join(outdir,
'rejects' + "%01d.txt" % jj))
else:
write_text_row(
rejects, os.path.join(outdir, 'rejects' + "%01d.txt" % jj))
if (len(rejects) != 0):
for ii in xrange(ll):
write_text_row(
[
params[ii][rejects[k][1]]
for k in xrange(len(rejects))
],
os.path.join(outdir,
"params-rejects%01d%01d.txt" % (jj, ii)))
hi = hist_list([perr[j][0] for j in xrange(nn)], 16)
print("Pixel errors for BAD GROUP ", chr(65 + jj))
for ii in xrange(len(hi[0])):
print("%4d %12.3f %12.0f " % (ii, hi[0][ii], hi[1][ii]))
elif options.phase == 5 and len(args) == 1:
# This version is for meridien refinement. There are simply three full sets of params.
outdir = args[0]
bp = 'badparams'
#outgrouparms= args[1]
radius = options.ou
thresherr = options.thresherr
sym = int(options.sym[1:])
qsym = 360.0 / sym
#params = [[None for i in xrange(3)] for j in xrange(4)]
ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general.
for jj in xrange(1):
params = [None for ii in xrange(ll)]
#newbad = map(int, read_text_file(options.params+"%01d.txt"%jj) )
#nn = len(newbad)
for ii in xrange(ll):
params[ii] = read_text_row(
os.path.join(outdir, "params%01d.txt" % (ii)))
#assert(nn == len(params[ii]) )
nn = len(params[0])
newbad = range(nn)
# Compute average projection params and pixel errors
avgtrans = [None] * nn
pixer = [0.0] * nn
for j in xrange(nn):
nas = [0.0, 0.0, 0.0]
if (sym == 1):
#pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius)
for i in xrange(ll):
n1, n2, n3 = getfvec(params[i][j][0], params[i][j][1])
nas[0] += n1
nas[1] += n2
nas[2] += n3
else:
m1, m2, m3 = getfvec(params[0][j][0], params[0][j][1])
nas[0] = m1
nas[1] = m2
nas[2] = m3
for i in xrange(1, ll):
qnom = -1.e10
for j in xrange(-1, 2, 1):
t1, t2, t3 = getfvec(params[i][j][0] + j * qsym,
params[i][j][1])
nom = t1 * m1 + t2 * m2 + t3 * m3
if (nom > qnom):
qnom = nom
n1 = t1
n2 = t2
n3 = t3
#if(k == 2):
# print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom
nas[0] += n1
nas[1] += n2
nas[2] += n3
print(qnom, n1, n2, n3, nas)
# To get the correct pixer phi angle has to be taken from the above!!
nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2)
if (nom < 1.e-6):
nphi = 0.0
ntheta = 0.0
else:
ntheta = degrees(acos(nas[2] / nom)) % 360.0
if (sym > 1 and ntheta > 90.0):
nphi = (degrees(atan2(nas[1], nas[0])) -
180.0) % qsym + 180.0
else:
nphi = degrees(atan2(nas[1], nas[0])) % qsym
#print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta)
twod = average2dtransform(
[params[ii][j][2:] for ii in xrange(ll)])
avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]]
write_text_row(avgtrans, os.path.join(outdir, "avgtrans.txt"))
perr = errors_per_image(params, avgtrans, thresherr, radius)
rescued = []
rejects = []
for j in xrange(nn):
#print chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)]
if (perr[j][0] <= thresherr): rescued.append([newbad[j], j])
else: rejects.append([newbad[j], j])
if (len(rescued) == 0):
write_text_row([-1, -1],
os.path.join(outdir, "rescued%01d.txt" % jj))
else:
write_text_row(rescued,
os.path.join(outdir, "rescued%01d.txt" % jj))
# We also have to write params.
if (len(rescued) != 0):
for ii in xrange(ll):
write_text_row(
[
params[ii][rescued[k][1]]
for k in xrange(len(rescued))
],
os.path.join(outdir,
"params-rescued%01d%01d.txt" % (jj, ii)))
if (len(rejects) == 0):
write_text_row([-1, -1],
os.path.join(outdir,
'rejects' + "%01d.txt" % jj))
else:
write_text_row(
rejects, os.path.join(outdir, 'rejects' + "%01d.txt" % jj))
if (len(rejects) != 0):
for ii in xrange(ll):
write_text_row(
[
params[ii][rejects[k][1]]
for k in xrange(len(rejects))
],
os.path.join(outdir,
"params-rejects%01d%01d.txt" % (jj, ii)))
hi = hist_list([perr[j][0] for j in xrange(nn)], 16)
print("Pixel errors for BAD GROUP ", chr(65 + jj))
for ii in xrange(len(hi[0])):
print("%4d %12.3f %12.0f " % (ii, hi[0][ii], hi[1][ii]))
else:
print("Usage: ")
print("""
Phase 1: sxconsistency.py --phase=1 bdb:data outdir
output files are:
in directory outdir: lili0.txt to lili5.txt contain indices of images in resampled six groups
bdb:dataX0 to bdb:dataX5 are metafiles containing six resampled groups of images derived from bdb:data
Phase 2 sxconsistency.py --phase=2 outdir --ndigits=1 --params=paramsb howmanythesame.txt
outdir - directory containing files lili0.txt to lili5.txt produced in phase 1
--params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be
master/main/params0.txt to master/main/params5.txt
output files:
howmanythesame.txt - contains one number, a ratio of number of images that did not change orientations
to the total number of images
Phase 3: sxconsistency.py --phase=3 outdir --ou=133 --thresherr=3.0 --params=paramsa outgrouparms
input files:
outdir - directory containing files lili0.txt to lili5.txt produced in phase 1
--params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be
master/main/params0.txt to master/main/params5.txt
output files:
outgrouparms*.txt - Root of of six parameters files with average 3D orientation parameters computed from six runs, can be imported into bdb:data
The next two files contain the original image numbers refering to the top bdb.
outdir/newgood.txt - Text file with indices of images whose orientation parameters arrors are below specified thresherr (to be kept)
outdir/bad.txt - Text file with indices of images whose orientation parameters arrors are above specified thresherr (to be rejected)
Phase 4: sxconsistency.py --phase=4 outdir --ou=133 --thresherr=3.0 --params=master/main001/newbad
input files:
outdir - directory containing files badparamsij.txt i= 0,3, j=0,2, which resulted from three alignments of four badchunk images
--params= - Root of four files with original indices of bad images, they will be read and a subset corresponding to rescued ones will be outputed
output files:
rescued*.txt - four files with indices of accepted images from badchunk.
rejects*.txt - four files with indices of rejected images from badchunk.
There are two columns: [number in original bdb:chunk, number in bdb:newbad]
""")
print("Please run '" + progname + " -h' for detailed options")
global_def.BATCH = False
def ali3d_MPI(stack, ref_vol, outdir, maskfile = None, ir = 1, ou = -1, rs = 1,
xr = "4 2 2 1", yr = "-1", ts = "1 1 0.5 0.25", delta = "10 6 4 4", an = "-1",
center = 0, maxit = 5, term = 95, CTF = False, fourvar = False, snr = 1.0, ref_a = "S", sym = "c1",
sort=True, cutoff=999.99, pix_cutoff="0", two_tail=False, model_jump="1 1 1 1 1", restart=False, save_half=False,
protos=None, oplane=None, lmask=-1, ilmask=-1, findseam=False, vertstep=None, hpars="-1", hsearch="73.0 170.0",
full_output = False, compare_repro = False, compare_ref_free = "-1", ref_free_cutoff= "-1 -1 -1 -1",
wcmask = None, debug = False, recon_pad = 4):
from alignment import Numrinit, prepare_refrings
from utilities import model_circle, get_image, drop_image, get_input_from_string
from utilities import bcast_list_to_all, bcast_number_to_all, reduce_EMData_to_root, bcast_EMData_to_all
from utilities import send_attr_dict
from utilities import get_params_proj, file_type
from fundamentals import rot_avg_image
import os
import types
from utilities import print_begin_msg, print_end_msg, print_msg
from mpi import mpi_bcast, mpi_comm_size, mpi_comm_rank, MPI_FLOAT, MPI_COMM_WORLD, mpi_barrier, mpi_reduce
from mpi import mpi_reduce, MPI_INT, MPI_SUM, mpi_finalize
from filter import filt_ctf
from projection import prep_vol, prgs
from statistics import hist_list, varf3d_MPI, fsc_mask
from numpy import array, bincount, array2string, ones
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
if myid == main_node:
if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "ali3d_MPI", 1)
os.mkdir(outdir)
mpi_barrier(MPI_COMM_WORLD)
if debug:
from time import sleep
while not os.path.exists(outdir):
print "Node ",myid," waiting..."
sleep(5)
info_file = os.path.join(outdir, "progress%04d"%myid)
finfo = open(info_file, 'w')
else:
finfo = None
mjump = get_input_from_string(model_jump)
xrng = get_input_from_string(xr)
if yr == "-1": yrng = xrng
else : yrng = get_input_from_string(yr)
step = get_input_from_string(ts)
delta = get_input_from_string(delta)
ref_free_cutoff = get_input_from_string(ref_free_cutoff)
pix_cutoff = get_input_from_string(pix_cutoff)
lstp = min(len(xrng), len(yrng), len(step), len(delta))
if an == "-1":
an = [-1] * lstp
else:
an = get_input_from_string(an)
# make sure pix_cutoff is set for all iterations
if len(pix_cutoff)<lstp:
for i in xrange(len(pix_cutoff),lstp):
pix_cutoff.append(pix_cutoff[-1])
# don't waste time on sub-pixel alignment for low-resolution ang incr
for i in range(len(step)):
if (delta[i] > 4 or delta[i] == -1) and step[i] < 1:
step[i] = 1
first_ring = int(ir)
rstep = int(rs)
last_ring = int(ou)
max_iter = int(maxit)
center = int(center)
nrefs = EMUtil.get_image_count( ref_vol )
nmasks = 0
if maskfile:
# read number of masks within each maskfile (mc)
nmasks = EMUtil.get_image_count( maskfile )
# open masks within maskfile (mc)
maskF = EMData.read_images(maskfile, xrange(nmasks))
vol = EMData.read_images(ref_vol, xrange(nrefs))
nx = vol[0].get_xsize()
## make sure box sizes are the same
if myid == main_node:
im=EMData.read_images(stack,[0])
bx = im[0].get_xsize()
if bx!=nx:
print_msg("Error: Stack box size (%i) differs from initial model (%i)\n"%(bx,nx))
sys.exit()
del im,bx
# for helical processing:
helicalrecon = False
if protos is not None or hpars != "-1" or findseam is True:
helicalrecon = True
# if no out-of-plane param set, use 5 degrees
if oplane is None:
oplane=5.0
if protos is not None:
proto = get_input_from_string(protos)
if len(proto) != nrefs:
print_msg("Error: insufficient protofilament numbers supplied")
sys.exit()
if hpars != "-1":
hpars = get_input_from_string(hpars)
if len(hpars) != 2*nrefs:
print_msg("Error: insufficient helical parameters supplied")
sys.exit()
## create helical parameter file for helical reconstruction
if helicalrecon is True and myid == main_node:
from hfunctions import createHpar
# create initial helical parameter files
dp=[0]*nrefs
dphi=[0]*nrefs
vdp=[0]*nrefs
vdphi=[0]*nrefs
for iref in xrange(nrefs):
hpar = os.path.join(outdir,"hpar%02d.spi"%(iref))
params = False
if hpars != "-1":
# if helical parameters explicitly given, set twist & rise
params = [float(hpars[iref*2]),float(hpars[(iref*2)+1])]
dp[iref],dphi[iref],vdp[iref],vdphi[iref] = createHpar(hpar,proto[iref],params,vertstep)
# get values for helical search parameters
hsearch = get_input_from_string(hsearch)
if len(hsearch) != 2:
print_msg("Error: specify outer and inner radii for helical search")
sys.exit()
if last_ring < 0 or last_ring > int(nx/2)-2 : last_ring = int(nx/2) - 2
if myid == main_node:
# import user_functions
# user_func = user_functions.factory[user_func_name]
print_begin_msg("ali3d_MPI")
print_msg("Input stack : %s\n"%(stack))
print_msg("Reference volume : %s\n"%(ref_vol))
print_msg("Output directory : %s\n"%(outdir))
if nmasks > 0:
print_msg("Maskfile (number of masks) : %s (%i)\n"%(maskfile,nmasks))
print_msg("Inner radius : %i\n"%(first_ring))
print_msg("Outer radius : %i\n"%(last_ring))
print_msg("Ring step : %i\n"%(rstep))
print_msg("X search range : %s\n"%(xrng))
print_msg("Y search range : %s\n"%(yrng))
print_msg("Translational step : %s\n"%(step))
print_msg("Angular step : %s\n"%(delta))
print_msg("Angular search range : %s\n"%(an))
print_msg("Maximum iteration : %i\n"%(max_iter))
print_msg("Center type : %i\n"%(center))
print_msg("CTF correction : %s\n"%(CTF))
print_msg("Signal-to-Noise Ratio : %f\n"%(snr))
print_msg("Reference projection method : %s\n"%(ref_a))
print_msg("Symmetry group : %s\n"%(sym))
print_msg("Fourier padding for 3D : %i\n"%(recon_pad))
print_msg("Number of reference models : %i\n"%(nrefs))
print_msg("Sort images between models : %s\n"%(sort))
print_msg("Allow images to jump : %s\n"%(mjump))
print_msg("CC cutoff standard dev : %f\n"%(cutoff))
print_msg("Two tail cutoff : %s\n"%(two_tail))
print_msg("Termination pix error : %f\n"%(term))
print_msg("Pixel error cutoff : %s\n"%(pix_cutoff))
print_msg("Restart : %s\n"%(restart))
print_msg("Full output : %s\n"%(full_output))
print_msg("Compare reprojections : %s\n"%(compare_repro))
print_msg("Compare ref free class avgs : %s\n"%(compare_ref_free))
print_msg("Use cutoff from ref free : %s\n"%(ref_free_cutoff))
if protos:
print_msg("Protofilament numbers : %s\n"%(proto))
print_msg("Using helical search range : %s\n"%hsearch)
if findseam is True:
print_msg("Using seam-based reconstruction\n")
if hpars != "-1":
print_msg("Using hpars : %s\n"%hpars)
if vertstep != None:
print_msg("Using vertical step : %.2f\n"%vertstep)
if save_half is True:
print_msg("Saving even/odd halves\n")
for i in xrange(100) : print_msg("*")
print_msg("\n\n")
if maskfile:
if type(maskfile) is types.StringType: mask3D = get_image(maskfile)
else: mask3D = maskfile
else: mask3D = model_circle(last_ring, nx, nx, nx)
numr = Numrinit(first_ring, last_ring, rstep, "F")
mask2D = model_circle(last_ring,nx,nx) - model_circle(first_ring,nx,nx)
fscmask = model_circle(last_ring,nx,nx,nx)
if CTF:
from filter import filt_ctf
from reconstruction_rjh import rec3D_MPI_noCTF
if myid == main_node:
active = EMUtil.get_all_attributes(stack, 'active')
list_of_particles = []
for im in xrange(len(active)):
if active[im]: list_of_particles.append(im)
del active
nima = len(list_of_particles)
else:
nima = 0
total_nima = bcast_number_to_all(nima, source_node = main_node)
if myid != main_node:
list_of_particles = [-1]*total_nima
list_of_particles = bcast_list_to_all(list_of_particles, source_node = main_node)
image_start, image_end = MPI_start_end(total_nima, number_of_proc, myid)
# create a list of images for each node
list_of_particles = list_of_particles[image_start: image_end]
nima = len(list_of_particles)
if debug:
finfo.write("image_start, image_end: %d %d\n" %(image_start, image_end))
finfo.flush()
data = EMData.read_images(stack, list_of_particles)
t_zero = Transform({"type":"spider","phi":0,"theta":0,"psi":0,"tx":0,"ty":0})
transmulti = [[t_zero for i in xrange(nrefs)] for j in xrange(nima)]
for iref,im in ((iref,im) for iref in xrange(nrefs) for im in xrange(nima)):
if nrefs == 1:
transmulti[im][iref] = data[im].get_attr("xform.projection")
else:
# if multi models, keep track of eulers for all models
try:
transmulti[im][iref] = data[im].get_attr("eulers_txty.%i"%iref)
except:
data[im].set_attr("eulers_txty.%i"%iref,t_zero)
scoremulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)]
pixelmulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)]
ref_res = [0.0 for x in xrange(nrefs)]
apix = data[0].get_attr('apix_x')
# for oplane parameter, create cylindrical mask
if oplane is not None and myid == main_node:
from hfunctions import createCylMask
cmaskf=os.path.join(outdir, "mask3D_cyl.mrc")
mask3D = createCylMask(data,ou,lmask,ilmask,cmaskf)
# if finding seam of helix, create wedge masks
if findseam is True:
wedgemask=[]
for pf in xrange(nrefs):
wedgemask.append(EMData())
# wedgemask option
if wcmask is not None:
wcmask = get_input_from_string(wcmask)
if len(wcmask) != 3:
print_msg("Error: wcmask option requires 3 values: x y radius")
sys.exit()
# determine if particles have helix info:
try:
data[0].get_attr('h_angle')
original_data = []
boxmask = True
from hfunctions import createBoxMask
except:
boxmask = False
# prepare particles
for im in xrange(nima):
data[im].set_attr('ID', list_of_particles[im])
data[im].set_attr('pix_score', int(0))
if CTF:
# only phaseflip particles, not full CTF correction
ctf_params = data[im].get_attr("ctf")
st = Util.infomask(data[im], mask2D, False)
data[im] -= st[0]
data[im] = filt_ctf(data[im], ctf_params, sign = -1, binary=1)
data[im].set_attr('ctf_applied', 1)
# for window mask:
if boxmask is True:
h_angle = data[im].get_attr("h_angle")
original_data.append(data[im].copy())
bmask = createBoxMask(nx,apix,ou,lmask,h_angle)
data[im]*=bmask
del bmask
if debug:
finfo.write( '%d loaded \n' % nima )
finfo.flush()
if myid == main_node:
# initialize data for the reference preparation function
ref_data = [ mask3D, max(center,0), None, None, None, None ]
# for method -1, switch off centering in user function
from time import time
# this is needed for gathering of pixel errors
disps = []
recvcount = []
disps_score = []
recvcount_score = []
for im in xrange(number_of_proc):
if( im == main_node ):
disps.append(0)
disps_score.append(0)
else:
disps.append(disps[im-1] + recvcount[im-1])
disps_score.append(disps_score[im-1] + recvcount_score[im-1])
ib, ie = MPI_start_end(total_nima, number_of_proc, im)
recvcount.append( ie - ib )
recvcount_score.append((ie-ib)*nrefs)
pixer = [0.0]*nima
cs = [0.0]*3
total_iter = 0
volodd = EMData.read_images(ref_vol, xrange(nrefs))
voleve = EMData.read_images(ref_vol, xrange(nrefs))
if restart:
# recreate initial volumes from alignments stored in header
itout = "000_00"
for iref in xrange(nrefs):
if(nrefs == 1):
modout = ""
else:
modout = "_model_%02d"%(iref)
if(sort):
group = iref
for im in xrange(nima):
imgroup = data[im].get_attr('group')
if imgroup == iref:
data[im].set_attr('xform.projection',transmulti[im][iref])
else:
group = int(999)
for im in xrange(nima):
data[im].set_attr('xform.projection',transmulti[im][iref])
fscfile = os.path.join(outdir, "fsc_%s%s"%(itout,modout))
vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF(data, sym, fscmask, fscfile, myid, main_node, index = group, npad = recon_pad)
if myid == main_node:
if helicalrecon:
from hfunctions import processHelicalVol
vstep=None
if vertstep is not None:
vstep=(vdp[iref],vdphi[iref])
print_msg("Old rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref]))
hvals=processHelicalVol(vol[iref],voleve[iref],volodd[iref],iref,outdir,itout,
dp[iref],dphi[iref],apix,hsearch,findseam,vstep,wcmask)
(vol[iref],voleve[iref],volodd[iref],dp[iref],dphi[iref],vdp[iref],vdphi[iref])=hvals
print_msg("New rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref]))
# get new FSC from symmetrized half volumes
fscc = fsc_mask( volodd[iref], voleve[iref], mask3D, rstep, fscfile)
else:
vol[iref].write_image(os.path.join(outdir, "vol_%s.hdf"%itout),-1)
if save_half is True:
volodd[iref].write_image(os.path.join(outdir, "volodd_%s.hdf"%itout),-1)
voleve[iref].write_image(os.path.join(outdir, "voleve_%s.hdf"%itout),-1)
if nmasks > 1:
# Read mask for multiplying
ref_data[0] = maskF[iref]
ref_data[2] = vol[iref]
ref_data[3] = fscc
# call user-supplied function to prepare reference image, i.e., center and filter it
vol[iref], cs,fl = ref_ali3d(ref_data)
vol[iref].write_image(os.path.join(outdir, "volf_%s.hdf"%(itout)),-1)
if (apix == 1):
res_msg = "Models filtered at spatial frequency of:\t"
res = fl
else:
res_msg = "Models filtered at resolution of: \t"
res = apix / fl
ares = array2string(array(res), precision = 2)
print_msg("%s%s\n\n"%(res_msg,ares))
bcast_EMData_to_all(vol[iref], myid, main_node)
# write out headers, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
# projection matching
for N_step in xrange(lstp):
terminate = 0
Iter = -1
while(Iter < max_iter-1 and terminate == 0):
Iter += 1
total_iter += 1
itout = "%03g_%02d" %(delta[N_step], Iter)
if myid == main_node:
print_msg("ITERATION #%3d, inner iteration #%3d\nDelta = %4.1f, an = %5.2f, xrange = %5.2f, yrange = %5.2f, step = %5.2f\n\n"%(N_step, Iter, delta[N_step], an[N_step], xrng[N_step],yrng[N_step],step[N_step]))
for iref in xrange(nrefs):
if myid == main_node: start_time = time()
volft,kb = prep_vol( vol[iref] )
## constrain projections to out of plane parameter
theta1 = None
theta2 = None
if oplane is not None:
theta1 = 90-oplane
theta2 = 90+oplane
refrings = prepare_refrings( volft, kb, nx, delta[N_step], ref_a, sym, numr, MPI=True, phiEqpsi = "Minus", initial_theta=theta1, delta_theta=theta2)
del volft,kb
if myid== main_node:
print_msg( "Time to prepare projections for model %i: %s\n" % (iref, legibleTime(time()-start_time)) )
start_time = time()
for im in xrange( nima ):
data[im].set_attr("xform.projection", transmulti[im][iref])
if an[N_step] == -1:
t1, peak, pixer[im] = proj_ali_incore(data[im],refrings,numr,xrng[N_step],yrng[N_step],step[N_step],finfo)
else:
t1, peak, pixer[im] = proj_ali_incore_local(data[im],refrings,numr,xrng[N_step],yrng[N_step],step[N_step],an[N_step],finfo)
#data[im].set_attr("xform.projection"%iref, t1)
if nrefs > 1: data[im].set_attr("eulers_txty.%i"%iref,t1)
scoremulti[im][iref] = peak
from pixel_error import max_3D_pixel_error
# t1 is the current param, t2 is old
t2 = transmulti[im][iref]
pixelmulti[im][iref] = max_3D_pixel_error(t1,t2,numr[-3])
transmulti[im][iref] = t1
if myid == main_node:
print_msg("Time of alignment for model %i: %s\n"%(iref, legibleTime(time()-start_time)))
start_time = time()
# gather scoring data from all processors
from mpi import mpi_gatherv
scoremultisend = sum(scoremulti,[])
pixelmultisend = sum(pixelmulti,[])
tmp = mpi_gatherv(scoremultisend,len(scoremultisend),MPI_FLOAT, recvcount_score, disps_score, MPI_FLOAT, main_node,MPI_COMM_WORLD)
tmp1 = mpi_gatherv(pixelmultisend,len(pixelmultisend),MPI_FLOAT, recvcount_score, disps_score, MPI_FLOAT, main_node,MPI_COMM_WORLD)
tmp = mpi_bcast(tmp,(total_nima * nrefs), MPI_FLOAT,0, MPI_COMM_WORLD)
tmp1 = mpi_bcast(tmp1,(total_nima * nrefs), MPI_FLOAT,0, MPI_COMM_WORLD)
tmp = map(float,tmp)
tmp1 = map(float,tmp1)
score = array(tmp).reshape(-1,nrefs)
pixelerror = array(tmp1).reshape(-1,nrefs)
score_local = array(scoremulti)
mean_score = score.mean(axis=0)
std_score = score.std(axis=0)
cut = mean_score - (cutoff * std_score)
cut2 = mean_score + (cutoff * std_score)
res_max = score_local.argmax(axis=1)
minus_cc = [0.0 for x in xrange(nrefs)]
minus_pix = [0.0 for x in xrange(nrefs)]
minus_ref = [0.0 for x in xrange(nrefs)]
#output pixel errors
if(myid == main_node):
from statistics import hist_list
lhist = 20
pixmin = pixelerror.min(axis=1)
region, histo = hist_list(pixmin, lhist)
if(region[0] < 0.0): region[0] = 0.0
print_msg("Histogram of pixel errors\n ERROR number of particles\n")
for lhx in xrange(lhist):
print_msg(" %10.3f %7d\n"%(region[lhx], histo[lhx]))
# Terminate if 95% within 1 pixel error
im = 0
for lhx in xrange(lhist):
if(region[lhx] > 1.0): break
im += histo[lhx]
print_msg( "Percent of particles with pixel error < 1: %f\n\n"% (im/float(total_nima)*100))
term_cond = float(term)/100
if(im/float(total_nima) > term_cond):
terminate = 1
print_msg("Terminating internal loop\n")
del region, histo
terminate = mpi_bcast(terminate, 1, MPI_INT, 0, MPI_COMM_WORLD)
terminate = int(terminate[0])
for im in xrange(nima):
if(sort==False):
data[im].set_attr('group',999)
elif (mjump[N_step]==1):
data[im].set_attr('group',int(res_max[im]))
pix_run = data[im].get_attr('pix_score')
if (pix_cutoff[N_step]==1 and (terminate==1 or Iter == max_iter-1)):
if (pixelmulti[im][int(res_max[im])] > 1):
data[im].set_attr('pix_score',int(777))
if (score_local[im][int(res_max[im])]<cut[int(res_max[im])]) or (two_tail and score_local[im][int(res_max[im])]>cut2[int(res_max[im])]):
data[im].set_attr('group',int(888))
minus_cc[int(res_max[im])] = minus_cc[int(res_max[im])] + 1
if(pix_run == 777):
data[im].set_attr('group',int(777))
minus_pix[int(res_max[im])] = minus_pix[int(res_max[im])] + 1
if (compare_ref_free != "-1") and (ref_free_cutoff[N_step] != -1) and (total_iter > 1):
id = data[im].get_attr('ID')
if id in rejects:
data[im].set_attr('group',int(666))
minus_ref[int(res_max[im])] = minus_ref[int(res_max[im])] + 1
minus_cc_tot = mpi_reduce(minus_cc,nrefs,MPI_FLOAT,MPI_SUM,0,MPI_COMM_WORLD)
minus_pix_tot = mpi_reduce(minus_pix,nrefs,MPI_FLOAT,MPI_SUM,0,MPI_COMM_WORLD)
minus_ref_tot = mpi_reduce(minus_ref,nrefs,MPI_FLOAT,MPI_SUM,0,MPI_COMM_WORLD)
if (myid == main_node):
if(sort):
tot_max = score.argmax(axis=1)
res = bincount(tot_max)
else:
res = ones(nrefs) * total_nima
print_msg("Particle distribution: \t\t%s\n"%(res*1.0))
afcut1 = res - minus_cc_tot
afcut2 = afcut1 - minus_pix_tot
afcut3 = afcut2 - minus_ref_tot
print_msg("Particle distribution after cc cutoff:\t\t%s\n"%(afcut1))
print_msg("Particle distribution after pix cutoff:\t\t%s\n"%(afcut2))
print_msg("Particle distribution after ref cutoff:\t\t%s\n\n"%(afcut3))
res = [0.0 for i in xrange(nrefs)]
for iref in xrange(nrefs):
if(center == -1):
from utilities import estimate_3D_center_MPI, rotate_3D_shift
dummy=EMData()
cs[0], cs[1], cs[2], dummy, dummy = estimate_3D_center_MPI(data, total_nima, myid, number_of_proc, main_node)
cs = mpi_bcast(cs, 3, MPI_FLOAT, main_node, MPI_COMM_WORLD)
cs = [-float(cs[0]), -float(cs[1]), -float(cs[2])]
rotate_3D_shift(data, cs)
if(sort):
group = iref
for im in xrange(nima):
imgroup = data[im].get_attr('group')
if imgroup == iref:
data[im].set_attr('xform.projection',transmulti[im][iref])
else:
group = int(999)
for im in xrange(nima):
data[im].set_attr('xform.projection',transmulti[im][iref])
if(nrefs == 1):
modout = ""
else:
modout = "_model_%02d"%(iref)
fscfile = os.path.join(outdir, "fsc_%s%s"%(itout,modout))
vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF(data, sym, fscmask, fscfile, myid, main_node, index=group, npad=recon_pad)
if myid == main_node:
print_msg("3D reconstruction time for model %i: %s\n"%(iref, legibleTime(time()-start_time)))
start_time = time()
# Compute Fourier variance
if fourvar:
outvar = os.path.join(outdir, "volVar_%s.hdf"%(itout))
ssnr_file = os.path.join(outdir, "ssnr_%s"%(itout))
varf = varf3d_MPI(data, ssnr_text_file=ssnr_file, mask2D=None, reference_structure=vol[iref], ou=last_ring, rw=1.0, npad=1, CTF=None, sign=1, sym=sym, myid=myid)
if myid == main_node:
print_msg("Time to calculate 3D Fourier variance for model %i: %s\n"%(iref, legibleTime(time()-start_time)))
start_time = time()
varf = 1.0/varf
varf.write_image(outvar,-1)
else: varf = None
if myid == main_node:
if helicalrecon:
from hfunctions import processHelicalVol
vstep=None
if vertstep is not None:
vstep=(vdp[iref],vdphi[iref])
print_msg("Old rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref]))
hvals=processHelicalVol(vol[iref],voleve[iref],volodd[iref],iref,outdir,itout,
dp[iref],dphi[iref],apix,hsearch,findseam,vstep,wcmask)
(vol[iref],voleve[iref],volodd[iref],dp[iref],dphi[iref],vdp[iref],vdphi[iref])=hvals
print_msg("New rise and twist for model %i : %8.3f, %8.3f\n"%(iref,dp[iref],dphi[iref]))
# get new FSC from symmetrized half volumes
fscc = fsc_mask( volodd[iref], voleve[iref], mask3D, rstep, fscfile)
print_msg("Time to search and apply helical symmetry for model %i: %s\n\n"%(iref, legibleTime(time()-start_time)))
start_time = time()
else:
vol[iref].write_image(os.path.join(outdir, "vol_%s.hdf"%(itout)),-1)
if save_half is True:
volodd[iref].write_image(os.path.join(outdir, "volodd_%s.hdf"%(itout)),-1)
voleve[iref].write_image(os.path.join(outdir, "voleve_%s.hdf"%(itout)),-1)
if nmasks > 1:
# Read mask for multiplying
ref_data[0] = maskF[iref]
ref_data[2] = vol[iref]
ref_data[3] = fscc
ref_data[4] = varf
# call user-supplied function to prepare reference image, i.e., center and filter it
vol[iref], cs,fl = ref_ali3d(ref_data)
vol[iref].write_image(os.path.join(outdir, "volf_%s.hdf"%(itout)),-1)
if (apix == 1):
res_msg = "Models filtered at spatial frequency of:\t"
res[iref] = fl
else:
res_msg = "Models filtered at resolution of: \t"
res[iref] = apix / fl
del varf
bcast_EMData_to_all(vol[iref], myid, main_node)
if compare_ref_free != "-1": compare_repro = True
if compare_repro:
outfile_repro = comp_rep(refrings, data, itout, modout, vol[iref], group, nima, nx, myid, main_node, outdir)
mpi_barrier(MPI_COMM_WORLD)
if compare_ref_free != "-1":
ref_free_output = os.path.join(outdir,"ref_free_%s%s"%(itout,modout))
rejects = compare(compare_ref_free, outfile_repro,ref_free_output,yrng[N_step], xrng[N_step], rstep,nx,apix,ref_free_cutoff[N_step], number_of_proc, myid, main_node)
# retrieve alignment params from all processors
par_str = ['xform.projection','ID','group']
if nrefs > 1:
for iref in xrange(nrefs):
par_str.append('eulers_txty.%i'%iref)
if myid == main_node:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, image_start, image_end, number_of_proc)
else: send_attr_dict(main_node, data, par_str, image_start, image_end)
if myid == main_node:
ares = array2string(array(res), precision = 2)
print_msg("%s%s\n\n"%(res_msg,ares))
dummy = EMData()
if full_output:
nimat = EMUtil.get_image_count(stack)
output_file = os.path.join(outdir, "paramout_%s"%itout)
foutput = open(output_file, 'w')
for im in xrange(nimat):
# save the parameters for each of the models
outstring = ""
dummy.read_image(stack,im,True)
param3d = dummy.get_attr('xform.projection')
g = dummy.get_attr("group")
# retrieve alignments in EMAN-format
pE = param3d.get_params('eman')
outstring += "%f\t%f\t%f\t%f\t%f\t%i\n" %(pE["az"], pE["alt"], pE["phi"], pE["tx"], pE["ty"],g)
foutput.write(outstring)
foutput.close()
del dummy
mpi_barrier(MPI_COMM_WORLD)
# mpi_finalize()
if myid == main_node: print_end_msg("ali3d_MPI")
def ali3d_MPI(stack,
ref_vol,
outdir,
maskfile=None,
ir=1,
ou=-1,
rs=1,
xr="4 2 2 1",
yr="-1",
ts="1 1 0.5 0.25",
delta="10 6 4 4",
an="-1",
center=0,
maxit=5,
term=95,
CTF=False,
fourvar=False,
snr=1.0,
ref_a="S",
sym="c1",
sort=True,
cutoff=999.99,
pix_cutoff="0",
two_tail=False,
model_jump="1 1 1 1 1",
restart=False,
save_half=False,
protos=None,
oplane=None,
lmask=-1,
ilmask=-1,
findseam=False,
vertstep=None,
hpars="-1",
hsearch="0.0 50.0",
full_output=False,
compare_repro=False,
compare_ref_free="-1",
ref_free_cutoff="-1 -1 -1 -1",
wcmask=None,
debug=False,
recon_pad=4,
olmask=75):
from alignment import Numrinit, prepare_refrings
from utilities import model_circle, get_image, drop_image, get_input_from_string
from utilities import bcast_list_to_all, bcast_number_to_all, reduce_EMData_to_root, bcast_EMData_to_all
from utilities import send_attr_dict
from utilities import get_params_proj, file_type
from fundamentals import rot_avg_image
import os
import types
from utilities import print_begin_msg, print_end_msg, print_msg
from mpi import mpi_bcast, mpi_comm_size, mpi_comm_rank, MPI_FLOAT, MPI_COMM_WORLD, mpi_barrier, mpi_reduce
from mpi import mpi_reduce, MPI_INT, MPI_SUM, mpi_finalize
from filter import filt_ctf
from projection import prep_vol, prgs
from statistics import hist_list, varf3d_MPI, fsc_mask
from numpy import array, bincount, array2string, ones
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
if myid == main_node:
if os.path.exists(outdir):
ERROR(
'Output directory exists, please change the name and restart the program',
"ali3d_MPI", 1)
os.mkdir(outdir)
mpi_barrier(MPI_COMM_WORLD)
if debug:
from time import sleep
while not os.path.exists(outdir):
print "Node ", myid, " waiting..."
sleep(5)
info_file = os.path.join(outdir, "progress%04d" % myid)
finfo = open(info_file, 'w')
else:
finfo = None
mjump = get_input_from_string(model_jump)
xrng = get_input_from_string(xr)
if yr == "-1": yrng = xrng
else: yrng = get_input_from_string(yr)
step = get_input_from_string(ts)
delta = get_input_from_string(delta)
ref_free_cutoff = get_input_from_string(ref_free_cutoff)
pix_cutoff = get_input_from_string(pix_cutoff)
lstp = min(len(xrng), len(yrng), len(step), len(delta))
if an == "-1":
an = [-1] * lstp
else:
an = get_input_from_string(an)
# make sure pix_cutoff is set for all iterations
if len(pix_cutoff) < lstp:
for i in xrange(len(pix_cutoff), lstp):
pix_cutoff.append(pix_cutoff[-1])
# don't waste time on sub-pixel alignment for low-resolution ang incr
for i in range(len(step)):
if (delta[i] > 4 or delta[i] == -1) and step[i] < 1:
step[i] = 1
first_ring = int(ir)
rstep = int(rs)
last_ring = int(ou)
max_iter = int(maxit)
center = int(center)
nrefs = EMUtil.get_image_count(ref_vol)
nmasks = 0
if maskfile:
# read number of masks within each maskfile (mc)
nmasks = EMUtil.get_image_count(maskfile)
# open masks within maskfile (mc)
maskF = EMData.read_images(maskfile, xrange(nmasks))
vol = EMData.read_images(ref_vol, xrange(nrefs))
nx = vol[0].get_xsize()
## make sure box sizes are the same
if myid == main_node:
im = EMData.read_images(stack, [0])
bx = im[0].get_xsize()
if bx != nx:
print_msg(
"Error: Stack box size (%i) differs from initial model (%i)\n"
% (bx, nx))
sys.exit()
del im, bx
# for helical processing:
helicalrecon = False
if protos is not None or hpars != "-1" or findseam is True:
helicalrecon = True
# if no out-of-plane param set, use 5 degrees
if oplane is None:
oplane = 5.0
if protos is not None:
proto = get_input_from_string(protos)
if len(proto) != nrefs:
print_msg("Error: insufficient protofilament numbers supplied")
sys.exit()
if hpars != "-1":
hpars = get_input_from_string(hpars)
if len(hpars) != 2 * nrefs:
print_msg("Error: insufficient helical parameters supplied")
sys.exit()
## create helical parameter file for helical reconstruction
if helicalrecon is True and myid == main_node:
from hfunctions import createHpar
# create initial helical parameter files
dp = [0] * nrefs
dphi = [0] * nrefs
vdp = [0] * nrefs
vdphi = [0] * nrefs
for iref in xrange(nrefs):
hpar = os.path.join(outdir, "hpar%02d.spi" % (iref))
params = False
if hpars != "-1":
# if helical parameters explicitly given, set twist & rise
params = [float(hpars[iref * 2]), float(hpars[(iref * 2) + 1])]
dp[iref], dphi[iref], vdp[iref], vdphi[iref] = createHpar(
hpar, proto[iref], params, vertstep)
# get values for helical search parameters
hsearch = get_input_from_string(hsearch)
if len(hsearch) != 2:
print_msg("Error: specify outer and inner radii for helical search")
sys.exit()
if last_ring < 0 or last_ring > int(nx / 2) - 2:
last_ring = int(nx / 2) - 2
if myid == main_node:
# import user_functions
# user_func = user_functions.factory[user_func_name]
print_begin_msg("ali3d_MPI")
print_msg("Input stack : %s\n" % (stack))
print_msg("Reference volume : %s\n" % (ref_vol))
print_msg("Output directory : %s\n" % (outdir))
if nmasks > 0:
print_msg("Maskfile (number of masks) : %s (%i)\n" %
(maskfile, nmasks))
print_msg("Inner radius : %i\n" % (first_ring))
print_msg("Outer radius : %i\n" % (last_ring))
print_msg("Ring step : %i\n" % (rstep))
print_msg("X search range : %s\n" % (xrng))
print_msg("Y search range : %s\n" % (yrng))
print_msg("Translational step : %s\n" % (step))
print_msg("Angular step : %s\n" % (delta))
print_msg("Angular search range : %s\n" % (an))
print_msg("Maximum iteration : %i\n" % (max_iter))
print_msg("Center type : %i\n" % (center))
print_msg("CTF correction : %s\n" % (CTF))
print_msg("Signal-to-Noise Ratio : %f\n" % (snr))
print_msg("Reference projection method : %s\n" % (ref_a))
print_msg("Symmetry group : %s\n" % (sym))
print_msg("Fourier padding for 3D : %i\n" % (recon_pad))
print_msg("Number of reference models : %i\n" % (nrefs))
print_msg("Sort images between models : %s\n" % (sort))
print_msg("Allow images to jump : %s\n" % (mjump))
print_msg("CC cutoff standard dev : %f\n" % (cutoff))
print_msg("Two tail cutoff : %s\n" % (two_tail))
print_msg("Termination pix error : %f\n" % (term))
print_msg("Pixel error cutoff : %s\n" % (pix_cutoff))
print_msg("Restart : %s\n" % (restart))
print_msg("Full output : %s\n" % (full_output))
print_msg("Compare reprojections : %s\n" % (compare_repro))
print_msg("Compare ref free class avgs : %s\n" % (compare_ref_free))
print_msg("Use cutoff from ref free : %s\n" % (ref_free_cutoff))
if protos:
print_msg("Protofilament numbers : %s\n" % (proto))
print_msg("Using helical search range : %s\n" % hsearch)
if findseam is True:
print_msg("Using seam-based reconstruction\n")
if hpars != "-1":
print_msg("Using hpars : %s\n" % hpars)
if vertstep != None:
print_msg("Using vertical step : %.2f\n" % vertstep)
if save_half is True:
print_msg("Saving even/odd halves\n")
for i in xrange(100):
print_msg("*")
print_msg("\n\n")
if maskfile:
if type(maskfile) is types.StringType: mask3D = get_image(maskfile)
else: mask3D = maskfile
else: mask3D = model_circle(last_ring, nx, nx, nx)
numr = Numrinit(first_ring, last_ring, rstep, "F")
mask2D = model_circle(last_ring, nx, nx) - model_circle(first_ring, nx, nx)
fscmask = model_circle(last_ring, nx, nx, nx)
if CTF:
from filter import filt_ctf
from reconstruction_rjh import rec3D_MPI_noCTF
if myid == main_node:
active = EMUtil.get_all_attributes(stack, 'active')
list_of_particles = []
for im in xrange(len(active)):
if active[im]: list_of_particles.append(im)
del active
nima = len(list_of_particles)
else:
nima = 0
total_nima = bcast_number_to_all(nima, source_node=main_node)
if myid != main_node:
list_of_particles = [-1] * total_nima
list_of_particles = bcast_list_to_all(list_of_particles,
source_node=main_node)
image_start, image_end = MPI_start_end(total_nima, number_of_proc, myid)
# create a list of images for each node
list_of_particles = list_of_particles[image_start:image_end]
nima = len(list_of_particles)
if debug:
finfo.write("image_start, image_end: %d %d\n" %
(image_start, image_end))
finfo.flush()
data = EMData.read_images(stack, list_of_particles)
t_zero = Transform({
"type": "spider",
"phi": 0,
"theta": 0,
"psi": 0,
"tx": 0,
"ty": 0
})
transmulti = [[t_zero for i in xrange(nrefs)] for j in xrange(nima)]
for iref, im in ((iref, im) for iref in xrange(nrefs)
for im in xrange(nima)):
if nrefs == 1:
transmulti[im][iref] = data[im].get_attr("xform.projection")
else:
# if multi models, keep track of eulers for all models
try:
transmulti[im][iref] = data[im].get_attr("eulers_txty.%i" %
iref)
except:
data[im].set_attr("eulers_txty.%i" % iref, t_zero)
scoremulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)]
pixelmulti = [[0.0 for i in xrange(nrefs)] for j in xrange(nima)]
ref_res = [0.0 for x in xrange(nrefs)]
apix = data[0].get_attr('apix_x')
# for oplane parameter, create cylindrical mask
if oplane is not None and myid == main_node:
from hfunctions import createCylMask
cmaskf = os.path.join(outdir, "mask3D_cyl.mrc")
mask3D = createCylMask(data, olmask, lmask, ilmask, cmaskf)
# if finding seam of helix, create wedge masks
if findseam is True:
wedgemask = []
for pf in xrange(nrefs):
wedgemask.append(EMData())
# wedgemask option
if wcmask is not None:
wcmask = get_input_from_string(wcmask)
if len(wcmask) != 3:
print_msg(
"Error: wcmask option requires 3 values: x y radius")
sys.exit()
# determine if particles have helix info:
try:
data[0].get_attr('h_angle')
original_data = []
boxmask = True
from hfunctions import createBoxMask
except:
boxmask = False
# prepare particles
for im in xrange(nima):
data[im].set_attr('ID', list_of_particles[im])
data[im].set_attr('pix_score', int(0))
if CTF:
# only phaseflip particles, not full CTF correction
ctf_params = data[im].get_attr("ctf")
st = Util.infomask(data[im], mask2D, False)
data[im] -= st[0]
data[im] = filt_ctf(data[im], ctf_params, sign=-1, binary=1)
data[im].set_attr('ctf_applied', 1)
# for window mask:
if boxmask is True:
h_angle = data[im].get_attr("h_angle")
original_data.append(data[im].copy())
bmask = createBoxMask(nx, apix, ou, lmask, h_angle)
data[im] *= bmask
del bmask
if debug:
finfo.write('%d loaded \n' % nima)
finfo.flush()
if myid == main_node:
# initialize data for the reference preparation function
ref_data = [mask3D, max(center, 0), None, None, None, None]
# for method -1, switch off centering in user function
from time import time
# this is needed for gathering of pixel errors
disps = []
recvcount = []
disps_score = []
recvcount_score = []
for im in xrange(number_of_proc):
if (im == main_node):
disps.append(0)
disps_score.append(0)
else:
disps.append(disps[im - 1] + recvcount[im - 1])
disps_score.append(disps_score[im - 1] + recvcount_score[im - 1])
ib, ie = MPI_start_end(total_nima, number_of_proc, im)
recvcount.append(ie - ib)
recvcount_score.append((ie - ib) * nrefs)
pixer = [0.0] * nima
cs = [0.0] * 3
total_iter = 0
volodd = EMData.read_images(ref_vol, xrange(nrefs))
voleve = EMData.read_images(ref_vol, xrange(nrefs))
if restart:
# recreate initial volumes from alignments stored in header
itout = "000_00"
for iref in xrange(nrefs):
if (nrefs == 1):
modout = ""
else:
modout = "_model_%02d" % (iref)
if (sort):
group = iref
for im in xrange(nima):
imgroup = data[im].get_attr('group')
if imgroup == iref:
data[im].set_attr('xform.projection',
transmulti[im][iref])
else:
group = int(999)
for im in xrange(nima):
data[im].set_attr('xform.projection', transmulti[im][iref])
fscfile = os.path.join(outdir, "fsc_%s%s" % (itout, modout))
vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF(
data,
sym,
fscmask,
fscfile,
myid,
main_node,
index=group,
npad=recon_pad)
if myid == main_node:
if helicalrecon:
from hfunctions import processHelicalVol
vstep = None
if vertstep is not None:
vstep = (vdp[iref], vdphi[iref])
print_msg(
"Old rise and twist for model %i : %8.3f, %8.3f\n"
% (iref, dp[iref], dphi[iref]))
hvals = processHelicalVol(vol[iref], voleve[iref],
volodd[iref], iref, outdir,
itout, dp[iref], dphi[iref],
apix, hsearch, findseam, vstep,
wcmask)
(vol[iref], voleve[iref], volodd[iref], dp[iref],
dphi[iref], vdp[iref], vdphi[iref]) = hvals
print_msg(
"New rise and twist for model %i : %8.3f, %8.3f\n"
% (iref, dp[iref], dphi[iref]))
# get new FSC from symmetrized half volumes
fscc = fsc_mask(volodd[iref], voleve[iref], mask3D, rstep,
fscfile)
else:
vol[iref].write_image(
os.path.join(outdir, "vol_%s.hdf" % itout), -1)
if save_half is True:
volodd[iref].write_image(
os.path.join(outdir, "volodd_%s.hdf" % itout), -1)
voleve[iref].write_image(
os.path.join(outdir, "voleve_%s.hdf" % itout), -1)
if nmasks > 1:
# Read mask for multiplying
ref_data[0] = maskF[iref]
ref_data[2] = vol[iref]
ref_data[3] = fscc
# call user-supplied function to prepare reference image, i.e., center and filter it
vol[iref], cs, fl = ref_ali3d(ref_data)
vol[iref].write_image(
os.path.join(outdir, "volf_%s.hdf" % (itout)), -1)
if (apix == 1):
res_msg = "Models filtered at spatial frequency of:\t"
res = fl
else:
res_msg = "Models filtered at resolution of: \t"
res = apix / fl
ares = array2string(array(res), precision=2)
print_msg("%s%s\n\n" % (res_msg, ares))
bcast_EMData_to_all(vol[iref], myid, main_node)
# write out headers, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
# projection matching
for N_step in xrange(lstp):
terminate = 0
Iter = -1
while (Iter < max_iter - 1 and terminate == 0):
Iter += 1
total_iter += 1
itout = "%03g_%02d" % (delta[N_step], Iter)
if myid == main_node:
print_msg(
"ITERATION #%3d, inner iteration #%3d\nDelta = %4.1f, an = %5.2f, xrange = %5.2f, yrange = %5.2f, step = %5.2f\n\n"
% (N_step, Iter, delta[N_step], an[N_step], xrng[N_step],
yrng[N_step], step[N_step]))
for iref in xrange(nrefs):
if myid == main_node: start_time = time()
volft, kb = prep_vol(vol[iref])
## constrain projections to out of plane parameter
theta1 = None
theta2 = None
if oplane is not None:
theta1 = 90 - oplane
theta2 = 90 + oplane
refrings = prepare_refrings(volft,
kb,
nx,
delta[N_step],
ref_a,
sym,
numr,
MPI=True,
phiEqpsi="Minus",
initial_theta=theta1,
delta_theta=theta2)
del volft, kb
if myid == main_node:
print_msg(
"Time to prepare projections for model %i: %s\n" %
(iref, legibleTime(time() - start_time)))
start_time = time()
for im in xrange(nima):
data[im].set_attr("xform.projection", transmulti[im][iref])
if an[N_step] == -1:
t1, peak, pixer[im] = proj_ali_incore(
data[im], refrings, numr, xrng[N_step],
yrng[N_step], step[N_step], finfo)
else:
t1, peak, pixer[im] = proj_ali_incore_local(
data[im], refrings, numr, xrng[N_step],
yrng[N_step], step[N_step], an[N_step], finfo)
#data[im].set_attr("xform.projection"%iref, t1)
if nrefs > 1:
data[im].set_attr("eulers_txty.%i" % iref, t1)
scoremulti[im][iref] = peak
from pixel_error import max_3D_pixel_error
# t1 is the current param, t2 is old
t2 = transmulti[im][iref]
pixelmulti[im][iref] = max_3D_pixel_error(t1, t2, numr[-3])
transmulti[im][iref] = t1
if myid == main_node:
print_msg("Time of alignment for model %i: %s\n" %
(iref, legibleTime(time() - start_time)))
start_time = time()
# gather scoring data from all processors
from mpi import mpi_gatherv
scoremultisend = sum(scoremulti, [])
pixelmultisend = sum(pixelmulti, [])
tmp = mpi_gatherv(scoremultisend, len(scoremultisend), MPI_FLOAT,
recvcount_score, disps_score, MPI_FLOAT,
main_node, MPI_COMM_WORLD)
tmp1 = mpi_gatherv(pixelmultisend, len(pixelmultisend), MPI_FLOAT,
recvcount_score, disps_score, MPI_FLOAT,
main_node, MPI_COMM_WORLD)
tmp = mpi_bcast(tmp, (total_nima * nrefs), MPI_FLOAT, 0,
MPI_COMM_WORLD)
tmp1 = mpi_bcast(tmp1, (total_nima * nrefs), MPI_FLOAT, 0,
MPI_COMM_WORLD)
tmp = map(float, tmp)
tmp1 = map(float, tmp1)
score = array(tmp).reshape(-1, nrefs)
pixelerror = array(tmp1).reshape(-1, nrefs)
score_local = array(scoremulti)
mean_score = score.mean(axis=0)
std_score = score.std(axis=0)
cut = mean_score - (cutoff * std_score)
cut2 = mean_score + (cutoff * std_score)
res_max = score_local.argmax(axis=1)
minus_cc = [0.0 for x in xrange(nrefs)]
minus_pix = [0.0 for x in xrange(nrefs)]
minus_ref = [0.0 for x in xrange(nrefs)]
#output pixel errors
if (myid == main_node):
from statistics import hist_list
lhist = 20
pixmin = pixelerror.min(axis=1)
region, histo = hist_list(pixmin, lhist)
if (region[0] < 0.0): region[0] = 0.0
print_msg(
"Histogram of pixel errors\n ERROR number of particles\n"
)
for lhx in xrange(lhist):
print_msg(" %10.3f %7d\n" % (region[lhx], histo[lhx]))
# Terminate if 95% within 1 pixel error
im = 0
for lhx in xrange(lhist):
if (region[lhx] > 1.0): break
im += histo[lhx]
print_msg("Percent of particles with pixel error < 1: %f\n\n" %
(im / float(total_nima) * 100))
term_cond = float(term) / 100
if (im / float(total_nima) > term_cond):
terminate = 1
print_msg("Terminating internal loop\n")
del region, histo
terminate = mpi_bcast(terminate, 1, MPI_INT, 0, MPI_COMM_WORLD)
terminate = int(terminate[0])
for im in xrange(nima):
if (sort == False):
data[im].set_attr('group', 999)
elif (mjump[N_step] == 1):
data[im].set_attr('group', int(res_max[im]))
pix_run = data[im].get_attr('pix_score')
if (pix_cutoff[N_step] == 1
and (terminate == 1 or Iter == max_iter - 1)):
if (pixelmulti[im][int(res_max[im])] > 1):
data[im].set_attr('pix_score', int(777))
if (score_local[im][int(res_max[im])] < cut[int(
res_max[im])]) or (two_tail and score_local[im][int(
res_max[im])] > cut2[int(res_max[im])]):
data[im].set_attr('group', int(888))
minus_cc[int(res_max[im])] = minus_cc[int(res_max[im])] + 1
if (pix_run == 777):
data[im].set_attr('group', int(777))
minus_pix[int(
res_max[im])] = minus_pix[int(res_max[im])] + 1
if (compare_ref_free != "-1") and (ref_free_cutoff[N_step] !=
-1) and (total_iter > 1):
id = data[im].get_attr('ID')
if id in rejects:
data[im].set_attr('group', int(666))
minus_ref[int(
res_max[im])] = minus_ref[int(res_max[im])] + 1
minus_cc_tot = mpi_reduce(minus_cc, nrefs, MPI_FLOAT, MPI_SUM, 0,
MPI_COMM_WORLD)
minus_pix_tot = mpi_reduce(minus_pix, nrefs, MPI_FLOAT, MPI_SUM, 0,
MPI_COMM_WORLD)
minus_ref_tot = mpi_reduce(minus_ref, nrefs, MPI_FLOAT, MPI_SUM, 0,
MPI_COMM_WORLD)
if (myid == main_node):
if (sort):
tot_max = score.argmax(axis=1)
res = bincount(tot_max)
else:
res = ones(nrefs) * total_nima
print_msg("Particle distribution: \t\t%s\n" % (res * 1.0))
afcut1 = res - minus_cc_tot
afcut2 = afcut1 - minus_pix_tot
afcut3 = afcut2 - minus_ref_tot
print_msg("Particle distribution after cc cutoff:\t\t%s\n" %
(afcut1))
print_msg("Particle distribution after pix cutoff:\t\t%s\n" %
(afcut2))
print_msg("Particle distribution after ref cutoff:\t\t%s\n\n" %
(afcut3))
res = [0.0 for i in xrange(nrefs)]
for iref in xrange(nrefs):
if (center == -1):
from utilities import estimate_3D_center_MPI, rotate_3D_shift
dummy = EMData()
cs[0], cs[1], cs[2], dummy, dummy = estimate_3D_center_MPI(
data, total_nima, myid, number_of_proc, main_node)
cs = mpi_bcast(cs, 3, MPI_FLOAT, main_node, MPI_COMM_WORLD)
cs = [-float(cs[0]), -float(cs[1]), -float(cs[2])]
rotate_3D_shift(data, cs)
if (sort):
group = iref
for im in xrange(nima):
imgroup = data[im].get_attr('group')
if imgroup == iref:
data[im].set_attr('xform.projection',
transmulti[im][iref])
else:
group = int(999)
for im in xrange(nima):
data[im].set_attr('xform.projection',
transmulti[im][iref])
if (nrefs == 1):
modout = ""
else:
modout = "_model_%02d" % (iref)
fscfile = os.path.join(outdir, "fsc_%s%s" % (itout, modout))
vol[iref], fscc, volodd[iref], voleve[iref] = rec3D_MPI_noCTF(
data,
sym,
fscmask,
fscfile,
myid,
main_node,
index=group,
npad=recon_pad)
if myid == main_node:
print_msg("3D reconstruction time for model %i: %s\n" %
(iref, legibleTime(time() - start_time)))
start_time = time()
# Compute Fourier variance
if fourvar:
outvar = os.path.join(outdir, "volVar_%s.hdf" % (itout))
ssnr_file = os.path.join(outdir, "ssnr_%s" % (itout))
varf = varf3d_MPI(data,
ssnr_text_file=ssnr_file,
mask2D=None,
reference_structure=vol[iref],
ou=last_ring,
rw=1.0,
npad=1,
CTF=None,
sign=1,
sym=sym,
myid=myid)
if myid == main_node:
print_msg(
"Time to calculate 3D Fourier variance for model %i: %s\n"
% (iref, legibleTime(time() - start_time)))
start_time = time()
varf = 1.0 / varf
varf.write_image(outvar, -1)
else:
varf = None
if myid == main_node:
if helicalrecon:
from hfunctions import processHelicalVol
vstep = None
if vertstep is not None:
vstep = (vdp[iref], vdphi[iref])
print_msg(
"Old rise and twist for model %i : %8.3f, %8.3f\n"
% (iref, dp[iref], dphi[iref]))
hvals = processHelicalVol(vol[iref], voleve[iref],
volodd[iref], iref, outdir,
itout, dp[iref], dphi[iref],
apix, hsearch, findseam,
vstep, wcmask)
(vol[iref], voleve[iref], volodd[iref], dp[iref],
dphi[iref], vdp[iref], vdphi[iref]) = hvals
print_msg(
"New rise and twist for model %i : %8.3f, %8.3f\n"
% (iref, dp[iref], dphi[iref]))
# get new FSC from symmetrized half volumes
fscc = fsc_mask(volodd[iref], voleve[iref], mask3D,
rstep, fscfile)
print_msg(
"Time to search and apply helical symmetry for model %i: %s\n\n"
% (iref, legibleTime(time() - start_time)))
start_time = time()
else:
vol[iref].write_image(
os.path.join(outdir, "vol_%s.hdf" % (itout)), -1)
if save_half is True:
volodd[iref].write_image(
os.path.join(outdir, "volodd_%s.hdf" % (itout)),
-1)
voleve[iref].write_image(
os.path.join(outdir, "voleve_%s.hdf" % (itout)),
-1)
if nmasks > 1:
# Read mask for multiplying
ref_data[0] = maskF[iref]
ref_data[2] = vol[iref]
ref_data[3] = fscc
ref_data[4] = varf
# call user-supplied function to prepare reference image, i.e., center and filter it
vol[iref], cs, fl = ref_ali3d(ref_data)
vol[iref].write_image(
os.path.join(outdir, "volf_%s.hdf" % (itout)), -1)
if (apix == 1):
res_msg = "Models filtered at spatial frequency of:\t"
res[iref] = fl
else:
res_msg = "Models filtered at resolution of: \t"
res[iref] = apix / fl
del varf
bcast_EMData_to_all(vol[iref], myid, main_node)
if compare_ref_free != "-1": compare_repro = True
if compare_repro:
outfile_repro = comp_rep(refrings, data, itout, modout,
vol[iref], group, nima, nx, myid,
main_node, outdir)
mpi_barrier(MPI_COMM_WORLD)
if compare_ref_free != "-1":
ref_free_output = os.path.join(
outdir, "ref_free_%s%s" % (itout, modout))
rejects = compare(compare_ref_free, outfile_repro,
ref_free_output, yrng[N_step],
xrng[N_step], rstep, nx, apix,
ref_free_cutoff[N_step],
number_of_proc, myid, main_node)
# retrieve alignment params from all processors
par_str = ['xform.projection', 'ID', 'group']
if nrefs > 1:
for iref in xrange(nrefs):
par_str.append('eulers_txty.%i' % iref)
if myid == main_node:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, image_start,
image_end, number_of_proc)
else:
send_attr_dict(main_node, data, par_str, image_start,
image_end)
if myid == main_node:
ares = array2string(array(res), precision=2)
print_msg("%s%s\n\n" % (res_msg, ares))
dummy = EMData()
if full_output:
nimat = EMUtil.get_image_count(stack)
output_file = os.path.join(outdir, "paramout_%s" % itout)
foutput = open(output_file, 'w')
for im in xrange(nimat):
# save the parameters for each of the models
outstring = ""
dummy.read_image(stack, im, True)
param3d = dummy.get_attr('xform.projection')
g = dummy.get_attr("group")
# retrieve alignments in EMAN-format
pE = param3d.get_params('eman')
outstring += "%f\t%f\t%f\t%f\t%f\t%i\n" % (
pE["az"], pE["alt"], pE["phi"], pE["tx"], pE["ty"],
g)
foutput.write(outstring)
foutput.close()
del dummy
mpi_barrier(MPI_COMM_WORLD)
# mpi_finalize()
if myid == main_node: print_end_msg("ali3d_MPI")
def proj_ali_incore_local(data,
refrings,
numr,
xrng,
yrng,
step,
an,
finfo=None):
from utilities import compose_transform2
#from utilities import set_params_proj, get_params_proj
from math import cos, sin, pi
ID = data.get_attr("ID")
mode = "F"
nx = data.get_xsize()
ny = data.get_ysize()
# center is in SPIDER convention
cnx = nx // 2 + 1
cny = ny // 2 + 1
ant = cos(an * pi / 180.0)
#phi, theta, psi, sxo, syo = get_params_proj(data)
t1 = data.get_attr("xform.projection")
dp = t1.get_params("spider")
# get translations from data
tx = dp["tx"]
ty = dp["ty"]
if finfo:
finfo.write("Image id: %6d\n" % (ID))
finfo.write("Old parameters: %9.4f %9.4f %9.4f %9.4f %9.4f\n" %
(dp["phi"], dp["theta"], dp["psi"], -tx, -ty))
finfo.flush()
[ang, sxs, sys, mirror, iref,
peak] = Util.multiref_polar_ali_2d_local(data, refrings, xrng, yrng, step,
ant, mode, numr, cnx + tx,
cny + ty)
iref = int(iref)
data.set_attr("assign", iref)
if iref > -1:
# The ormqip returns parameters such that the transformation is applied first, the mirror operation second.
# What that means is that one has to change the the Eulerian angles so they point into mirrored direction: phi+180, 180-theta, 180-psi
angb, sxb, syb, ct = compose_transform2(0.0, sxs, sys, 1, -ang, 0.0,
0.0, 1)
if mirror:
phi = (refrings[iref].get_attr("phi") + 540.0) % 360.0
theta = 180.0 - refrings[iref].get_attr("theta")
psi = (540.0 - refrings[iref].get_attr("psi") + angb) % 360.0
s2x = sxb - tx
s2y = syb - ty
else:
phi = refrings[iref].get_attr("phi")
theta = refrings[iref].get_attr("theta")
psi = (refrings[iref].get_attr("psi") + angb + 360.0) % 360.0
s2x = sxb - tx
s2y = syb - ty
t2 = Transform({
"type": "spider",
"phi": phi,
"theta": theta,
"psi": psi
})
t2.set_trans(Vec2f(-s2x, -s2y))
from pixel_error import max_3D_pixel_error
pixel_error = max_3D_pixel_error(t1, t2, numr[-3])
if finfo:
finfo.write(
"New parameters: %9.4f %9.4f %9.4f %9.4f %9.4f %10.5f %11.3e\n\n"
% (phi, theta, psi, s2x, s2y, peak, pixel_error))
finfo.flush()
return t2, peak, pixel_error
else:
return -1.0e23, 0.0
def main():
arglist = []
for arg in sys.argv: arglist.append( arg )
progname = os.path.basename(arglist[0])
usage = progname + " stack outdir --phase=1 --ou=outer_radius|sxconsistency.py --phase=3 newlocal/main000 --ou=133 --thresherr=3.0 --params=paramsa outgrouparms"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--phase", type= "int", default= 1, help="Phase =1 prepares resampled stacks, =2 analyzes consistency of orientation parameters")
parser.add_option("--ou", type= "int", default= -1, help="outer radius for calculation of pixel error")
parser.add_option("--sym", type="string", default= "c1", help="symmetry of the refined structure")
parser.add_option("--thresherr", type="float", default=1.0, help="Threshold for accpetable orientation errors (in pixels)")
parser.add_option("--ndigits", type="int", default=1, help="Accuracy for checking whether parameters are identical")
parser.add_option("--chunk", type="string", default="", help="Root of of four chunk files with indeces")
parser.add_option("--params", type="string", default="", help="Root of of six parameter file names with refinement results")
(options, args) = parser.parse_args(arglist[1:])
global_def.BATCH = True
if options.phase == 1 and len(args) == 2:
inputbdb = args[0]
outdir = args[1]
nn = EMUtil.get_image_count(inputbdb)
t = range(nn)
shuffle(t)
chunks = []
for i in xrange(4):
# I use the MPI function here just to easily get the balanced load
j,k = MPI_start_end(nn, 4, i)
chunks.append(t[j:k])
chunks[i].sort()
write_text_file(chunks[i],os.path.join(outdir,'chunk%01d.txt'%i))
del t
write_text_file([ len(chunks[i]) for i in xrange(4) ],os.path.join(outdir,'chunklengths.txt'))
"""
pt = [[None]]*6
ll=0
for i in xrange(3):
for j in xrange(i+1,4):
pt[ll] = chunks[i]+chunks[j]
ll+=1
for i in xrange(6):
listfile = os.path.join(outdir,'lili%01d.txt'%i)
write_text_file(pt[i],listfile)
outbdb = "bdb:"+ os.path.join(outdir,"X%01d"%i)
cmd = '{} {} {} {}'.format('e2bdb.py', inputbdb, '--makevstack='+outbdb, '--list='+listfile)
subprocess.call(cmd, shell=True)
"""
# Run 6 programs
elif options.phase == 2 and len(args) == 2:
outdir = args[0]
howmanythesame = args[1]
ndigits = options.ndigits #for chc5 1, for ribo 4#4.0
prms = []
for i in xrange(6):
prms.append( read_text_row(os.path.join(outdir,options.params+"%01d.txt"%i)) )
for j in xrange(len(prms[-1])):
for k in xrange(5):
prms[-1][j][k] = round(prms[-1][j][k], ndigits)
nn = 2*len(prms[0])
n4 = nn//4
qt=[[[-1.0]]*nn for i in xrange(6)]
ll=0
for i in xrange(3):
for j in xrange(i+1,4):
qt[ll][i*n4:(i+1)*n4]=prms[ll][:n4]
qt[ll][j*n4:(j+1)*n4]=prms[ll][n4:]
ll+=1
thesame = 0
for ll in xrange(len(qt[0])):
rw = []
for j in xrange(6):
if(len(qt[j][ll]) > 1): rw.append(qt[j][ll])
isame = True
for j in xrange(3):
if(rw[0][j] != rw[1][j]):
isame = False
#print ll,rw[0][j], rw[1][j]
break
if(rw[0][j] != rw[2][j]):
isame = False
#print ll,rw[0][j], rw[2][j]
break
if isame: thesame += 1
qt = float(thesame)/nn
print "Proportion of the same orientations ",qt
write_text_file([qt], os.path.join(outdir,howmanythesame) )
######### PHASE 3
elif options.phase == 3 and len(args) == 2:
outdir = args[0]
outgrouparms= args[1]
radius = options.ou
thresherr = options.thresherr #for chc5 1, for ribo 4#4.0
sym = int(options.sym[1:])
qsym = 360.0/sym
blocks = ['A','B','C','D']
params = {}
ll = 0
for i in xrange(3):
for j in xrange(i+1,4):
params[chr(65+i)+chr(48+ll)]=[]
params[chr(65+j)+chr(48+ll)]=[]
ll+=1
chunks = {}
chunklengths = {}
for i in xrange(4):
chunks[chr(65+i)] = map(int,read_text_file(os.path.join(outdir,options.chunk+"%01d.txt"%i)))
chunklengths[chr(65+i)] = len(chunks[chr(65+i)])
for i in xrange(6):
prms = read_text_row(os.path.join(outdir,options.params+"%01d.txt"%i))
for q in blocks:
if params.has_key(q+chr(48+i)):
params[q+chr(48+i)] = prms[:chunklengths[q]]
del prms[:chunklengths[q]]
pairs = [["A0","A1"],["B3","B4"],["C1","C5"],["D2","D4"]]
lefts = ["A2","B0","C3","D5"]
# Compute average projection params and pixel errors
avgtrans = {}
pixer = {}
for i,q in enumerate(pairs):
avgtrans[q[0][0]] = [0.0]*chunklengths[q[0][0]]
pixer[q[0][0]] = [0.0]*chunklengths[q[0][0]]
for j in xrange(chunklengths[q[0][0]]):
fifi = [ params[q[0]][j], params[q[1]][j] ]
nas = [0.0,0.0,0.0]
if( sym == 1):
pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius)
for i in xrange(2):
n1,n2,n3 = getfvec(fifi[i][0],fifi[i][1])
nas[0] += n1
nas[1] += n2
nas[2] += n3
else:
m1,m2,m3 = getfvec(fifi[0][0],fifi[0][1])
nas[0] = m1
nas[1] = m2
nas[2] = m3
#if(k == 2):
# print "XXXX"
# print fifi[0],nas
for i in xrange(1,2):
qnom = -1.e10
for j in xrange(-1,2,1):
t1,t2,t3 = getfvec(fifi[i][0]+j*qsym,fifi[i][1])
nom = t1*m1 + t2*m2 + t3*m3
if(nom > qnom):
qnom = nom
n1=t1
n2=t2
n3=t3
#if(k == 2):
# print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom
nas[0] += n1
nas[1] += n2
nas[2] += n3
print qnom, n1,n2,n3,nas
# To get the correct pixer phi angle has to be taken from the above!!
pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius)
nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2)
if(nom < 1.e-6):
nphi = 0.0
ntheta = 0.0
else:
ntheta = degrees(acos(nas[2]/nom))%360.0
if(sym>1 and ntheta>90.0): nphi = (degrees(atan2( nas[1], nas[0] ))-180.0)%qsym + 180.0
else: nphi = degrees(atan2( nas[1], nas[0] ))%qsym
#print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta)
twod = average2dtransform([fifi[0][2:],fifi[1][2:]])
avgtrans[q[0][0]][j] = [nphi, ntheta, twod[0], twod[1], twod[2]]
perr = {}
tgood = 0
for q in blocks:
perr[q] = [True]*chunklengths[q]
for k in xrange(chunklengths[q]):
if(pixer[q][k] > thresherr): perr[q][k] = False
if perr[q][k]: tgood += 1
if(tgood < 4):
print " No good images within the pixel error threshold specified"
exit()
print " tgood ", tgood
hi = hist_list([pixer[q][k] for q in blocks for k in xrange(chunklengths[q]) ],16)
for i in xrange(len(hi[0])):
print "%4d %12.3f %12.0f "%(i,hi[0][i],hi[1][i])
# Finished, store average orientation params and table of good images
# store lists of good images for each group
# blocks is indexed by first letter
good = [[] for i in xrange(4)]
bad = [[] for i in xrange(4)]
for i,q in enumerate(blocks):
for k in xrange(chunklengths[q]):
#deprt = max_3D_pixel_error(params[lefts[i]][k],avgtrans[q][k],r=radius)
if perr[q][k]:
good[i].append(chunks[q][k])
#[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k] ])
else:
bad[i].append(chunks[q][k])
#[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k]])
write_text_file( good[i], os.path.join(outdir,"newgood%01d.txt"%i) )
write_text_file( bad[i], os.path.join(outdir,"newbad%01d.txt"%i) )
# write out parameters, for those in pairs write out average, for leftouts leave them as they were
# These parameters refer to the original X files.
ll = 0
for m in xrange(6):
if params.has_key(q+chr(48+m)):
prmsgood = []
prmsbad = []
try:
j = lefts.index(q+chr(48+m))
for k in xrange(chunklengths[q]):
if perr[q][k]:
prmsgood.append(params[q+chr(48+m)][k])
else:
prmsbad.append(params[q+chr(48+m)][k])
except:
for k in xrange(chunklengths[q]):
if perr[q][k]:
prmsgood.append(avgtrans[q][k])
else:
prmsbad.append(avgtrans[q][k])
write_text_row(prmsgood, os.path.join(outdir,"params-newgood%01d%01d.txt"%(i,ll)))
write_text_row(prmsbad, os.path.join(outdir, "params-newbad%01d%01d.txt"%(i,ll)))
ll += 1
# Generate newlili files from newgood, these contain original numbering of the total single file
ll = 0
for i in xrange(3):
for j in xrange(i+1,4):
write_text_file( good[i] + good[j], os.path.join(outdir,"newlili%01d.txt"%ll) )
ll += 1
# write out parameters, for those in pairs write out average, for leftouts leave them as they were
# These parameters refer to the original X files.
for i in xrange(6):
prms = []
for q in blocks:
if params.has_key(q+chr(48+i)):
try:
j = lefts.index(q+chr(48+i))
prms += params[q+chr(48+i)]
except:
prms += avgtrans[q]
write_text_row(prms, os.path.join(outdir,outgrouparms+"%01d.txt"%i))
# Write chunklengths
chunklengths = [len(good[i]) for i in xrange(4)]
write_text_file(chunklengths, os.path.join(outdir,"chunklengths.txt") )
"""
# We do not use consecutive numbering anymore
# Generate newx files from newgood, these contain consecutive (with gaps) numbering that allows to generate truncated X files from the previous X files
ll = 0
for i in xrange(3):
firstblock = []
l = len(perr[blocks[i]])
for k in xrange(l):
if perr[blocks[i]][k]:
firstblock.append(k)
for j in xrange(i+1,4):
secondblock = []
for k in xrange(len(perr[blocks[j]])):
if perr[blocks[j]][k]:
secondblock.append(k+l)
write_text_file( firstblock + secondblock, os.path.join(outdir,"goodX%01d.txt"%ll) )
ll += 1
del good,bad,firstblock,secondblock,perr
"""
"""
0 1 2 3 4 5
A A A = = =
B = = B B =
= C = C = C
= = D = D D
"""
elif options.phase == 4 and len(args) == 1:
outdir = args[0]
bp = 'badparams'
#outgrouparms= args[1]
radius = options.ou
thresherr = options.thresherr
sym = int(options.sym[1:])
qsym = 360.0/sym
#params = [[None for i in xrange(3)] for j in xrange(4)]
ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general.
for jj in xrange(4):
params = [None for ii in xrange(ll)]
newbad = map(int, read_text_file(options.params+"%01d.txt"%jj) )
nn = len(newbad)
for ii in xrange(ll):
params[ii] = read_text_row(os.path.join(outdir,bp+"%01d%01d.txt"%(jj,ii)))
assert(nn == len(params[ii]) )
# Compute average projection params and pixel errors
avgtrans = [None]*nn
pixer = [0.0]*nn
for j in xrange(nn):
nas = [0.0,0.0,0.0]
if( sym == 1):
#pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius)
for i in xrange(ll):
n1,n2,n3 = getfvec(params[i][j][0],params[i][j][1])
nas[0] += n1
nas[1] += n2
nas[2] += n3
else:
m1,m2,m3 = getfvec(params[0][j][0],params[0][j][1])
nas[0] = m1
nas[1] = m2
nas[2] = m3
for i in xrange(1,ll):
qnom = -1.e10
for j in xrange(-1,2,1):
t1,t2,t3 = getfvec(params[i][j][0]+j*qsym,params[i][j][1])
nom = t1*m1 + t2*m2 + t3*m3
if(nom > qnom):
qnom = nom
n1=t1
n2=t2
n3=t3
#if(k == 2):
# print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom
nas[0] += n1
nas[1] += n2
nas[2] += n3
print qnom, n1,n2,n3,nas
# To get the correct pixer phi angle has to be taken from the above!!
nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2)
if(nom < 1.e-6):
nphi = 0.0
ntheta = 0.0
else:
ntheta = degrees(acos(nas[2]/nom))%360.0
if(sym>1 and ntheta>90.0): nphi = (degrees(atan2( nas[1], nas[0] ))-180.0)%qsym + 180.0
else: nphi = degrees(atan2( nas[1], nas[0] ))%qsym
#print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta)
twod = average2dtransform([params[ii][j][2:] for ii in xrange(ll)])
avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]]
perr = errors_per_image(params, avgtrans, thresherr, radius )
rescued = []
rejects = []
for j in xrange(nn):
#print chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)]
if( perr[j][0] <= thresherr ): rescued.append([newbad[j],j])
else: rejects.append([newbad[j],j])
if( len(rescued) == 0 ): write_text_row([-1,-1], os.path.join(outdir,"rescued%01d.txt"%jj))
else: write_text_row(rescued, os.path.join(outdir,"rescued%01d.txt"%jj))
# We also have to write params.
if( len(rescued) != 0 ):
for ii in xrange(ll): write_text_row([params[ii][rescued[k][1]] for k in xrange(len(rescued))] , os.path.join(outdir,"params-rescued%01d%01d.txt"%(jj,ii)))
if( len(rejects) == 0 ): write_text_row([-1,-1], os.path.join(outdir,'rejects'+"%01d.txt"%jj))
else: write_text_row(rejects, os.path.join(outdir,'rejects'+"%01d.txt"%jj))
if( len(rejects) != 0 ):
for ii in xrange(ll): write_text_row([params[ii][rejects[k][1]] for k in xrange(len(rejects))] , os.path.join(outdir,"params-rejects%01d%01d.txt"%(jj,ii)))
hi = hist_list([perr[j][0] for j in xrange(nn) ], 16)
print "Pixel errors for BAD GROUP ",chr(65+jj)
for ii in xrange(len(hi[0])):
print "%4d %12.3f %12.0f "%(ii,hi[0][ii],hi[1][ii])
elif options.phase == 5 and len(args) == 1:
# This version is for meridien refinement. There are simply three full sets of params.
outdir = args[0]
bp = 'badparams'
#outgrouparms= args[1]
radius = options.ou
thresherr = options.thresherr
sym = int(options.sym[1:])
qsym = 360.0/sym
#params = [[None for i in xrange(3)] for j in xrange(4)]
ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general.
for jj in xrange(1):
params = [None for ii in xrange(ll)]
#newbad = map(int, read_text_file(options.params+"%01d.txt"%jj) )
#nn = len(newbad)
for ii in xrange(ll):
params[ii] = read_text_row(os.path.join(outdir,"params%01d.txt"%(ii)))
#assert(nn == len(params[ii]) )
nn = len(params[0])
newbad = range(nn)
# Compute average projection params and pixel errors
avgtrans = [None]*nn
pixer = [0.0]*nn
for j in xrange(nn):
nas = [0.0,0.0,0.0]
if( sym == 1):
#pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius)
for i in xrange(ll):
n1,n2,n3 = getfvec(params[i][j][0],params[i][j][1])
nas[0] += n1
nas[1] += n2
nas[2] += n3
else:
m1,m2,m3 = getfvec(params[0][j][0],params[0][j][1])
nas[0] = m1
nas[1] = m2
nas[2] = m3
for i in xrange(1,ll):
qnom = -1.e10
for j in xrange(-1,2,1):
t1,t2,t3 = getfvec(params[i][j][0]+j*qsym,params[i][j][1])
nom = t1*m1 + t2*m2 + t3*m3
if(nom > qnom):
qnom = nom
n1=t1
n2=t2
n3=t3
#if(k == 2):
# print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom
nas[0] += n1
nas[1] += n2
nas[2] += n3
print qnom, n1,n2,n3,nas
# To get the correct pixer phi angle has to be taken from the above!!
nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2)
if(nom < 1.e-6):
nphi = 0.0
ntheta = 0.0
else:
ntheta = degrees(acos(nas[2]/nom))%360.0
if(sym>1 and ntheta>90.0): nphi = (degrees(atan2( nas[1], nas[0] ))-180.0)%qsym + 180.0
else: nphi = degrees(atan2( nas[1], nas[0] ))%qsym
#print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta)
twod = average2dtransform([params[ii][j][2:] for ii in xrange(ll)])
avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]]
write_text_row(avgtrans, os.path.join(outdir,"avgtrans.txt"))
perr = errors_per_image(params, avgtrans, thresherr, radius )
rescued = []
rejects = []
for j in xrange(nn):
#print chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)]
if( perr[j][0] <= thresherr ): rescued.append([newbad[j],j])
else: rejects.append([newbad[j],j])
if( len(rescued) == 0 ): write_text_row([-1,-1], os.path.join(outdir,"rescued%01d.txt"%jj))
else: write_text_row(rescued, os.path.join(outdir,"rescued%01d.txt"%jj))
# We also have to write params.
if( len(rescued) != 0 ):
for ii in xrange(ll): write_text_row([params[ii][rescued[k][1]] for k in xrange(len(rescued))] , os.path.join(outdir,"params-rescued%01d%01d.txt"%(jj,ii)))
if( len(rejects) == 0 ): write_text_row([-1,-1], os.path.join(outdir,'rejects'+"%01d.txt"%jj))
else: write_text_row(rejects, os.path.join(outdir,'rejects'+"%01d.txt"%jj))
if( len(rejects) != 0 ):
for ii in xrange(ll): write_text_row([params[ii][rejects[k][1]] for k in xrange(len(rejects))] , os.path.join(outdir,"params-rejects%01d%01d.txt"%(jj,ii)))
hi = hist_list([perr[j][0] for j in xrange(nn) ], 16)
print "Pixel errors for BAD GROUP ",chr(65+jj)
for ii in xrange(len(hi[0])):
print "%4d %12.3f %12.0f "%(ii,hi[0][ii],hi[1][ii])
else:
print "Usage: "
print """
Phase 1: sxconsistency.py --phase=1 bdb:data outdir
output files are:
in directory outdir: lili0.txt to lili5.txt contain indices of images in resampled six groups
bdb:dataX0 to bdb:dataX5 are metafiles containing six resampled groups of images derived from bdb:data
Phase 2 sxconsistency.py --phase=2 outdir --ndigits=1 --params=paramsb howmanythesame.txt
outdir - directory containing files lili0.txt to lili5.txt produced in phase 1
--params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be
master/main/params0.txt to master/main/params5.txt
output files:
howmanythesame.txt - contains one number, a ratio of number of images that did not change orientations
to the total number of images
Phase 3: sxconsistency.py --phase=3 outdir --ou=133 --thresherr=3.0 --params=paramsa outgrouparms
input files:
outdir - directory containing files lili0.txt to lili5.txt produced in phase 1
--params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be
master/main/params0.txt to master/main/params5.txt
output files:
outgrouparms*.txt - Root of of six parameters files with average 3D orientation parameters computed from six runs, can be imported into bdb:data
The next two files contain the original image numbers refering to the top bdb.
outdir/newgood.txt - Text file with indices of images whose orientation parameters arrors are below specified thresherr (to be kept)
outdir/bad.txt - Text file with indices of images whose orientation parameters arrors are above specified thresherr (to be rejected)
Phase 4: sxconsistency.py --phase=4 outdir --ou=133 --thresherr=3.0 --params=master/main001/newbad
input files:
outdir - directory containing files badparamsij.txt i= 0,3, j=0,2, which resulted from three alignments of four badchunk images
--params= - Root of four files with original indices of bad images, they will be read and a subset corresponding to rescued ones will be outputed
output files:
rescued*.txt - four files with indices of accepted images from badchunk.
rejects*.txt - four files with indices of rejected images from badchunk.
There are two columns: [number in original bdb:chunk, number in bdb:newbad]
"""
print "Please run '" + progname + " -h' for detailed options"
global_def.BATCH = False
