def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack .. average eigvol output_factcoords --rad=radius --neigvol=number_of_eigvol --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--rad", type="int", default=-1, help="radius of mask")
parser.add_option("--neigvol",
type="int",
default=-1,
help="number of eigvenvectors to use (default all)")
parser.add_option(
"--fl",
type="float",
default=0.0,
help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
parser.add_option(
"--aa",
type="float",
default=0.0,
help="fall-off of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--CTF",
action="store_true",
default=False,
help="Use CTF")
parser.add_option("--MPI", action="store_true", help="use MPI")
(options, args) = parser.parse_args()
if (len(args) < 4):
print "usage: " + usage
print "Please run '" + progname + " -h' for details"
else:
stacks = args[0:-3]
avgvol = args[-3]
eigvol = args[-2]
output = args[-1]
if options.rad < 0:
print "Error: mask radius is not given"
sys.exit(-1)
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
if options.MPI:
from mpi import mpi_init
sys.argv = mpi_init(len(sys.argv), sys.argv)
from utilities import get_im
global_def.BATCH = True
if (get_im(stacks[0]).get_zsize() == 1
and get_im(eigvol).get_zsize() > 1):
from applications import factcoords_prj
factcoords_prj(stacks, avgvol, eigvol, output, options.rad,
options.neigvol, options.fl, options.aa,
options.CTF, options.MPI)
else:
from applications import factcoords_vol
factcoords_vol(stacks, avgvol, eigvol, output, options.rad,
options.neigvol, options.fl, options.aa,
options.MPI)
global_def.BATCH = False
def spruce_up_var_m( refdata ): from utilities import print_msg from utilities import model_circle, get_im from filter import filt_tanl, filt_gaussl from morphology import threshold import os numref = refdata[0] outdir = refdata[1] fscc = refdata[2] total_iter = refdata[3] varf = refdata[4] mask = refdata[5] ali50S = refdata[6] if ali50S: mask_50S = get_im( "mask-50S.spi" ) if fscc is None: flmin = 0.4 aamin = 0.1 else: flmin,aamin,idmin=minfilt( fscc ) aamin = aamin msg = "Minimum tangent filter: cut-off frequency = %10.3f fall-off = %10.3f\n"%(fflmin, aamin) print_msg(msg) for i in xrange(numref): volf = get_im( os.path.join(outdir, "vol%04d.hdf"% total_iter) , i ) if(not (varf is None) ): volf = volf.filter_by_image( varf ) volf = filt_tanl(volf, flmin, aamin) stat = Util.infomask(volf, mask, True) volf -= stat[0] Util.mul_scalar(volf, 1.0/stat[1]) nx = volf.get_xsize() stat = Util.infomask(volf,model_circle(nx//2-2,nx,nx,nx)-model_circle(nx//2-6,nx,nx,nx), True) volf -= stat[0] Util.mul_img( volf, mask ) volf = threshold(volf) volf = filt_gaussl( volf, 0.4) if ali50S: if i==0: v50S_0 = volf.copy() v50S_0 *= mask_50S else: from applications import ali_vol_3 from fundamentals import rot_shift3D v50S_i = volf.copy() v50S_i *= mask_50S params = ali_vol_3(v50S_i, v50S_0, 10.0, 0.5, mask=mask_50S) volf = rot_shift3D( volf, params[0], params[1], params[2], params[3], params[4], params[5], 1.0) volf.write_image( os.path.join(outdir, "volf%04d.hdf"%total_iter), i )
def ref_ali3dm_new(refdata):
from utilities import print_msg
from utilities import model_circle, get_im
from filter import filt_tanl, filt_gaussl, filt_table
from morphology import threshold
from fundamentals import rops_table
from alignment import ali_nvol
from math import sqrt
import os
numref = refdata[0]
outdir = refdata[1]
fscc = refdata[2]
total_iter = refdata[3]
varf = refdata[4]
mask = refdata[5]
ali50S = refdata[6]
if fscc is None:
flmin = 0.38
aamin = 0.1
idmin = 0
else:
flmin, aamin, idmin = minfilt(fscc)
aamin /= 2.0
msg = "Minimum tangent filter derived from volume %2d: cut-off frequency = %10.3f, fall-off = %10.3f\n" % (
idmin, flmin, aamin)
print_msg(msg)
vol = []
for i in xrange(numref):
vol.append(get_im(os.path.join(outdir, "vol%04d.hdf" % total_iter), i))
stat = Util.infomask(vol[i], mask, False)
vol[i] -= stat[0]
vol[i] /= stat[1]
vol[i] *= mask
vol[i] = threshold(vol[i])
del stat
reftab = rops_table(vol[idmin])
for i in xrange(numref):
if (i != idmin):
vtab = rops_table(vol[i])
ftab = [None] * len(vtab)
for j in xrange(len(vtab)):
ftab[j] = sqrt(reftab[j] / vtab[j])
vol[i] = filt_table(vol[i], ftab)
if ali50S:
vol = ali_nvol(vol, get_im("mask-50S.spi"))
for i in xrange(numref):
if (not (varf is None)): vol[i] = vol[i].filter_by_image(varf)
filt_tanl(vol[i], flmin, aamin).write_image(
os.path.join(outdir, "volf%04d.hdf" % total_iter), i)
def ref_ali3dm_new( refdata ): from utilities import print_msg from utilities import model_circle, get_im from filter import filt_tanl, filt_gaussl, filt_table from morphology import threshold from fundamentals import rops_table from alignment import ali_nvol from math import sqrt import os numref = refdata[0] outdir = refdata[1] fscc = refdata[2] total_iter = refdata[3] varf = refdata[4] mask = refdata[5] ali50S = refdata[6] if fscc is None: flmin = 0.38 aamin = 0.1 idmin = 0 else: flmin, aamin, idmin = minfilt( fscc ) aamin /= 2.0 msg = "Minimum tangent filter derived from volume %2d: cut-off frequency = %10.3f, fall-off = %10.3f\n"%(idmin, flmin, aamin) print_msg(msg) vol = [] for i in xrange(numref): vol.append(get_im( os.path.join(outdir, "vol%04d.hdf"%total_iter), i )) stat = Util.infomask( vol[i], mask, False ) vol[i] -= stat[0] vol[i] /= stat[1] vol[i] *= mask vol[i] = threshold(vol[i]) del stat reftab = rops_table( vol[idmin] ) for i in xrange(numref): if(i != idmin): vtab = rops_table( vol[i] ) ftab = [None]*len(vtab) for j in xrange(len(vtab)): ftab[j] = sqrt( reftab[j]/vtab[j] ) vol[i] = filt_table( vol[i], ftab ) if ali50S: vol = ali_nvol(vol, get_im( "mask-50S.spi" )) for i in xrange(numref): if(not (varf is None) ): vol[i] = vol[i].filter_by_image( varf ) filt_tanl( vol[i], flmin, aamin ).write_image( os.path.join(outdir, "volf%04d.hdf" % total_iter), i )
def resample_prepare(prjfile, nvol, snr, CTF, npad):
from utilities import get_im
nx = get_im(prjfile, 0).get_xsize()
fftvols = [None] * nvol
wgtvols = [None] * nvol
rectors = [None] * nvol
for i in range(nvol):
fftvols[i] = EMData()
wgtvols[i] = EMData()
if CTF:
params = {
"size": nx,
"npad": npad,
"snr": snr,
"weight": wgtvols[i],
"fftvol": fftvols[i]
}
rectors[i] = Reconstructors.get("nn4_ctf", params)
else:
params = {
"size": nx,
"npad": npad,
"snr": snr,
"weight": wgtvols[i],
"fftvol": fftvols[i]
}
rectors[i] = Reconstructors.get("nn4", params)
rectors[i].setup()
return rectors, fftvols, wgtvols
def rot_avg_image(image_to_be_averaged): """ Rotational average Returns a 2-D or 3-D image containing a rotational average of image e """ import types from utilities import get_im if type(image_to_be_averaged) is types.StringType: image_to_be_averaged = get_im(image_to_be_averaged) return image_to_be_averaged.rotavg_i()
def rot_avg_image(image_to_be_averaged): """ Rotational average Returns a 2-D or 3-D image containing a rotational average of image e """ import types from utilities import get_im if type(image_to_be_averaged) is types.StringType: image_to_be_averaged = get_im(image_to_be_averaged) return image_to_be_averaged.rotavg_i()
def main():
import os, sys
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = progname + " volume binarymask smoothmask --variance --repair"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--variance",
type="string",
default=None,
help="variance map")
parser.add_option("--repair",
type="string",
default="repair.hdf",
help="repair map")
(options, args) = parser.parse_args(arglist[1:])
if (len(args) != 3):
print("usage: " + usage)
return None
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from utilities import get_im
from morphology import adaptive_mask, binarize
v = get_im(args[0])
m = adaptive_mask(v, 2.0, 2)
bm = binarize(m, 0.5)
bm.write_image(args[1])
adaptive_mask(v, 2.0, 2, 9, 3).write_image(args[2])
if (options.variance != None):
from fundamentals import rot_avg_image
from morphology import square_root
trovc = rot_avg_image(get_im(options.variance))
nc = trovc.get_xsize() // 2
trovc /= trovc[nc, nc, nc]
square_root(trovc).write_image(options.repair)
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack .. average eigvol output_factcoords --rad=radius --neigvol=number_of_eigvol --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--rad", type="int", default=-1, help="radius of mask")
parser.add_option("--neigvol", type="int", default=-1, help="number of eigvenvectors to use (default all)")
parser.add_option("--fl", type="float", default=0.0, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--aa", type="float", default=0.0, help="fall-off of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF")
parser.add_option("--MPI", action="store_true", help="use MPI")
(options, args) = parser.parse_args()
if( len(args) < 4 ):
print "usage: " + usage
print "Please run '" + progname + " -h' for details"
else:
stacks = args[0:-3]
avgvol = args[-3]
eigvol = args[-2]
output = args[-1]
if options.rad < 0:
print "Error: mask radius is not given"
sys.exit(-1)
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
if options.MPI:
from mpi import mpi_init
sys.argv = mpi_init(len(sys.argv), sys.argv)
from utilities import get_im
global_def.BATCH = True
if( get_im( stacks[0]).get_zsize() == 1 and get_im( eigvol).get_zsize() > 1):
from applications import factcoords_prj
factcoords_prj(stacks, avgvol, eigvol, output, options.rad, options.neigvol, options.fl, options.aa, options.CTF, options.MPI)
else:
from applications import factcoords_vol
factcoords_vol(stacks, avgvol, eigvol, output, options.rad, options.neigvol, options.fl, options.aa, options.MPI)
global_def.BATCH = False
def generate_helimic(refvol, outdir, pixel, CTF=False, Cs=2.0,voltage = 200.0, ampcont = 10.0, nonoise = False, rand_seed=14567):
from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im
from random import random
from projection import prgs, prep_vol
from filter import filt_gaussl, filt_ctf
from EMAN2 import EMAN2Ctf
if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxhelical_demo", 1)
os.mkdir(outdir)
seed(rand_seed)
Util.set_randnum_seed(rand_seed)
angles =[]
for i in xrange(3):
angles.append( [0.0+60.0*i, 90.0-i*5, 0.0, 0.0, 0.0] )
nangle = len(angles)
volfts = get_im(refvol)
nx = volfts.get_xsize()
ny = volfts.get_ysize()
nz = volfts.get_zsize()
volfts, kbx, kby, kbz = prep_vol( volfts )
iprj = 0
width = 500
xstart = 0
ystart = 0
for idef in xrange(3,6):
mic = model_blank(2048, 2048)
#defocus = idef*0.2
defocus = idef*0.6 ##@ming
if CTF :
#ctf = EMAN2Ctf()
#ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
from utilities import generate_ctf
ctf = generate_ctf([defocus,2,200,1.84,0.0,ampcont,defocus*0.2,80]) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
i = idef - 4
for k in xrange(1):
psi = 90 + 10*i
proj = prgs(volfts, kbz, [angles[idef-3][0], angles[idef-3][1], psi, 0.0, 0.0], kbx, kby)
proj = Util.window(proj, 320, nz)
mic += pad(proj, 2048, 2048, 1, 0.0, 750*i, 20*i, 0)
if not nonoise: mic += model_gauss_noise(30.0,2048,2048)
if CTF :
#apply CTF
mic = filt_ctf(mic, ctf)
if not nonoise: mic += filt_gaussl(model_gauss_noise(17.5,2048,2048), 0.3)
mic.write_image("%s/mic%1d.hdf"%(outdir, idef-3),0)
def main():
import os,sys
arglist = []
for arg in sys.argv:
arglist.append( arg )
progname = os.path.basename(arglist[0])
usage = progname + " volume binarymask smoothmask --variance --repair"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--variance", type="string", default=None, help="variance map")
parser.add_option("--repair", type="string", default="repair.hdf", help="repair map")
(options, args) = parser.parse_args( arglist[1:] )
if( len(args) != 3):
print "usage: " + usage
return None
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
from utilities import get_im
from morphology import adaptive_mask, binarize
v = get_im( args[0] )
m = adaptive_mask( v , 2.0, 2)
bm = binarize( m, 0.5 )
bm.write_image( args[1] )
adaptive_mask( v , 2.0, 2, 9, 3).write_image( args[2] )
if(options.variance != None):
from fundamentals import rot_avg_image
from morphology import square_root
trovc = rot_avg_image(get_im(options.variance))
nc = trovc.get_xsize()//2
trovc /= trovc[nc,nc,nc]
square_root(trovc).write_image( options.repair )
def genbuf( prjfile, bufprefix, beg, end, CTF, npad, verbose = 0 ):
from EMAN2 import newfile_store
from utilities import get_im
from time import time
import os
if(verbose == 1): finfo=open( os.path.join(outdir, "progress.txt"), "w" )
else: finfo = None
start_time = time()
istore = newfile_store( bufprefix, npad, CTF )
for i in xrange( beg, end ):
prj = get_im( prjfile, i )
istore.add_image( prj, prj.get_attr("xform.projection") )
if( not(finfo is None) and ((i%100==99 or i==end-1))):
finfo.write( "%6d buffered, time: %10.3f\n" % (i+1, time()-start_time) )
finfo.flush()
def genbuf( prjfile, bufprefix, beg, end, CTF, npad, verbose = 0 ):
from EMAN2 import newfile_store
from utilities import get_im
from time import time
import os
if(verbose == 1): finfo=open( os.path.join(outdir, "progress.txt"), "w" )
else: finfo = None
start_time = time()
istore = newfile_store( bufprefix, npad, CTF )
for i in xrange( beg, end ):
prj = get_im( prjfile, i )
istore.add_image( prj, prj.get_attr("xform.projection") )
if( not(finfo is None) and ((i%100==99 or i==end-1))):
finfo.write( "%6d buffered, time: %10.3f\n" % (i+1, time()-start_time) )
finfo.flush()
def metamove(paramsdict, partids, partstack, outputdir, procid, myid, main_node, nproc):
# Reads from paramsdict["stack"] particles partids set parameters in partstack
# and do refinement as specified in paramsdict
#
# Will create outputdir
# Will write to outputdir output parameters: params-chunk0.txt and params-chunk1.txt
if(myid == main_node):
# Create output directory
log = Logger(BaseLogger_Files())
log.prefix = os.path.join(outputdir)
cmd = "mkdir "+log.prefix
cmdexecute(cmd)
log.prefix += "/"
else: log = None
mpi_barrier(MPI_COMM_WORLD)
ali3d_options.delta = paramsdict["delta"]
ali3d_options.center = paramsdict["center"]
ali3d_options.ts = paramsdict["ts"]
ali3d_options.xr = paramsdict["xr"]
ali3d_options.fl = paramsdict["currentres"]
ali3d_options.aa = paramsdict["aa"]
ali3d_options.maxit = paramsdict["maxit"]
ali3d_options.mask3D = paramsdict["mask3D"]
projdata = getindexdata(paramsdict["stack"], partids, partstack, myid, nproc)
if(paramsdict["delpreviousmax"]):
for i in xrange(len(projdata)):
try: projdata[i].del_attr("previousmax")
except: pass
ali3d_options.ou = paramsdict["radius"] # This is changed in ali3d_base, but the shrank value is needed in vol recons, fixt it!
if(myid == main_node):
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"METAMOVE parameters")
spaces = " "
for q in paramsdict: print(" => ",q+spaces[len(q):],": ",paramsdict[q])
print(" => partids : ",partids)
print(" => partstack : ",partstack)
# Run alignment command
params = ali3d_base(projdata, get_im(paramsdict["refvol"]), \
ali3d_options, paramsdict["shrink"], mpi_comm = MPI_COMM_WORLD, log = log, \
nsoft = paramsdict["nsoft"], saturatecrit = paramsdict["saturatecrit"] )
del log, projdata
# store params
if(myid == main_node):
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"Executed successfully: ","ali3d_base_MPI %d"%paramsdict["nsoft"]," number of images:%7d"%len(params))
write_text_row(params, os.path.join(outputdir,"params-chunk%01d.txt"%procid) )
def ref_ali3dm_ali_50S( refdata ): from filter import fit_tanh, filt_tanl from utilities import get_im from fundamentals import rot_shift3D import os numref = refdata[0] outdir = refdata[1] fscc = refdata[2] total_iter = refdata[3] varf = refdata[4] #mask_50S = get_im( "mask-50S.spi" ) flmin = 1.0 flmax = -1.0 for iref in xrange(numref): fl, aa = fit_tanh( fscc[iref] ) if (fl < flmin): flmin = fl aamin = aa if (fl > flmax): flmax = fl aamax = aa print 'iref,fl,aa: ', iref, fl, aa # filter to minimum resolution print 'flmin,aamin:', flmin, aamin for iref in xrange(numref): v = get_im(os.path.join(outdir, "vol%04d.hdf"%total_iter), iref) v = filt_tanl(v, flmin, aamin) if ali50s: from utilities import get_params3D, set_params3D, combine_params3 from applications import ali_vol_shift, ali_vol_rotate if iref==0: v50S_ref = alivol_mask_getref( v, mask_50S ) else: v = alivol_mask( v, v50S_ref, mask_50S ) if not(varf is None): print 'filtering by fourier variance' v.filter_by_image( varf ) v.write_image(os.path.join(outdir, "volf%04d.hdf"%total_iter), iref)
def ref_ali3dm_ali_50S(refdata):
from filter import fit_tanh, filt_tanl
from utilities import get_im
from fundamentals import rot_shift3D
import os
numref = refdata[0]
outdir = refdata[1]
fscc = refdata[2]
total_iter = refdata[3]
varf = refdata[4]
#mask_50S = get_im( "mask-50S.spi" )
flmin = 1.0
flmax = -1.0
for iref in xrange(numref):
fl, aa = fit_tanh(fscc[iref])
if (fl < flmin):
flmin = fl
aamin = aa
if (fl > flmax):
flmax = fl
aamax = aa
print 'iref,fl,aa: ', iref, fl, aa
# filter to minimum resolution
print 'flmin,aamin:', flmin, aamin
for iref in xrange(numref):
v = get_im(os.path.join(outdir, "vol%04d.hdf" % total_iter), iref)
v = filt_tanl(v, flmin, aamin)
if ali50s:
from utilities import get_params3D, set_params3D, combine_params3
from applications import ali_vol_shift, ali_vol_rotate
if iref == 0:
v50S_ref = alivol_mask_getref(v, mask_50S)
else:
v = alivol_mask(v, v50S_ref, mask_50S)
if not (varf is None):
print 'filtering by fourier variance'
v.filter_by_image(varf)
v.write_image(os.path.join(outdir, "volf%04d.hdf" % total_iter), iref)
def cml_export_struc(stack, outdir, irun, Ori): from projection import plot_angles from utilities import set_params_proj, get_im global g_n_prj pagls = [] for i in xrange(g_n_prj): data = get_im(stack, i) p = [Ori[4*i], Ori[4*i+1], Ori[4*i+2], 0.0, 0.0] set_params_proj(data, p) data.write_image(outdir + '/structure_%03i.hdf' % irun, i) # prepare angles to plot pagls.append([Ori[4*i], Ori[4*i+1], Ori[4*i+2]]) # plot angles im = plot_angles(pagls) im.write_image(outdir + '/plot_agls_%03i.hdf' % irun)
def ref_ali3dm( refdata ): from filter import fit_tanh, filt_tanl from utilities import get_im from fundamentals import rot_shift3D import os numref = refdata[0] outdir = refdata[1] fscc = refdata[2] total_iter = refdata[3] #varf = refdata[4] mask = refdata[5] print 'filter every volume at (0.4, 0.1)' for iref in xrange(numref): v = get_im(os.path.join(outdir, "vol%04d.hdf"%total_iter), iref) v = filt_tanl(v, 0.4, 0.1) v *= mask v.write_image(os.path.join(outdir, "volf%04d.hdf"%total_iter), iref)
def resample_prepare( prjfile, nvol, snr, CTF, npad ):
from utilities import get_im
nx = get_im( prjfile, 0 ).get_xsize()
fftvols = [None]*nvol
wgtvols = [None]*nvol
rectors = [None]*nvol
for i in xrange(nvol):
fftvols[i] = EMData()
wgtvols[i] = EMData()
if CTF:
params = {"size":nx, "npad":npad, "snr":snr, "weight":wgtvols[i], "fftvol":fftvols[i]}
rectors[i] = Reconstructors.get( "nn4_ctf", params )
else:
params = {"size":nx, "npad":npad, "snr":snr, "weight":wgtvols[i], "fftvol":fftvols[i]}
rectors[i] = Reconstructors.get( "nn4", params )
rectors[i].setup()
return rectors, fftvols, wgtvols
def ref_ali3dm(refdata):
from filter import fit_tanh, filt_tanl
from utilities import get_im
from fundamentals import rot_shift3D
import os
numref = refdata[0]
outdir = refdata[1]
fscc = refdata[2]
total_iter = refdata[3]
#varf = refdata[4]
mask = refdata[5]
print 'filter every volume at (0.4, 0.1)'
for iref in xrange(numref):
v = get_im(os.path.join(outdir, "vol%04d.hdf" % total_iter), iref)
v = filt_tanl(v, 0.4, 0.1)
v *= mask
v.write_image(os.path.join(outdir, "volf%04d.hdf" % total_iter), iref)
def get_resolution(vol, radi, nnxo, fscoutputdir):
# this function is single processor
# Get updated FSC curves
if(ali3d_options.mask3D is None): mask = model_circle(radi,nnxo,nnxo,nnxo)
else: mask = get_im(ali3d_options.mask3D)
nfsc = fsc(vol[0]*mask,vol[1]*mask, 1.0,os.path.join(fscoutputdir,"fsc.txt") )
currentres = 0.5
ns = len(nfsc[1])
for i in xrange(1,ns-1):
if ( (2*nfsc[1][i]/(1.0+nfsc[1][i]) ) < 0.5):
currentres = nfsc[0][i-1]
break
#print(" Current resolution ",i,currentres)
if(currentres < 0.0):
print(" Something wrong with the resolution, cannot continue")
mpi_finalize()
exit()
return currentres
def cml_export_struc(stack, outdir, irun, Ori):
from projection import plot_angles
from utilities import set_params_proj, get_im
global g_n_prj
pagls = []
for i in xrange(g_n_prj):
data = get_im(stack, i)
p = [Ori[4 * i], Ori[4 * i + 1], Ori[4 * i + 2], 0.0, 0.0]
set_params_proj(data, p)
data.write_image(outdir + '/structure_%03i.hdf' % irun, i)
# prepare angles to plot
pagls.append([Ori[4 * i], Ori[4 * i + 1], Ori[4 * i + 2]])
# plot angles
im = plot_angles(pagls)
im.write_image(outdir + '/plot_agls_%03i.hdf' % irun)
def ref_sort3d(refdata):
from filter import fit_tanh, filt_tanl
from utilities import get_im
from fundamentals import rot_shift3D
import os
numref = refdata[0]
outdir = refdata[1]
fscc = refdata[2]
total_iter = refdata[3]
#varf = refdata[4]
mask = refdata[5]
low_pass_filter = refdata[6]
import time
from time import strftime, localtime
theme='filter every volume at (%f, 0.1)'%low_pass_filter
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line+theme)
print 'filter every volume at (%f, 0.1)'%low_pass_filter
for iref in xrange(numref):
v = get_im(os.path.join(outdir, "vol%04d.hdf"%total_iter), iref)
v = filt_tanl(v, low_pass_filter, 0.1)
v *= mask
v.write_image(os.path.join(outdir, "volf%04d.hdf"%total_iter), iref)
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=0.2 --aa=0.1 --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--ave2D", type="string" , default=False, help="write to the disk a stack of 2D averages")
parser.add_option("--var2D", type="string" , default=False, help="write to the disk a stack of 2D variances")
parser.add_option("--ave3D", type="string" , default=False, help="write to the disk reconstructed 3D average")
parser.add_option("--var3D", type="string" , default=False, help="compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp", type="int" , default=10 , help="number of neighbouring projections")
parser.add_option("--no_norm", action="store_true", default=False, help="do not use normalization")
parser.add_option("--radiusvar", type="int" , default=-1 , help="radius for 3D var" )
parser.add_option("--npad", type="int" , default=2 , help="number of time to pad the original images")
parser.add_option("--sym" , type="string" , default="c1" , help="symmetry")
parser.add_option("--fl", type="float" , default=0.0 , help="stop-band frequency (Default - no filtration)")
parser.add_option("--aa", type="float" , default=0.0 , help="fall off of the filter (Default - no filtration)")
parser.add_option("--CTF", action="store_true", default=False, help="use CFT correction")
parser.add_option("--VERBOSE", action="store_true", default=False, help="Long output for debugging")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option("--VAR" , action="store_true", default=False, help="stack on input consists of 2D variances (Default False)")
parser.add_option("--decimate", type="float", default=1.0, help="image decimate rate, a number large than 1. default is 1")
parser.add_option("--window", type="int", default=0, help="reduce images to a small image size without changing pixel_size. Default value is zero.")
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
parser.add_option("--nvec", type="int" , default=0 , help="number of eigenvectors, default = 0 meaning no PCA calculated")
parser.add_option("--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)")
(options,args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD, MPI_TAG_UB
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
if options.symmetrize :
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if( number_of_proc > 1 ):
ERROR("Cannot use more than one CPU for symmetry prepration","sx3dvariability",1)
except:
pass
except:
pass
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
instack = args[0]
sym = options.sym
if( sym == "c1" ):
ERROR("Thre is no need to symmetrize stack for C1 symmetry","sx3dvariability",1)
if(instack[:4] !="bdb:"):
stack = "bdb:data"
delete_bdb(stack)
cmdexecute("sxcpy.py "+instack+" "+stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack,'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None]*na
for k in xrange(ks):
delete_bdb("bdb:Q%1d"%k)
cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
DB = db_open_dict("bdb:Q%1d"%k)
for i in xrange(na):
ut = qt[i]*ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
delete_bdb("bdb:sdata")
cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
#cmdexecute("ls EMAN2DB/sdata*")
a = get_im("bdb:sdata")
a.set_attr("variabilitysymmetry",sym)
a.write_image("bdb:sdata")
else:
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 1:
stack = args[0]
else:
print( "usage: " + usage)
print( "Please run '" + progname + " -h' for detailed options")
return 1
t0 = time()
# obsolete flags
options.MPI = True
options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid)
if options.VAR and options.SND:
ERROR("Only one of var and SND can be set!", "sx3dvariability", myid)
exit()
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid)
exit()
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
# exit()
if options.nvec > 0 :
ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
exit()
if options.nvec > 0 and options.ave3D == None:
ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", myid=myid)
exit()
import string
options.sym = options.sym.lower()
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
global_def.BATCH = True
if myid == main_node:
print_begin_msg("sx3dvariability")
print_msg("%-70s: %s\n"%("Input stack", stack))
img_per_grp = options.img_per_grp
nvec = options.nvec
radiuspca = options.radiuspca
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
if options.sym != "c1" :
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry")
if(i != options.sym):
ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", myid=myid)
except:
ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", myid=myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if((nima/i)*i != nima):
ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", myid=myid)
symbaselen = nima/i
else: symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
Tracker ={}
Tracker["nx"] =nx
Tracker["ny"] =ny
Tracker["total_stack"]=nima
if options.decimate==1.:
if options.window !=0:
nx = options.window
ny = options.window
else:
if options.window ==0:
nx = int(nx/options.decimate)
ny = int(ny/options.decimate)
else:
nx = int(options.window/options.decimate)
ny = nx
symbaselen = bcast_number_to_all(symbaselen)
if radiuspca == -1: radiuspca = nx/2-2
if myid == main_node:
print_msg("%-70s: %d\n"%("Number of projection", nima))
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR:
#varList = EMData.read_images(stack, range(img_begin, img_end))
varList = []
this_image = EMData()
for index_of_particle in xrange(img_begin,img_end):
this_image.read_image(stack,index_of_particle)
varList.append(image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in xrange(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x*10000+psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
print_msg("%-70s: %d\n"%("Number of neighboring projections", img_per_grp))
print_msg("...... Finding neighboring projections\n")
if options.VERBOSE:
print "Number of images per group: ", img_per_grp
print "Now grouping projections"
proj_angles.sort()
proj_angles_list = [0.0]*(nima*4)
if myid == main_node:
for i in xrange(nima):
proj_angles_list[i*4] = proj_angles[i][1]
proj_angles_list[i*4+1] = proj_angles[i][2]
proj_angles_list[i*4+2] = proj_angles[i][3]
proj_angles_list[i*4+3] = proj_angles[i][4]
proj_angles_list = bcast_list_to_all(proj_angles_list, myid, main_node)
proj_angles = []
for i in xrange(nima):
proj_angles.append([proj_angles_list[i*4], proj_angles_list[i*4+1], proj_angles_list[i*4+2], int(proj_angles_list[i*4+3])])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
if options.VERBOSE:
print "On node %2d, number of images needed to be read = %5d"%(myid, len(all_proj))
index = {}
for i in xrange(len(all_proj)): index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2))
print_msg("%-70s: %d\n"%("Number of groups processed on the main node", len(proj_list)))
if options.VERBOSE:
print "Grouping projections took: ", (time()-t2)/60 , "[min]"
print "Number of groups on main node: ", len(proj_list)
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
print_msg("...... calculating the stack of 2D variances \n")
if options.VERBOSE:
print "Now calculating the stack of 2D variances"
proj_params = [0.0]*(nima*5)
aveList = []
varList = []
if nvec > 0:
eigList = [[] for i in xrange(nvec)]
if options.VERBOSE: print "Begin to read images on processor %d"%(myid)
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
img = EMData()
imgdata = []
for index_of_proj in xrange(len(all_proj)):
img.read_image(stack, all_proj[index_of_proj])
dmg = image_decimate_window_xform_ctf(img,options.decimate,options.window,options.CTF)
#print dmg.get_xsize(), "init"
imgdata.append(dmg)
if options.VERBOSE:
print "Reading images on processor %d done, time = %.2f"%(myid, time()-ttt)
print "On processor %d, we got %d images"%(myid, len(imgdata))
mpi_barrier(MPI_COMM_WORLD)
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
for i in xrange(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])
grp_imgdata = []
for j in xrange(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
alpha, sx, sy, mirror = params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM-phi, 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM-phi), 0.0, 0.0, 1.0)
else:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM-phi), 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM-phi)), 0.0, 0.0, 1.0)
set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
grp_imgdata.append(imgdata[mj])
#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()
if not options.no_norm:
#print grp_imgdata[j].get_xsize()
mask = model_circle(nx/2-2, nx, nx)
for k in xrange(img_per_grp):
ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
del mask
if options.fl > 0.0:
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2*nx
ny2 = 2*ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(fft( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa) ),nx,ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
for k in xrange(img_per_grp):
grp_imgdata[k] = filt_tanl( grp_imgdata[k], options.fl, options.aa)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
from utilities import pad, read_text_file
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2*nx
ny2 = 2*ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d( fft( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1) ) , nx,ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
'''
if i < 10 and myid == main_node:
for k in xrange(10):
grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
'''
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("pp.hdf", pp)
"""
ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("qq.hdf", pp)
"""
var = model_blank(nx,ny)
for q in grp_imgdata: Util.add_img2( var, q )
Util.mul_scalar( var, 1.0/(len(grp_imgdata)-1))
# Switch to std dev
var = square_root(threshold(var))
#if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a")
#else: ave, var = avgvar(grp_imgdata, mode="a")
"""
if myid == main_node:
ave.write_image("avgv.hdf",i)
var.write_image("varv.hdf",i)
"""
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
if options.VERBOSE:
print "%5.2f%% done on processor %d"%(i*100.0/len(proj_list), myid)
if nvec > 0:
eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
for k in xrange(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
del imgdata
# To this point, all averages, variances, and eigenvectors are computed
if options.ave2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(aveList)):
aveList[im].write_image(options.ave2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
"""
nm = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, MPI_TAG_UB, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol=fpol(ave, Tracker["nx"],Tracker["nx"],Tracker["nx"])
tmpvol.write_image(options.ave2D, km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node,im+myid+70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
"""
if options.ave3D:
from fundamentals import fpol
if options.VERBOSE:
print "Reconstructing 3D average volume"
ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
ave3D=fpol(ave3D,Tracker["nx"],Tracker["nx"],Tracker["nx"])
ave3D.write_image(options.ave3D)
print_msg("%-70s: %s\n"%("Writing to the disk volume reconstructed from averages as", options.ave3D))
del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList
if nvec > 0:
for k in xrange(nvec):
if options.VERBOSE:
print "Reconstruction eigenvolumes", k
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image("eig3d_%03d.hdf"%k, ITER)
Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in xrange(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
u = int(u[0])
print " Eigenvector: ",k," number changed ",int(icont[0])
else: u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node :
for im in xrange(len(varList)):
tmpvol=fpol(varList[im], Tracker["nx"], Tracker["nx"],1)
tmpvol.write_image(options.var2D, km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, MPI_TAG_UB, MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im+i+70000)
tmpvol=fpol(ave, Tracker["nx"], Tracker["nx"],1)
tmpvol.write_image(options.var2D, km)
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node, MPI_TAG_UB, MPI_COMM_WORLD)
for im in xrange(len(varList)):
send_EMData(varList[im], main_node, im+myid+70000)# What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node and options.VERBOSE:
print "Reconstructing 3D variability volume"
t6 = time()
radiusvar = options.radiusvar
if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid, varList, symmetry=options.sym, npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
res =fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
res.write_image(options.var3D)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Reconstructing 3D variability took [s]", time()-t6))
if options.VERBOSE:
print "Reconstruction took: %.2f [min]"%((time()-t6)/60)
if myid == main_node:
print_msg("%-70s: %.2f\n"%("Total time for these computations [s]", time()-t0))
if options.VERBOSE:
print "Total time for these computations: %.2f [min]"%((time()-t0)/60)
print_end_msg("sx3dvariability")
global_def.BATCH = False
from mpi import mpi_finalize
mpi_finalize()
def main():
from utilities import write_text_row, drop_image, model_gauss_noise, get_im, set_params_proj, wrap_mpi_bcast, model_circle
import user_functions
from applications import MPI_start_end
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from multi_shc import multi_shc, do_volume
from logger import Logger, BaseLogger_Files
import sys
import os
import time
import socket
progname = os.path.basename(sys.argv[0])
usage = progname + " stack [output_directory] initial_volume --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --center=center_type --fl --aa --ref_a=S --sym=c1"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--ir", type= "int", default= 1, help="inner radius for rotational correlation > 0 (set to 1)")
parser.add_option("--ou", type= "int", default= -1, help="outer radius for rotational correlation < int(nx/2)-1 (set to the radius of the particle)")
parser.add_option("--rs", type= "int", default= 1, help="step between rings in rotational correlation >0 (set to 1)" )
parser.add_option("--xr", type="string", default= "-1", help="range for translation search in x direction, search is +/xr (default 0)")
parser.add_option("--yr", type="string", default= "-1", help="range for translation search in y direction, search is +/yr (default = same as xr)")
parser.add_option("--ts", type="string", default= "1", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional")
parser.add_option("--delta", type="string", default= "-1", help="angular step of reference projections during initialization step (default automatically selected based on radius of the structure.)")
#parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches (phi and theta)")
parser.add_option("--center", type="float", default= -1, help="-1: average shift method; 0: no centering; 1: center of gravity (default=-1)")
parser.add_option("--maxit", type="int", default= 400, help="maximum number of iterations performed for the GA part (set to 400) ")
parser.add_option("--outlier_percentile",type="float", default= 95, help="percentile above which outliers are removed every iteration")
parser.add_option("--iteration_start",type="int", default= 0, help="starting iteration for rviper, 0 means go to the most recent one (default).")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF (Default no CTF correction)")
parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data (default 1.0)")
parser.add_option("--ref_a", type="string", default= "S", help="method for generating the quasi-uniformly distributed projection directions (default S)")
parser.add_option("--sym", type="string", default= "c1", help="symmetry of the refined structure")
parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default=2)")
parser.add_option("--startangles", action="store_true", default=False, help="Use orientation parameters in the input file header to jumpstart the procedure")
#options introduced for the do_volume function
parser.add_option("--fl", type="float", default=0.12, help="cut-off frequency of hyperbolic tangent low-pass Fourier filte (default 0.12)")
parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass Fourier filter (default 0.1)")
parser.add_option("--pwreference", type="string", default="", help="text file with a reference power spectrum (default no power spectrum adjustment)")
parser.add_option("--mask3D", type="string", default=None, help="3D mask file (default a sphere WHAT RADIUS??)")
(options, args) = parser.parse_args(sys.argv[1:])
#print( " args ",args)
if( len(args) == 3):
volinit = args[2]
masterdir = args[1]
elif(len(args) == 2):
volinit = args[1]
masterdir = ""
else:
print( "usage: " + usage)
print( "Please run '" + progname + " -h' for detailed options")
return 1
orgstack = args[0]
#print( orgstack,masterdir,volinit )
# INPUT PARAMETERS
radi = options.ou
global_def.BATCH = True
ali3d_options.ir = options.ir
ali3d_options.rs = options.rs
ali3d_options.ou = options.ou
ali3d_options.xr = options.xr
ali3d_options.yr = options.yr
ali3d_options.ts = options.ts
ali3d_options.an = "-1"
ali3d_options.sym = options.sym
ali3d_options.delta = options.delta
ali3d_options.npad = options.npad
ali3d_options.center = options.center
ali3d_options.CTF = options.CTF
ali3d_options.ref_a = options.ref_a
ali3d_options.snr = options.snr
ali3d_options.mask3D = options.mask3D
ali3d_options.pwreference = options.pwreference
ali3d_options.fl = 0.4
ali3d_options.aa = 0.1
if( ali3d_options.xr == "-1" ): ali3d_options.xr = "2"
"""
print( options)
print( 'ali3d_options', ali3d_options.ir ,\
ali3d_options.rs ,\
ali3d_options.ou ,\
ali3d_options.xr ,\
ali3d_options.yr ,\
ali3d_options.ts ,\
ali3d_options.an ,\
ali3d_options.sym ,\
ali3d_options.delta ,\
ali3d_options.npad ,\
ali3d_options.center ,\
ali3d_options.CTF ,\
ali3d_options.ref_a ,\
ali3d_options.snr ,\
ali3d_options.mask3D ,\
ali3d_options.fl ,\
ali3d_options.aa \
)
#exit()
"""
mpi_init(0, [])
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
#mpi_finalize()
#exit()
nxinit = -1 #int(280*0.3*2)
nsoft = 0
mempernode = 4.0e9
# PARAMETERS OF THE PROCEDURE
# threshold error
thresherr = 0
fq = 0.11 # low-freq limit to which fuse ref volumes. Should it be estimated somehow?
# Get the pixel size, if none set to 1.0, and the original image size
if(myid == main_node):
a = get_im(orgstack)
nnxo = a.get_xsize()
if ali3d_options.CTF:
i = a.get_attr('ctf')
pixel_size = i.apix
else:
pixel_size = 1.0
del a
else:
nnxo = 0
pixel_size = 1.0
pixel_size = bcast_number_to_all(pixel_size, source_node = main_node)
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
if(radi < 1): radi = nnxo//2-2
elif((2*radi+2)>nnxo): ERROR("HERE","particle radius set too large!",1)
ali3d_options.ou = radi
if(nxinit < 0): nxinit = min(32, nnxo)
nxshrink = nxinit
minshrink = 32.0/float(nnxo)
shrink = max(float(nxshrink)/float(nnxo),minshrink)
# MASTER DIRECTORY
if(myid == main_node):
print( " masterdir ",masterdir)
if( masterdir == ""):
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "master"+timestring
li = len(masterdir)
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
keepchecking = 0
else:
li = 0
keepchecking = 1
else:
li = 0
keepchecking = 1
li = mpi_bcast(li,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
if( li > 0 ):
masterdir = mpi_bcast(masterdir,li,MPI_CHAR,main_node,MPI_COMM_WORLD)
masterdir = string.join(masterdir,"")
# create a vstack from input stack to the local stack in masterdir
# Stack name set to default
stack = "bdb:"+masterdir+"/rdata"
# Initialization of stacks
if(myid == main_node):
if keepchecking:
if(os.path.exists(os.path.join(masterdir,"EMAN2DB/rdata.bdb"))): doit = False
else: doit = True
else: doit = True
if doit:
if(orgstack[:4] == "bdb:"): cmd = "{} {} {}".format("e2bdb.py", orgstack,"--makevstack="+stack)
else: cmd = "{} {} {}".format("sxcpy.py", orgstack, stack)
cmdexecute(cmd)
cmd = "{} {}".format("sxheader.py --consecutive --params=originalid", stack)
cmdexecute(cmd)
keepchecking = False
total_stack = EMUtil.get_image_count(stack)
junk = get_im(stack)
nnxo = junk.get_xsize()
del junk
else:
total_stack = 0
nnxo = 0
total_stack = bcast_number_to_all(total_stack, source_node = main_node)
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
# INITIALIZATION
# Run exhaustive projection matching to get initial orientation parameters
# Estimate initial resolution
initdir = os.path.join(masterdir,"main000")
# make sure the initial volume is not set to zero outside of a mask, as if it is it will crach the program
if( myid == main_node and (not options.startangles)):
viv = get_im(volinit)
if(options.mask3D == None): mask33d = model_circle(radi,nnxo,nnxo,nnxo)
else: mask33d = (options.mask3D).copy()
st = Util.infomask(viv, mask33d, False)
if( st[0] == 0.0 ):
viv += (model_blank(nnxo,nnxo,nnxo,1.0) - mask33d)*model_gauss_noise(st[1]/1000.0,nnxo,nnxo,nnxo)
viv.write_image(volinit)
del mask33d, viv
doit, keepchecking = checkstep(initdir, keepchecking, myid, main_node)
if doit:
partids = os.path.join(masterdir, "ids.txt")
partstack = os.path.join(masterdir, "paramszero.txt")
xr = min(8,(nnxo - (2*radi+1))//2)
if(xr > 3): ts = "2"
else: ts = "1"
delta = int(options.delta)
if(delta <= 0.0):
delta = "%f"%round(degrees(atan(1.0/float(radi))), 2)
paramsdict = { "stack":stack,"delta":"2.0", "ts":ts, "xr":"%f"%xr, "an":"-1", "center":options.center, "maxit":1, \
"currentres":0.4, "aa":0.1, "radius":radi, "nsoft":0, "delpreviousmax":True, "shrink":1.0, "saturatecrit":1.0, \
"refvol":volinit, "mask3D":options.mask3D}
if(options.startangles):
if( myid == main_node ):
cmd = "mkdir "+initdir
cmdexecute(cmd)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"INITIALIZATION")
cmd = "{} {}".format("sxheader.py --params=xform.projection --export="+os.path.join(initdir,"params-chunk0.txt"), stack)
cmdexecute(cmd)
print(line,"Executed successfully: ","Imported initial parameters from the input stack")
else:
if( myid == main_node ):
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"INITIALIZATION")
write_text_file(range(total_stack), partids)
write_text_row([[0.0,0.0,0.0,0.0,0.0] for i in xrange(total_stack) ], partstack)
metamove(paramsdict, partids, partstack, initdir, 0, myid, main_node, nproc)
if(myid == main_node):
print(line,"Executed successfully: ","initialization ali3d_base_MPI %d"%nsoft)
# store params
partids = [None]*2
for procid in xrange(2): partids[procid] = os.path.join(initdir,"chunk%01d.txt"%procid)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(initdir,"params-chunk%01d.txt"%procid)
from random import shuffle
if(myid == main_node):
# split randomly
params = read_text_row(os.path.join(initdir,"params-chunk0.txt"))
assert(len(params) == total_stack)
ll = range(total_stack)
shuffle(ll)
l1 = ll[:total_stack//2]
l2 = ll[total_stack//2:]
del ll
l1.sort()
l2.sort()
write_text_file(l1,partids[0])
write_text_file(l2,partids[1])
write_text_row([params[i] for i in l1], partstack[0])
write_text_row([params[i] for i in l2], partstack[1])
del params, l1, l2
mpi_barrier(MPI_COMM_WORLD)
# Now parallel
vol = [None]*2
for procid in xrange(2):
projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc)
if ali3d_options.CTF: vol[procid] = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol[procid] = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if( myid == main_node):
vol[procid].write_image(os.path.join(initdir,"vol%01d.hdf"%procid) )
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Generated inivol #%01d "%procid)
if(myid == main_node):
currentres = get_resolution(vol, radi, nnxo, initdir)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Initial resolution %6.4f"%currentres)
write_text_file([currentres],os.path.join(initdir,"current_resolution.txt"))
else: currentres = 0.0
currentres = bcast_number_to_all(currentres, source_node = main_node)
else:
if(myid == main_node): currentres = read_text_file(os.path.join(initdir,"current_resolution.txt"))[0]
else: currentres = 0.0
currentres = bcast_number_to_all(currentres, source_node = main_node)
# set for the first iteration
nxshrink = min(max(32, int((currentres+paramsdict["aa"]/2.)*2*nnxo + 0.5)), nnxo)
shrink = float(nxshrink)/nnxo
tracker = {"previous-resolution":currentres, "movedup":False,"eliminated-outliers":False,\
"previous-nx":nxshrink, "previous-shrink":shrink, "extension":0, "bestsolution":0}
previousoutputdir = initdir
# MAIN ITERATION
mainiteration = 0
keepgoing = 1
while(keepgoing):
mainiteration += 1
# prepare output directory
mainoutputdir = os.path.join(masterdir,"main%03d"%mainiteration)
if(myid == main_node):
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"MAIN ITERATION #",mainiteration, shrink, nxshrink)
if keepchecking:
if(os.path.exists(mainoutputdir)):
doit = 0
print("Directory ",mainoutputdir," exists!")
else:
doit = 1
keepchecking = False
else:
doit = 1
if doit:
cmd = "{} {}".format("mkdir", mainoutputdir)
cmdexecute(cmd)
# prepare names of input file names, they are in main directory,
# log subdirectories contain outputs from specific refinements
partids = [None]*2
for procid in xrange(2): partids[procid] = os.path.join(previousoutputdir,"chunk%01d.txt"%procid)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(previousoutputdir,"params-chunk%01d.txt"%procid)
mpi_barrier(MPI_COMM_WORLD)
#mpi_finalize()
#exit()
#print("RACING A ",myid)
outvol = [os.path.join(previousoutputdir,"vol%01d.hdf"%procid) for procid in xrange(2)]
for procid in xrange(2):
doit, keepchecking = checkstep(outvol[procid], keepchecking, myid, main_node)
if doit:
from multi_shc import do_volume
projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc)
if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if( myid == main_node):
vol.write_image(outvol[procid])
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Generated inivol #%01d "%procid)
del vol
if(myid == main_node):
if keepchecking:
procid = 1
if(os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid)):
doit = 0
else:
doit = 1
keepchecking = False
else:
doit = 1
if doit:
vol = [get_im(outvol[procid]) for procid in xrange(2) ]
fq = 0.11 # which part to fuse
fuselowf(vol, fq)
for procid in xrange(2): vol[procid].write_image(os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid) )
del vol
else: doit = 0
mpi_barrier(MPI_COMM_WORLD)
doit = bcast_number_to_all(doit, source_node = main_node)
# Refine two groups at a current resolution
lastring = int(shrink*radi + 0.5)
if(lastring < 2):
print( line,"ERROR!! lastring too small ", radi, shrink, lastring)
break
# REFINEMENT
# Part "a" SHC
for procid in xrange(2):
coutdir = os.path.join(mainoutputdir,"loga%01d"%procid)
doit, keepchecking = checkstep(coutdir , keepchecking, myid, main_node)
paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":1500, \
"currentres":currentres, "aa":0.1, "radius":radi, "nsoft":1, "saturatecrit":0.75, "delpreviousmax":True, "shrink":shrink, \
"refvol":os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid),"mask3D":options.mask3D }
if doit:
metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir, "loga%01d"%procid, "params-chunk%01d.txt"%procid)
for procid in xrange(2):
outvol = os.path.join(mainoutputdir,"loga%01d"%procid,"shcvol%01d.hdf"%procid)
doit, keepchecking = checkstep(outvol, keepchecking, myid, main_node)
if doit:
from multi_shc import do_volume
projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc)
if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if( myid == main_node):
vol.write_image(outvol)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Generated shcvol #%01d "%procid)
del vol
if(myid == main_node):
if keepchecking:
procid = 1
if(os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid) ):
doit = 0
else:
doit = 1
keepchecking = False
else:
doit = 1
if doit:
vol = []
for procid in xrange(2): vol.append(get_im(os.path.join(mainoutputdir,"loga%01d"%procid,"shcvol%01d.hdf"%procid) ))
fq = 0.11 # which part to fuse
fuselowf(vol, fq)
for procid in xrange(2): vol[procid].write_image( os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid) )
del vol
else: doit = 0
mpi_barrier(MPI_COMM_WORLD)
doit = bcast_number_to_all(doit, source_node = main_node)
# Part "b" deterministic
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir,"loga%01d"%procid,"params-chunk%01d.txt"%procid)
for procid in xrange(2):
coutdir = os.path.join(mainoutputdir,"logb%01d"%procid)
doit, keepchecking = checkstep(coutdir, keepchecking, myid, main_node)
# Run exhaustive to finish up matching
paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":10, \
"currentres":currentres, "aa":0.1, "radius":radi, "nsoft":0, "saturatecrit":0.95, "delpreviousmax":True, "shrink":shrink, \
"refvol":os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid), "mask3D":options.mask3D }
if doit:
metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir,"logb%01d"%procid,"params-chunk%01d.txt"%procid)
# Compute current resolution, store result in main directory
doit, keepchecking = checkstep(os.path.join(mainoutputdir,"current_resolution.txt"), keepchecking, myid, main_node)
newres = 0.0
if doit:
newres = compute_resolution(stack, mainoutputdir, partids, partstack, radi, nnxo, ali3d_options.CTF, myid, main_node, nproc)
else:
if(myid == main_node): newres = read_text_file( os.path.join(mainoutputdir,"current_resolution.txt") )[0]
newres = bcast_number_to_all(newres, source_node = main_node)
# Here I have code to generate presentable results. IDs and params have to be merged and stored and an overall volume computed.
doit, keepchecking = checkstep(os.path.join(mainoutputdir,"volf.hdf"), keepchecking, myid, main_node)
if doit:
if( myid == main_node ):
pinids = map(int, read_text_file(partids[0]) ) + map(int, read_text_file(partids[1]) )
params = read_text_row(partstack[0]) + read_text_row(partstack[1])
assert(len(pinids) == len(params))
for i in xrange(len(pinids)):
pinids[i] = [ pinids[i], params[i] ]
del params
pinids.sort()
write_text_file([pinids[i][0] for i in xrange(len(pinids))], os.path.join(mainoutputdir,"indexes.txt"))
write_text_row( [pinids[i][1] for i in xrange(len(pinids))], os.path.join(mainoutputdir,"params.txt"))
mpi_barrier(MPI_COMM_WORLD)
ali3d_options.fl = newres
ali3d_options.ou = radi
projdata = getindexdata(stack, os.path.join(mainoutputdir,"indexes.txt"), os.path.join(mainoutputdir,"params.txt"), myid, nproc)
volf = do_volume(projdata, ali3d_options, mainiteration, mpi_comm = MPI_COMM_WORLD)
if(myid == main_node): volf.write_image(os.path.join(mainoutputdir,"volf.hdf"))
mpi_barrier(MPI_COMM_WORLD)
#print("RACING X ",myid)
if(newres == currentres):
for procid in xrange(2):
coutdir = os.path.join(mainoutputdir,"logc%01d"%procid)
doit, keepchecking = checkstep(coutdir, keepchecking, myid, main_node)
if doit:
# Do cross-check of the results
paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":1, \
"currentres":newres, "aa":0.1, "radius":radi, "nsoft":0, "saturatecrit":0.95, "delpreviousmax":True, "shrink":shrink, \
"refvol":os.path.join(mainoutputdir,"vol%01d.hdf"%(1-procid)), "mask3D":options.mask3D }
# The cross-check uses parameters from step "b" to make sure shifts are correct.
# As the check is exhaustive, angles are ignored
metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc)
# identify bad apples
doit, keepchecking = checkstep(os.path.join(mainoutputdir,"badapples.txt"), keepchecking, myid, main_node)
if doit:
if(myid == main_node):
from utilities import get_symt
from pixel_error import max_3D_pixel_error
ts = get_symt(ali3d_options.sym)
badapples = []
deltaerror = 2.0
total_images_now = 0
for procid in xrange(2):
bad = []
ids = map(int,read_text_file( partids[procid] ))
total_images_now += len(ids)
oldp = read_text_row(partstack[procid])
newp = read_text_row(os.path.join(mainoutputdir,"logc%01d"%procid,"params-chunk%01d.txt"%procid))
for i in xrange(len(ids)):
t1 = Transform({"type":"spider","phi":oldp[i][0],"theta":oldp[i][1],"psi":oldp[i][2]})
t1.set_trans(Vec2f(-oldp[i][3]*shrink, -oldp[i][4]*shrink))
t2 = Transform({"type":"spider","phi":newp[i][0],"theta":newp[i][1],"psi":newp[i][2]})
t2.set_trans(Vec2f(-newp[i][3]*shrink, -newp[i][4]*shrink))
if(len(ts) > 1):
# only do it if it is not c1
pixel_error = +1.0e23
for kts in ts:
ut = t2*kts
# we do not care which position minimizes the error
pixel_error = min(max_3D_pixel_error(t1, ut, lastring), pixel_error)
else:
pixel_error = max_3D_pixel_error(t1, t2, lastring)
if(pixel_error > deltaerror):
bad.append(i)
if(len(bad)>0):
badapples += [ids[bad[i]] for i in xrange(len(bad))]
for i in xrange(len(bad)-1,-1,-1):
del oldp[bad[i]],ids[bad[i]]
if(len(ids) == 0):
ERROR("sxpetite","program diverged, all images have large angular errors, most likely the initial model is badly off",1)
else:
# This generate new parameters, hopefully to be used as starting ones in the new iteration
write_text_file(ids,os.path.join(mainoutputdir,"chunk%01d.txt"%procid))
write_text_row(oldp,os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))
if(len(badapples)>0):
badapples.sort()
write_text_file(badapples,os.path.join(mainoutputdir,"badapples.txt"))
eli = 100*float(len(badapples))/float(total_images_now)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"Elimination of outliers: %5.1f percent"%eli )
else: eli = 0.0
del badapples, oldp,ids,bad,newp,ts
else: eli =0.0
eli = bcast_number_to_all(eli, source_node = main_node)
# This part under MPI
if(eli > 0.0):
# Compute current resolution
depres = compute_resolution(stack, mainoutputdir, \
[os.path.join(mainoutputdir,"chunk%01d.txt"%procid) for procid in xrange(2)], \
[os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid) for procid in xrange(2)], \
radi, nnxo, ali3d_options.CTF, myid, main_node, nproc)
depres = bcast_number_to_all(depres, source_node = main_node)
if(depres < newres):
# elimination of outliers decreased resolution, ignore the effort
eliminated_outliers = False
else:
eliminated_outliers = True
newres = depres
"""
# It does not seem to be needed, as data is there, we just point to the directory
for procid in xrange(2):
# set pointers to current parameters in main, which are for the reduced set stored above
partids[procid] = os.path.join(mainoutputdir,"chunk%01d.txt"%procid
partstack[procid] = os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid)
"""
else:
eliminated_outliers = False
else:
eliminated_outliers = False
if(myid == main_node and not eliminated_outliers):
for procid in xrange(2):
# This is standard path, copy parameters to be used to the main
cmd = "{} {} {}".format("cp -p ", partids[procid] , os.path.join(mainoutputdir,"chunk%01d.txt"%procid))
cmdexecute(cmd)
cmd = "{} {} {}".format("cp -p ", partstack[procid], os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))
cmdexecute(cmd)
keepgoing = 0
if( newres > currentres or (eliminated_outliers and not tracker["eliminated-outliers"])):
if(myid == main_node): print(" Resolution improved, full steam ahead!")
if( newres > currentres ): tracker["movedup"] = True
else: tracker["movedup"] = False
shrink = max(min(2*newres + paramsdict["aa"], 1.0),minshrink)
tracker["extension"] = 4
nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo)
tracker["previous-resolution"] = newres
currentres = newres
tracker["bestsolution"] = mainiteration
bestoutputdir = mainoutputdir
tracker["eliminated-outliers"] = eliminated_outliers
keepgoing = 1
elif(newres < currentres):
if(not tracker["movedup"] and tracker["extension"] < 2 and mainiteration > 1):
keepgoing = 0
if(myid == main_node): print(" Cannot improve resolution, the best result is in the directory main%03d"%tracker["bestsolution"])
else:
if(not tracker["movedup"] and tracker["extension"] > 1 and mainiteration > 1):
if(myid == main_node): print(" Resolution decreased. Will decrease target resolution and will fall back on the best so far: main%03d"%tracker["bestsolution"])
bestoutputdir = os.path.join(masterdir,"main%03d"%tracker["bestsolution"])
elif( tracker["movedup"] and tracker["extension"] > 1 and mainiteration > 1):
if(myid == main_node): print(" Resolution decreased. Will decrease target resolution and will try starting from previous stage: main%03d"%(mainiteration - 1))
bestoutputdir = os.path.join(masterdir,"main%03d"%(mainiteration-1))
elif( mainiteration == 1):
if(myid == main_node): print(" Resolution decreased in the first iteration. It is expected, not to worry")
bestoutputdir = mainoutputdir
tracker["extension"] += 1
else: # missing something here?
if(myid == main_node): print(" Should not be here, ERROR 175!")
break
mpi_finalize()
exit()
if( bestoutputdir != mainoutputdir ):
# This is the key, we just reset the main to previous, so it will be eventually used as a starting in the next iteration
mainoutputdir = bestoutputdir
"""
# Set data from the main previous best to the current.
for procid in xrange(2):
partids[procid] = os.path.join(bestoutputdir,"chunk%01d.txt"%procid)
partstack[procid] = os.path.join(bestoutputdir,"params-chunk%01d.txt"%procid)
"""
if(myid == main_node):
currentres = read_text_file( os.path.join(bestoutputdir,"current_resolution.txt") )[0]
currentres = bcast_number_to_all(currentres, source_node = main_node)
shrink = max(min(2*currentres + paramsdict["aa"], 1.0), minshrink)
tracker["extension"] -= 1
nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo)
tracker["previous-resolution"] = newres
tracker["eliminated-outliers"] = eliminated_outliers
tracker["movedup"] = False
keepgoing = 1
elif(newres == currentres):
if( tracker["extension"] > 0 ):
if(myid == main_node): print("The resolution did not improve. This is look ahead move. Let's try to relax slightly and hope for the best")
tracker["extension"] -= 1
tracker["movedup"] = False
shrink = max(min(2*currentres + paramsdict["aa"], 1.0), minshrink)
nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo)
if( tracker["previous-nx"] == nnxo ):
keepgoing = 0
else:
tracker["previous-resolution"] = newres
currentres = newres
tracker["eliminated-outliers"] = eliminated_outliers
tracker["movedup"] = False
keepgoing = 1
else:
if(myid == main_node): print("The resolution did not improve.")
keepgoing = 0
if( keepgoing == 1 ):
if(myid == main_node):
print(" New shrink and image dimension :",shrink,nxshrink)
"""
# It does not look like it is necessary, we just have to point to the directory as the files should be there.
# Will continue, so update the params files
for procid in xrange(2):
# partids ads partstack contain parameters to be used as starting in the next iteration
if(not os.path.exists(os.path.join(mainoutputdir,"chunk%01d.txt"%procid))):
cmd = "{} {} {}".format("cp -p ", partids[procid] , os.path.join(mainoutputdir,"chunk%01d.txt"%procid))
cmdexecute(cmd)
if(not os.path.exists(os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))):
cmd = "{} {} {}".format("cp -p ", partstack[procid], os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))
cmdexecute(cmd)
"""
previousoutputdir = mainoutputdir
tracker["previous-shrink"] = shrink
tracker["previous-nx"] = nxshrink
else:
if(myid == main_node):
print(" Terminating, the best solution is in the directory main%03d"%tracker["bestsolution"])
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def compute_fscs(stack, outputdir, chunkname, newgoodname, fscoutputdir, doit, keepchecking, nproc, myid, main_node):
# Compute reconstructions per group from good particles only to get FSC curves
# We will compute two FSC curves - from not averaged parameters and from averaged parameters
# So, we have to build two sets:
# not averaged (A2+C3) versus (B0+D5)
# averaged (A0+C1) versus (B3+D4)
# This requires pulling good subsets given by goodX*; I am not sure why good, sxconsistency above produced newgood text files.
# Otherwise, I am not sure what newbad will contain.
# Input that should vary:
# "bdb:"+os.path.join(outputdir,"chunk%01d%01d"%(procid,i))
# os.path.join(outputdir,"newgood%01d.txt"%procid)
# Output should be in a separate directory "fscoutputdir"
if(myid == main_node):
if keepchecking:
if(os.path.exists(fscoutputdir)):
doit = 0
print("Directory ",fscoutputdir," exists!")
else:
doit = 1
keepchecking = False
else:
doit = 1
if doit:
cmd = "{} {}".format("mkdir", fscoutputdir)
cmdexecute(cmd)
mpi_barrier(MPI_COMM_WORLD)
# not averaged
doit, keepchecking = checkstep(os.path.join(fscoutputdir,"volfscn0.hdf"), keepchecking, myid, main_node)
if doit:
if(myid == main_node):
# A2+C3
# indices
write_text_file( \
map(int, read_text_file(os.path.join(outputdir,chunkname+"0.txt")))+map(int, read_text_file(os.path.join(outputdir,chunkname+"2.txt"))), \
os.path.join(fscoutputdir,"chunkfn0.txt"))
# params
write_text_row( \
read_text_row(os.path.join(outputdir,newgoodname+"02.txt"))+read_text_row(os.path.join(outputdir,newgoodname+"22.txt")), \
os.path.join(fscoutputdir,"params-chunkfn0.txt"))
mpi_barrier(MPI_COMM_WORLD)
projdata = getindexdata(stack, os.path.join(fscoutputdir,"chunkfn0.txt"), os.path.join(fscoutputdir,"params-chunkfn0.txt"), myid, nproc)
if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if(myid == main_node):
vol.write_image(os.path.join(fscoutputdir,"volfscn0.hdf"))
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"Executed successfully: 3D reconstruction of", os.path.join(fscoutputdir,"volfscn0.hdf"))
del vol
doit, keepchecking = checkstep(os.path.join(fscoutputdir,"volfscn1.hdf"), keepchecking, myid, main_node)
if doit:
if(myid == main_node):
# B0+D5
# indices
write_text_file( \
map(int, read_text_file(os.path.join(outputdir,chunkname+"1.txt")))+map(int, read_text_file(os.path.join(outputdir,chunkname+"3.txt"))), \
os.path.join(fscoutputdir,"chunkfn1.txt"))
# params
write_text_row( \
read_text_row(os.path.join(outputdir,newgoodname+"10.txt"))+read_text_row(os.path.join(outputdir,newgoodname+"32.txt")), \
os.path.join(fscoutputdir,"params-chunkfn1.txt"))
mpi_barrier(MPI_COMM_WORLD)
projdata = getindexdata(stack, os.path.join(fscoutputdir,"chunkfn1.txt"), os.path.join(fscoutputdir,"params-chunkfn1.txt"), myid, nproc)
if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if(myid == main_node):
vol.write_image(os.path.join(fscoutputdir,"volfscn1.hdf"))
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"Executed successfully: 3D reconstruction of", os.path.join(fscoutputdir,"volfscn1.hdf"))
del vol
# averaged
doit, keepchecking = checkstep(os.path.join(fscoutputdir,"volfsca0.hdf"), keepchecking, myid, main_node)
if doit:
if(myid == main_node):
# A0+C1
# indices
write_text_file( \
map(int, read_text_file(os.path.join(outputdir,chunkname+"0.txt")))+map(int, read_text_file(os.path.join(outputdir,chunkname+"2.txt"))), \
os.path.join(fscoutputdir,"chunkfa0.txt"))
# params
write_text_row( \
read_text_row(os.path.join(outputdir,newgoodname+"00.txt"))+read_text_row(os.path.join(outputdir,newgoodname+"20.txt")), \
os.path.join(fscoutputdir,"params-chunkfa0.txt"))
mpi_barrier(MPI_COMM_WORLD)
projdata = getindexdata(stack, os.path.join(fscoutputdir,"chunkfa0.txt"), os.path.join(fscoutputdir,"params-chunkfa0.txt"), myid, nproc)
if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if(myid == main_node):
vol.write_image(os.path.join(fscoutputdir,"volfsca0.hdf"))
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"Executed successfully: 3D reconstruction of", os.path.join(fscoutputdir,"volfsca0.hdf"))
del vol
doit, keepchecking = checkstep(os.path.join(fscoutputdir,"volfsca1.hdf"), keepchecking, myid, main_node)
if doit:
if(myid == main_node):
# B3+D4
write_text_file( \
map(int, read_text_file(os.path.join(outputdir,chunkname+"1.txt")))+map(int, read_text_file(os.path.join(outputdir,chunkname+"3.txt"))), \
os.path.join(fscoutputdir,"chunkfa1.txt"))
# params
write_text_row( \
read_text_row(os.path.join(outputdir,newgoodname+"11.txt"))+read_text_row(os.path.join(outputdir,newgoodname+"31.txt")), \
os.path.join(fscoutputdir,"params-chunkfa1.txt"))
mpi_barrier(MPI_COMM_WORLD)
projdata = getindexdata(stack, os.path.join(fscoutputdir,"chunkfa1.txt"), os.path.join(fscoutputdir,"params-chunkfa1.txt"), myid, nproc)
if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if(myid == main_node):
vol.write_image(os.path.join(fscoutputdir,"volfsca1.hdf"))
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"Executed successfully: 3D reconstruction of", os.path.join(fscoutputdir,"volfsca1.hdf"))
del vol
# Get updated FSC curves
if(myid == main_node):
if(ali3d_options.mask3D is None): mask = model_circle(radi,nnxo,nnxo,nnxo)
else:
mask = get_im(ali3d_options.mask3D)
if keepchecking:
if(os.path.exists(os.path.join(fscoutputdir,"fscn.txt"))):
doit = 0
else:
doit = 1
keepchecking = False
else: doit = 1
if doit: fsc(get_im(os.path.join(fscoutputdir,"volfscn0.hdf"))*mask,\
get_im(os.path.join(fscoutputdir,"volfscn1.hdf"))*mask,\
1.0,os.path.join(fscoutputdir,"fscn.txt") )
if keepchecking:
if(os.path.exists(os.path.join(fscoutputdir,"fsca.txt"))):
doit = 0
else:
doit = 1
keepchecking = False
else: doit = 1
if doit: fsc(get_im(os.path.join(fscoutputdir,"volfsca0.hdf"))*mask,\
get_im(os.path.join(fscoutputdir,"volfsca1.hdf"))*mask,\
1.0,os.path.join(fscoutputdir,"fsca.txt") )
nfsc = read_text_file(os.path.join(fscoutputdir,"fscn.txt") ,-1)
currentres = 0.5
ns = len(nfsc[1])
for i in xrange(1,ns-1):
if ( (2*nfsc[1][i]/(1.0+nfsc[1][i]) ) < 0.5):
currentres = nfsc[0][i-1]
break
print(" Current resolution ",i,currentres)
else:
currentres = 0.0
currentres = bcast_number_to_all(currentres, source_node = main_node)
if(currentres < 0.0):
if(myid == main_node):
print(" Something wrong with the resolution, cannot continue")
mpi_finalize()
exit()
mpi_barrier(MPI_COMM_WORLD)
return currentres, doit, keepchecking
def spruce_up_var_m(refdata):
from utilities import print_msg
from utilities import model_circle, get_im
from filter import filt_tanl, filt_gaussl
from morphology import threshold
import os
numref = refdata[0]
outdir = refdata[1]
fscc = refdata[2]
total_iter = refdata[3]
varf = refdata[4]
mask = refdata[5]
ali50S = refdata[6]
if ali50S:
mask_50S = get_im("mask-50S.spi")
if fscc is None:
flmin = 0.4
aamin = 0.1
else:
flmin, aamin, idmin = minfilt(fscc)
aamin = aamin
msg = "Minimum tangent filter: cut-off frequency = %10.3f fall-off = %10.3f\n" % (
fflmin, aamin)
print_msg(msg)
for i in xrange(numref):
volf = get_im(os.path.join(outdir, "vol%04d.hdf" % total_iter), i)
if (not (varf is None)): volf = volf.filter_by_image(varf)
volf = filt_tanl(volf, flmin, aamin)
stat = Util.infomask(volf, mask, True)
volf -= stat[0]
Util.mul_scalar(volf, 1.0 / stat[1])
nx = volf.get_xsize()
stat = Util.infomask(
volf,
model_circle(nx // 2 - 2, nx, nx, nx) -
model_circle(nx // 2 - 6, nx, nx, nx), True)
volf -= stat[0]
Util.mul_img(volf, mask)
volf = threshold(volf)
volf = filt_gaussl(volf, 0.4)
if ali50S:
if i == 0:
v50S_0 = volf.copy()
v50S_0 *= mask_50S
else:
from applications import ali_vol_3
from fundamentals import rot_shift3D
v50S_i = volf.copy()
v50S_i *= mask_50S
params = ali_vol_3(v50S_i, v50S_0, 10.0, 0.5, mask=mask_50S)
volf = rot_shift3D(volf, params[0], params[1], params[2],
params[3], params[4], params[5], 1.0)
volf.write_image(os.path.join(outdir, "volf%04d.hdf" % total_iter), i)
def shiftali_MPI(stack,
maskfile=None,
maxit=100,
CTF=False,
snr=1.0,
Fourvar=False,
search_rng=-1,
oneDx=False,
search_rng_y=-1):
from applications import MPI_start_end
from utilities import model_circle, model_blank, get_image, peak_search, get_im
from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
from pap_statistics import varf2d_MPI
from fundamentals import fft, ccf, rot_shift3D, rot_shift2D
from utilities import get_params2D, set_params2D
from utilities import print_msg, print_begin_msg, print_end_msg
import os
import sys
from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
from EMAN2 import Processor
from time import time
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("shiftali_MPI")
max_iter = int(maxit)
if myid == main_node:
if ftp == "bdb":
from EMAN2db import db_open_dict
dummy = db_open_dict(stack, True)
nima = EMUtil.get_image_count(stack)
else:
nima = 0
nima = bcast_number_to_all(nima, source_node=main_node)
list_of_particles = list(range(nima))
image_start, image_end = MPI_start_end(nima, number_of_proc, myid)
list_of_particles = list_of_particles[image_start:image_end]
# read nx and ctf_app (if CTF) and broadcast to all nodes
if myid == main_node:
ima = EMData()
ima.read_image(stack, list_of_particles[0], True)
nx = ima.get_xsize()
ny = ima.get_ysize()
if CTF: ctf_app = ima.get_attr_default('ctf_applied', 2)
del ima
else:
nx = 0
ny = 0
if CTF: ctf_app = 0
nx = bcast_number_to_all(nx, source_node=main_node)
ny = bcast_number_to_all(ny, source_node=main_node)
if CTF:
ctf_app = bcast_number_to_all(ctf_app, source_node=main_node)
if ctf_app > 0:
ERROR("data cannot be ctf-applied", "shiftali_MPI", 1, myid)
if maskfile == None:
mrad = min(nx, ny)
mask = model_circle(mrad // 2 - 2, nx, ny)
else:
mask = get_im(maskfile)
if CTF:
from filter import filt_ctf
from morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
from global_def import CACHE_DISABLE
if CACHE_DISABLE:
data = EMData.read_images(stack, list_of_particles)
else:
for i in range(number_of_proc):
if myid == i:
data = EMData.read_images(stack, list_of_particles)
if ftp == "bdb": mpi_barrier(MPI_COMM_WORLD)
for im in range(len(data)):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
else:
ctf_2_sum = None
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
for i in range(0, nx, 2):
for j in range(ny):
temp.set_value_at(i, j, snr)
Util.add_img(ctf_2_sum, temp)
del temp
total_iter = 0
# apply initial xform.align2d parameters stored in header
init_params = []
for im in range(len(data)):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im],
p['alpha'],
sx=p['tx'],
sy=p['ty'],
mirror=p['mirror'],
scale=p['scale'])
# fourier transform all images, and apply ctf if CTF
for im in range(len(data)):
if CTF:
ctf_params = data[im].get_attr("ctf")
data[im] = filt_ctf(fft(data[im]), ctf_params)
else:
data[im] = fft(data[im])
sx_sum = 0
sy_sum = 0
sx_sum_total = 0
sy_sum_total = 0
shift_x = [0.0] * len(data)
shift_y = [0.0] * len(data)
ishift_x = [0.0] * len(data)
ishift_y = [0.0] * len(data)
for Iter in range(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n" % (total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in data:
Util.add_img(avg, im)
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF:
tavg = Util.divn_filter(avg, ctf_2_sum)
else:
tavg = Util.mult_scalar(avg, 1.0 / float(nima))
else:
tavg = EMData(nx, ny, 1, False)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask(tavg, mask, False)
tavg -= stat[0]
Util.mul_img(tavg, mask)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
tavg = fft(tavg)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
sx_sum = 0
sy_sum = 0
if search_rng > 0: nwx = 2 * search_rng + 1
else: nwx = nx
if search_rng_y > 0: nwy = 2 * search_rng_y + 1
else: nwy = ny
not_zero = 0
for im in range(len(data)):
if oneDx:
ctx = Util.window(ccf(data[im], tavg), nwx, 1)
p1 = peak_search(ctx)
p1_x = -int(p1[0][3])
ishift_x[im] = p1_x
sx_sum += p1_x
else:
p1 = peak_search(Util.window(ccf(data[im], tavg), nwx, nwy))
p1_x = -int(p1[0][4])
p1_y = -int(p1[0][5])
ishift_x[im] = p1_x
ishift_y[im] = p1_y
sx_sum += p1_x
sy_sum += p1_y
if not_zero == 0:
if (not (ishift_x[im] == 0.0)) or (not (ishift_y[im] == 0.0)):
not_zero = 1
sx_sum = mpi_reduce(sx_sum, 1, MPI_INT, MPI_SUM, main_node,
MPI_COMM_WORLD)
if not oneDx:
sy_sum = mpi_reduce(sy_sum, 1, MPI_INT, MPI_SUM, main_node,
MPI_COMM_WORLD)
if myid == main_node:
sx_sum_total = int(sx_sum[0])
if not oneDx:
sy_sum_total = int(sy_sum[0])
else:
sx_sum_total = 0
sy_sum_total = 0
sx_sum_total = bcast_number_to_all(sx_sum_total, source_node=main_node)
if not oneDx:
sy_sum_total = bcast_number_to_all(sy_sum_total,
source_node=main_node)
sx_ave = round(float(sx_sum_total) / nima)
sy_ave = round(float(sy_sum_total) / nima)
for im in range(len(data)):
p1_x = ishift_x[im] - sx_ave
p1_y = ishift_y[im] - sy_ave
params2 = {
"filter_type": Processor.fourier_filter_types.SHIFT,
"x_shift": p1_x,
"y_shift": p1_y,
"z_shift": 0.0
}
data[im] = Processor.EMFourierFilter(data[im], params2)
shift_x[im] += p1_x
shift_y[im] += p1_y
# stop if all shifts are zero
not_zero = mpi_reduce(not_zero, 1, MPI_INT, MPI_SUM, main_node,
MPI_COMM_WORLD)
if myid == main_node:
not_zero_all = int(not_zero[0])
else:
not_zero_all = 0
not_zero_all = bcast_number_to_all(not_zero_all, source_node=main_node)
if myid == main_node:
print_msg("Time of iteration = %12.2f\n" % (time() - start_time))
start_time = time()
if not_zero_all == 0: break
#for im in xrange(len(data)): data[im] = fft(data[im]) This should not be required as only header information is used
# combine shifts found with the original parameters
for im in range(len(data)):
t0 = init_params[im]
t1 = Transform()
t1.set_params({
"type": "2D",
"alpha": 0,
"scale": t0.get_scale(),
"mirror": 0,
"tx": shift_x[im],
"ty": shift_y[im]
})
# combine t0 and t1
tt = t1 * t0
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from utilities import file_type
if (file_type(stack) == "bdb"):
from utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, image_start,
image_end, number_of_proc)
else:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, image_start,
image_end, number_of_proc)
else:
send_attr_dict(main_node, data, par_str, image_start, image_end)
if myid == main_node: print_end_msg("shiftali_MPI")
def helicalshiftali_MPI(stack,
maskfile=None,
maxit=100,
CTF=False,
snr=1.0,
Fourvar=False,
search_rng=-1):
from applications import MPI_start_end
from utilities import model_circle, model_blank, get_image, peak_search, get_im, pad
from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
from pap_statistics import varf2d_MPI
from fundamentals import fft, ccf, rot_shift3D, rot_shift2D, fshift
from utilities import get_params2D, set_params2D, chunks_distribution
from utilities import print_msg, print_begin_msg, print_end_msg
import os
import sys
from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
from time import time
from pixel_error import ordersegments
from math import sqrt, atan2, tan, pi
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("helical-shiftali_MPI")
max_iter = int(maxit)
if (myid == main_node):
infils = EMUtil.get_all_attributes(stack, "filament")
ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord')
filaments = ordersegments(infils, ptlcoords)
total_nfils = len(filaments)
inidl = [0] * total_nfils
for i in range(total_nfils):
inidl[i] = len(filaments[i])
linidl = sum(inidl)
nima = linidl
tfilaments = []
for i in range(total_nfils):
tfilaments += filaments[i]
del filaments
else:
total_nfils = 0
linidl = 0
total_nfils = bcast_number_to_all(total_nfils, source_node=main_node)
if myid != main_node:
inidl = [-1] * total_nfils
inidl = bcast_list_to_all(inidl, myid, source_node=main_node)
linidl = bcast_number_to_all(linidl, source_node=main_node)
if myid != main_node:
tfilaments = [-1] * linidl
tfilaments = bcast_list_to_all(tfilaments, myid, source_node=main_node)
filaments = []
iendi = 0
for i in range(total_nfils):
isti = iendi
iendi = isti + inidl[i]
filaments.append(tfilaments[isti:iendi])
del tfilaments, inidl
if myid == main_node:
print_msg("total number of filaments: %d" % total_nfils)
if total_nfils < nproc:
ERROR(
'number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'
% (nproc, total_nfils), "ehelix_MPI", 1, myid)
# balanced load
temp = chunks_distribution([[len(filaments[i]), i]
for i in range(len(filaments))],
nproc)[myid:myid + 1][0]
filaments = [filaments[temp[i][1]] for i in range(len(temp))]
nfils = len(filaments)
#filaments = [[0,1]]
#print "filaments",filaments
list_of_particles = []
indcs = []
k = 0
for i in range(nfils):
list_of_particles += filaments[i]
k1 = k + len(filaments[i])
indcs.append([k, k1])
k = k1
data = EMData.read_images(stack, list_of_particles)
ldata = len(data)
print("ldata=", ldata)
nx = data[0].get_xsize()
ny = data[0].get_ysize()
if maskfile == None:
mrad = min(nx, ny) // 2 - 2
mask = pad(model_blank(2 * mrad + 1, ny, 1, 1.0), nx, ny, 1, 0.0)
else:
mask = get_im(maskfile)
# apply initial xform.align2d parameters stored in header
init_params = []
for im in range(ldata):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'],
p['mirror'], p['scale'])
if CTF:
from filter import filt_ctf
from morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
ctf_abs_sum = None
from utilities import info
for im in range(ldata):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
qctf = data[im].get_attr("ctf_applied")
if qctf == 0:
data[im] = filt_ctf(fft(data[im]), ctf_params)
data[im].set_attr('ctf_applied', 1)
elif qctf != 1:
ERROR('Incorrectly set qctf flag', "helicalshiftali_MPI", 1,
myid)
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
else:
data[im] = fft(data[im])
del list_of_particles
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
tsnr = 1. / snr
for i in range(0, nx + 2, 2):
for j in range(ny):
temp.set_value_at(i, j, tsnr)
temp.set_value_at(i + 1, j, 0.0)
#info(ctf_2_sum)
Util.add_img(ctf_2_sum, temp)
#info(ctf_2_sum)
del temp
total_iter = 0
shift_x = [0.0] * ldata
for Iter in range(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n" % (total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in range(ldata):
Util.add_img(avg, fshift(data[im], shift_x[im]))
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF: tavg = Util.divn_filter(avg, ctf_2_sum)
else: tavg = Util.mult_scalar(avg, 1.0 / float(nima))
else:
tavg = model_blank(nx, ny)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask(tavg, mask, False)
tavg -= stat[0]
Util.mul_img(tavg, mask)
tavg.write_image("tavg.hdf", Iter)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
tavg = fft(tavg)
sx_sum = 0.0
nxc = nx // 2
for ifil in range(nfils):
"""
# Calculate filament average
avg = EMData(nx, ny, 1, False)
filnima = 0
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
Util.add_img(avg, data[im])
filnima += 1
tavg = Util.mult_scalar(avg, 1.0/float(filnima))
"""
# Calculate 1D ccf between each segment and filament average
nsegms = indcs[ifil][1] - indcs[ifil][0]
ctx = [None] * nsegms
pcoords = [None] * nsegms
for im in range(indcs[ifil][0], indcs[ifil][1]):
ctx[im - indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx,
1)
pcoords[im - indcs[ifil][0]] = data[im].get_attr(
'ptcl_source_coord')
#ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0])
#print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True)
# search for best x-shift
cents = nsegms // 2
dst = sqrt(
max((pcoords[cents][0] - pcoords[0][0])**2 +
(pcoords[cents][1] - pcoords[0][1])**2,
(pcoords[cents][0] - pcoords[-1][0])**2 +
(pcoords[cents][1] - pcoords[-1][1])**2))
maxincline = atan2(ny // 2 - 2 - float(search_rng), dst)
kang = int(dst * tan(maxincline) + 0.5)
#print " settings ",nsegms,cents,dst,search_rng,maxincline,kang
# ## C code for alignment. @ming
results = [0.0] * 3
results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline,
kang, search_rng, nxc)
sib = int(results[0])
bang = results[1]
qm = results[2]
#print qm, sib, bang
# qm = -1.e23
#
# for six in xrange(-search_rng, search_rng+1,1):
# q0 = ctx[cents].get_value_at(six+nxc)
# for incline in xrange(kang+1):
# qt = q0
# qu = q0
# if(kang>0): tang = tan(maxincline/kang*incline)
# else: tang = 0.0
# for kim in xrange(cents+1,nsegms):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# #print " A ", ifil,six,incline,kim,xl,ixl,dxl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# for kim in xrange(cents):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# if( qt > qm ):
# qm = qt
# sib = six
# bang = tang
# if( qu > qm ):
# qm = qu
# sib = six
# bang = -tang
#if incline == 0: print "incline = 0 ",six,tang,qt,qu
#print qm,six,sib,bang
#print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True)
for im in range(indcs[ifil][0], indcs[ifil][1]):
kim = im - indcs[ifil][0]
dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 +
(pcoords[cents][1] - pcoords[kim][1])**2)
if (kim < cents): xl = -dst * bang + sib
else: xl = dst * bang + sib
shift_x[im] = xl
# Average shift
sx_sum += shift_x[indcs[ifil][0] + cents]
# #print myid,sx_sum,total_nfils
sx_sum = mpi_reduce(sx_sum, 1, MPI_FLOAT, MPI_SUM, main_node,
MPI_COMM_WORLD)
if myid == main_node:
sx_sum = float(sx_sum[0]) / total_nfils
print_msg("Average shift %6.2f\n" % (sx_sum))
else:
sx_sum = 0.0
sx_sum = 0.0
sx_sum = bcast_number_to_all(sx_sum, source_node=main_node)
for im in range(ldata):
shift_x[im] -= sx_sum
#print " %3d %6.3f"%(im,shift_x[im])
#exit()
# combine shifts found with the original parameters
for im in range(ldata):
t1 = Transform()
##import random
##shix=random.randint(-10, 10)
##t1.set_params({"type":"2D","tx":shix})
t1.set_params({"type": "2D", "tx": shift_x[im]})
# combine t0 and t1
tt = t1 * init_params[im]
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from utilities import file_type
if (file_type(stack) == "bdb"):
from utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata,
nproc)
else:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc)
else:
send_attr_dict(main_node, data, par_str, 0, ldata)
if myid == main_node: print_end_msg("helical-shiftali_MPI")
def main():
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from logger import Logger, BaseLogger_Files
import sys, os, time
global Tracker, Blockdata
from global_def import ERROR
progname = os.path.basename(sys.argv[0])
usage = progname + " --output_dir=output_dir --isac_dir=output_dir_of_isac "
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--adjust_to_analytic_model",
action="store_true",
default=False,
help="adjust power spectrum of 2-D averages to an analytic model ")
parser.add_option(
"--adjust_to_given_pw2",
action="store_true",
default=False,
help="adjust power spectrum to 2-D averages to given 1D power spectrum"
)
parser.add_option("--B_enhance",
action="store_true",
default=False,
help="using B-factor to enhance 2-D averages")
parser.add_option("--no_adjustment",
action="store_true",
default=False,
help="No power spectrum adjustment")
options_list = []
adjust_to_analytic_model = False
for q in sys.argv[1:]:
if (q[:26] == "--adjust_to_analytic_model"):
adjust_to_analytic_model = True
options_list.append(q)
break
adjust_to_given_pw2 = False
for q in sys.argv[1:]:
if (q[:21] == "--adjust_to_given_pw2"):
adjust_to_given_pw2 = True
options_list.append(q)
break
B_enhance = False
for q in sys.argv[1:]:
if (q[:11] == "--B_enhance"):
B_enhance = True
options_list.append(q)
break
no_adjustment = False
for q in sys.argv[1:]:
if (q[:15] == "--no_adjustment"):
no_adjustment = True
options_list.append(q)
break
if len(options_list) == 0:
if (Blockdata["myid"] == Blockdata["main_node"]):
print(
"specify one of the following options to start: 1. adjust_to_analytic_model; 2. adjust_to_given_pw2; 3. B_enhance; 4. no_adjustment"
)
if len(options_list) > 1:
ERROR(
"The specified options are exclusive. Use only one of them to start",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
# options in common
parser.add_option(
"--isac_dir",
type="string",
default='',
help="ISAC run output directory, input directory for this command")
parser.add_option(
"--output_dir",
type="string",
default='',
help="output directory where computed averages are saved")
parser.add_option("--pixel_size",
type="float",
default=-1.0,
help="pixel_size of raw images")
parser.add_option(
"--fl",
type="float",
default=-1.0,
help=
"low pass filter, =-1, not applied; =1, using FH1 (initial resolution), =2 using FH2 (resolution after local alignment), or user provided value"
)
parser.add_option("--stack",
type="string",
default="",
help="data stack used in ISAC")
parser.add_option("--radius", type="int", default=-1, help="radius")
parser.add_option("--xr",
type="float",
default=-1.0,
help="local alignment search range")
parser.add_option("--ts",
type="float",
default=1.0,
help="local alignment search step")
parser.add_option("--fh",
type="float",
default=-1.,
help="local alignment high frequencies limit")
parser.add_option("--maxit",
type="int",
default=5,
help="local alignment iterations")
parser.add_option("--navg",
type="int",
default=-1,
help="number of aveages")
parser.add_option("--skip_local_alignment",
action="store_true",
default=False,
help="skip local alignment")
parser.add_option(
"--noctf",
action="store_true",
default=False,
help=
"no ctf correction, useful for negative stained data. always ctf for cryo data"
)
if B_enhance:
parser.add_option(
"--B_start",
type="float",
default=10.0,
help=
"start frequency (1./Angstrom) of power spectrum for B_factor estimation"
)
parser.add_option(
"--Bfactor",
type="float",
default=-1.0,
help=
"User defined bactors (e.g. 45.0[A^2]). By default, the program automatically estimates B-factor. "
)
if adjust_to_given_pw2:
parser.add_option("--modelpw",
type="string",
default='',
help="1-D reference power spectrum")
checking_flag = 0
if (Blockdata["myid"] == Blockdata["main_node"]):
if not os.path.exists(options.modelpw): checking_flag = 1
checking_flag = bcast_number_to_all(checking_flag,
Blockdata["main_node"],
MPI_COMM_WORLD)
if checking_flag == 1:
ERROR("User provided power spectrum does not exist",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
(options, args) = parser.parse_args(sys.argv[1:])
Tracker = {}
Constants = {}
Constants["isac_dir"] = options.isac_dir
Constants["masterdir"] = options.output_dir
Constants["pixel_size"] = options.pixel_size
Constants["orgstack"] = options.stack
Constants["radius"] = options.radius
Constants["xrange"] = options.xr
Constants["xstep"] = options.ts
Constants["FH"] = options.fh
Constants["maxit"] = options.maxit
Constants["navg"] = options.navg
Constants["low_pass_filter"] = options.fl
if B_enhance:
Constants["B_start"] = options.B_start
Constants["Bfactor"] = options.Bfactor
if adjust_to_given_pw2: Constants["modelpw"] = options.modelpw
Tracker["constants"] = Constants
# -------------------------------------------------------------
#
# Create and initialize Tracker dictionary with input options # State Variables
#<<<---------------------->>>imported functions<<<---------------------------------------------
from utilities import get_im, bcast_number_to_all, write_text_file, read_text_file, wrap_mpi_bcast, write_text_row
from utilities import cmdexecute
from filter import filt_tanl
from time import sleep
from logger import Logger, BaseLogger_Files
import user_functions
import string
from string import split, atoi, atof
import json
#x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1)
#y_range = x_range
####-----------------------------------------------------------
# Create Master directory
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]:
masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen")
else:
masterdir = Tracker["constants"]["masterdir"]
if (Blockdata["myid"] == Blockdata["main_node"]):
msg = "Postprocessing ISAC 2D averages starts"
print(line, "Postprocessing ISAC 2D averages starts")
if not masterdir:
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "sharpen_" + Tracker["constants"]["isac_dir"]
os.mkdir(masterdir)
else:
if os.path.exists(masterdir):
print("%s already exists" % masterdir)
else:
os.mkdir(masterdir)
li = len(masterdir)
else:
li = 0
li = mpi_bcast(li, 1, MPI_INT, Blockdata["main_node"], MPI_COMM_WORLD)[0]
masterdir = mpi_bcast(masterdir, li, MPI_CHAR, Blockdata["main_node"],
MPI_COMM_WORLD)
masterdir = string.join(masterdir, "")
Tracker["constants"]["masterdir"] = masterdir
log_main = Logger(BaseLogger_Files())
log_main.prefix = Tracker["constants"]["masterdir"] + "/"
while not os.path.exists(Tracker["constants"]["masterdir"]):
print("Node ", Blockdata["myid"], " waiting...",
Tracker["constants"]["masterdir"])
sleep(1)
mpi_barrier(MPI_COMM_WORLD)
if (Blockdata["myid"] == Blockdata["main_node"]):
init_dict = {}
print(Tracker["constants"]["isac_dir"])
Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"],
"2dalignment")
core = read_text_row(
os.path.join(Tracker["directory"], "initial2Dparams.txt"))
for im in xrange(len(core)):
init_dict[im] = core[im]
del core
else:
init_dict = 0
init_dict = wrap_mpi_bcast(init_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
###
if (Blockdata["myid"] == Blockdata["main_node"]):
#Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack")
image = get_im(Tracker["constants"]["orgstack"], 0)
Tracker["constants"]["nnxo"] = image.get_xsize()
try:
ctf_params = image.get_attr("ctf")
if Tracker["constants"]["pixel_size"] == -1.:
Tracker["constants"]["pixel_size"] = ctf_params.apix
except:
print("pixel size value is not given.")
Tracker["ini_shrink"] = float(
get_im(os.path.join(Tracker["directory"], "aqfinal.hdf"),
0).get_xsize()) / Tracker["constants"]["nnxo"]
else:
Tracker["ini_shrink"] = 0
Tracker = wrap_mpi_bcast(Tracker,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
#print(Tracker["constants"]["pixel_size"], "pixel_size")
x_range = max(Tracker["constants"]["xrange"],
int(1. / Tracker["ini_shrink"]) + 1)
y_range = x_range
if (Blockdata["myid"] == Blockdata["main_node"]):
parameters = read_text_row(
os.path.join(Tracker["constants"]["isac_dir"],
"all_parameters.txt"))
else:
parameters = 0
parameters = wrap_mpi_bcast(parameters,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
params_dict = {}
list_dict = {}
#parepare params_dict
if Tracker["constants"]["navg"] < 0:
navg = EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"))
else:
navg = min(
Tracker["constants"]["navg"],
EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf")))
global_dict = {}
ptl_list = []
memlist = []
if (Blockdata["myid"] == Blockdata["main_node"]):
for iavg in xrange(navg):
params_of_this_average = []
image = get_im(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"), iavg)
members = image.get_attr("members")
memlist.append(members)
for im in xrange(len(members)):
abs_id = members[im]
global_dict[abs_id] = [iavg, im]
P = combine_params2( init_dict[abs_id][0], init_dict[abs_id][1], init_dict[abs_id][2], init_dict[abs_id][3], \
parameters[abs_id][0], parameters[abs_id][1]/Tracker["ini_shrink"], parameters[abs_id][2]/Tracker["ini_shrink"], parameters[abs_id][3])
if parameters[abs_id][3] == -1: print("wrong one")
params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0])
ptl_list.append(abs_id)
params_dict[iavg] = params_of_this_average
list_dict[iavg] = members
write_text_row(
params_of_this_average,
os.path.join(Tracker["constants"]["masterdir"],
"params_avg_%03d.txt" % iavg))
ptl_list.sort()
init_params = [None for im in xrange(len(ptl_list))]
for im in xrange(len(ptl_list)):
init_params[im] = [ptl_list[im]] + params_dict[global_dict[
ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
init_params,
os.path.join(Tracker["constants"]["masterdir"],
"init_isac_params.txt"))
else:
params_dict = 0
list_dict = 0
memlist = 0
params_dict = wrap_mpi_bcast(params_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
list_dict = wrap_mpi_bcast(list_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
memlist = wrap_mpi_bcast(memlist,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
# Now computing!
del init_dict
tag_sharpen_avg = 1000
## always apply low pass filter to B_enhanced images to suppress noise in high frequencies
enforced_to_H1 = False
if options.B_enhance:
if Tracker["constants"]["low_pass_filter"] == -1:
print("User does not provide low pass filter")
enforced_to_H1 = True
if navg < Blockdata["nproc"]: # Each CPU do one average
FH_list = [None for im in xrange(navg)]
for iavg in xrange(navg):
if Blockdata["myid"] == iavg:
mlist = [None for i in xrange(len(list_dict[iavg]))]
for im in xrange(len(mlist)):
mlist[im] = get_im(Tracker["constants"]["orgstack"],
list_dict[iavg][im])
set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
FH1 = get_optimistic_res(frc)
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d_before_ali.txt"%iavg))
if not options.skip_local_alignment:
new_average1 = within_group_refinement([mlist[kik] for kik in xrange(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1)
new_average2 = within_group_refinement([mlist[kik] for kik in xrange(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1)
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
FH2 = get_optimistic_res(frc)
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg))
#if Tracker["constants"]["nopwadj"]: # pw adjustment, 1. analytic model 2. PDB model 3. B-facttor enhancement
else:
FH2 = 0.0
FH_list[iavg] = [FH1, FH2]
if options.B_enhance:
new_avg, gb = apply_enhancement(
new_avg, Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"])
print("Process avg %d %f %f %f" %
(iavg, gb, FH1, FH2))
elif options.adjust_to_given_pw2:
roo = read_text_file(Tracker["constants"]["modelpw"], -1)
roo = roo[0] # always put pw in the first column
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], roo)
elif options.adjust_to_analytic_model:
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], None)
elif options.no_adjustment:
pass
print("Process avg %d %f %f" % (iavg, FH1, FH2))
if Tracker["constants"]["low_pass_filter"] != -1.:
if Tracker["constants"]["low_pass_filter"] == 1.:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 2.:
low_pass_filter = FH2
if options.skip_local_alignment: low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"][
"low_pass_filter"]
if low_pass_filter >= 0.45: low_pass_filter = 0.45
new_avg = filt_tanl(new_avg, low_pass_filter, 0.1)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
mpi_barrier(MPI_COMM_WORLD)
for im in xrange(navg): # avg
if im == Blockdata[
"myid"] and Blockdata["myid"] != Blockdata["main_node"]:
send_EMData(new_avg, Blockdata["main_node"], tag_sharpen_avg)
elif Blockdata["myid"] == Blockdata["main_node"]:
if im != Blockdata["main_node"]:
new_avg_other_cpu = recv_EMData(im, tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
else:
new_avg.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
if not options.skip_local_alignment:
if im == Blockdata["myid"]:
write_text_row(
plist,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params_avg_%03d.txt" % im))
if Blockdata["myid"] == im and Blockdata["myid"] != Blockdata[
"main_node"]:
wrap_mpi_send(plist_dict[im], Blockdata["main_node"],
MPI_COMM_WORLD)
elif im != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(im, MPI_COMM_WORLD)
plist_dict[im] = dummy
if im == Blockdata["myid"] and im != Blockdata["main_node"]:
wrap_mpi_send(FH_list[im], Blockdata["main_node"],
MPI_COMM_WORLD)
elif im != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(im, MPI_COMM_WORLD)
FH_list[im] = dummy
else:
if im == Blockdata["myid"] and im != Blockdata["main_node"]:
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif im != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(im, MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi_barrier(MPI_COMM_WORLD)
else:
FH_list = [[0, 0.0, 0.0] for im in xrange(navg)]
image_start, image_end = MPI_start_end(navg, Blockdata["nproc"],
Blockdata["myid"])
if Blockdata["myid"] == Blockdata["main_node"]:
cpu_dict = {}
for iproc in xrange(Blockdata["nproc"]):
local_image_start, local_image_end = MPI_start_end(
navg, Blockdata["nproc"], iproc)
for im in xrange(local_image_start, local_image_end):
cpu_dict[im] = iproc
else:
cpu_dict = 0
cpu_dict = wrap_mpi_bcast(cpu_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
slist = [None for im in xrange(navg)]
ini_list = [None for im in xrange(navg)]
avg1_list = [None for im in xrange(navg)]
avg2_list = [None for im in xrange(navg)]
plist_dict = {}
data_list = [None for im in xrange(navg)]
if Blockdata["myid"] == Blockdata["main_node"]: print("read data")
for iavg in xrange(image_start, image_end):
mlist = [None for i in xrange(len(list_dict[iavg]))]
for im in xrange(len(mlist)):
mlist[im] = get_im(Tracker["constants"]["orgstack"],
list_dict[iavg][im])
set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
data_list[iavg] = mlist
print("read data done %d" % Blockdata["myid"])
#if Blockdata["myid"] == Blockdata["main_node"]: print("start to compute averages")
for iavg in xrange(image_start, image_end):
mlist = data_list[iavg]
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
FH1 = get_optimistic_res(frc)
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d_before_ali.txt"%iavg))
if not options.skip_local_alignment:
new_average1 = within_group_refinement([mlist[kik] for kik in xrange(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH=max(Tracker["constants"]["FH"], FH1), FF=0.1)
new_average2 = within_group_refinement([mlist[kik] for kik in xrange(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou= Tracker["constants"]["radius"], rs=1.0, xrng=[ x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.1)
if options.noctf:
new_avg, frc, plist = compute_average_noctf(
mlist, Tracker["constants"]["radius"])
else:
new_avg, frc, plist = compute_average_ctf(
mlist, Tracker["constants"]["radius"])
plist_dict[iavg] = plist
FH2 = get_optimistic_res(frc)
else:
FH2 = 0.0
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg))
FH_list[iavg] = [iavg, FH1, FH2]
if options.B_enhance:
new_avg, gb = apply_enhancement(
new_avg, Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"])
print("Process avg %d %f %f %f" % (iavg, gb, FH1, FH2))
elif options.adjust_to_given_pw2:
roo = read_text_file(Tracker["constants"]["modelpw"], -1)
roo = roo[0] # always on the first column
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], roo)
print("Process avg %d %f %f" % (iavg, FH1, FH2))
elif adjust_to_analytic_model:
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], None)
print("Process avg %d %f %f" % (iavg, FH1, FH2))
elif options.no_adjustment:
pass
if Tracker["constants"]["low_pass_filter"] != -1.:
new_avg = filt_tanl(new_avg,
Tracker["constants"]["low_pass_filter"],
0.1)
if Tracker["constants"]["low_pass_filter"] != -1.:
if Tracker["constants"]["low_pass_filter"] == 1.:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 2.:
low_pass_filter = FH2
if options.skip_local_alignment: low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"]["low_pass_filter"]
if low_pass_filter >= 0.45: low_pass_filter = 0.45
new_avg = filt_tanl(new_avg, low_pass_filter, 0.1)
else:
if enforced_to_H1: new_avg = filt_tanl(new_avg, FH1, 0.1)
if options.B_enhance: new_avg = fft(new_avg)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
slist[iavg] = new_avg
## send to main node to write
mpi_barrier(MPI_COMM_WORLD)
for im in xrange(navg):
# avg
if cpu_dict[im] == Blockdata[
"myid"] and Blockdata["myid"] != Blockdata["main_node"]:
send_EMData(slist[im], Blockdata["main_node"], tag_sharpen_avg)
elif cpu_dict[im] == Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
slist[im].set_attr("members", memlist[im])
slist[im].write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
elif cpu_dict[im] != Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
new_avg_other_cpu = recv_EMData(cpu_dict[im], tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
if not options.skip_local_alignment:
if cpu_dict[im] == Blockdata["myid"]:
write_text_row(
plist_dict[im],
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params_avg_%03d.txt" % im))
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(plist_dict[im], Blockdata["main_node"],
MPI_COMM_WORLD)
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
plist_dict[im] = dummy
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
else:
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi_barrier(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if not options.skip_local_alignment:
if Blockdata["myid"] == Blockdata["main_node"]:
ali3d_local_params = [None for im in xrange(len(ptl_list))]
for im in xrange(len(ptl_list)):
ali3d_local_params[im] = [ptl_list[im]] + plist_dict[
global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
ali3d_local_params,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params.txt"))
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
else:
if Blockdata["myid"] == Blockdata["main_node"]:
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
mpi_barrier(MPI_COMM_WORLD)
target_xr = 3
target_yr = 3
if (Blockdata["myid"] == 0):
cmd = "{} {} {} {} {} {} {} {} {} {}".format("sxchains.py", os.path.join(Tracker["constants"]["masterdir"],"class_averages.hdf"),\
os.path.join(Tracker["constants"]["masterdir"],"junk.hdf"),os.path.join(Tracker["constants"]["masterdir"],"ordered_class_averages.hdf"),\
"--circular","--radius=%d"%Tracker["constants"]["radius"] , "--xr=%d"%(target_xr+1),"--yr=%d"%(target_yr+1),"--align", ">/dev/null")
junk = cmdexecute(cmd)
cmd = "{} {}".format(
"rm -rf",
os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"))
junk = cmdexecute(cmd)
from mpi import mpi_finalize
mpi_finalize()
exit()
def cml_open_proj(stack, ir, ou, lf, hf, dpsi = 1):
from projection import cml_sinogram
from utilities import model_circle, get_params_proj, model_blank, get_im
from fundamentals import fftip
from filter import filt_tanh
# number of projections
if type(stack) == type(""): nprj = EMUtil.get_image_count(stack)
else: nprj = len(stack)
Prj = [] # list of projections
Ori = [-1] * 4 * nprj # orientation intial (phi, theta, psi, index) for each projection
for i in xrange(nprj):
image = get_im(stack, i)
# read initial angles if given
try: Ori[4*i], Ori[4*i+1], Ori[4*i+2], s2x, s2y = get_params_proj(image)
except: pass
if(i == 0):
nx = image.get_xsize()
if(ou < 1): ou = nx // 2 - 1
diameter = int(2 * ou)
mask2D = model_circle(ou, nx, nx)
if ir > 0: mask2D -= model_circle(ir, nx, nx)
# normalize under the mask
[mean_a, sigma, imin, imax] = Util.infomask(image, mask2D, True)
image -= mean_a
Util.mul_scalar(image, 1.0/sigma)
Util.mul_img(image, mask2D)
# sinogram
sino = cml_sinogram(image, diameter, dpsi)
# prepare the cut positions in order to filter (lf: low freq; hf: high freq)
ihf = min(int(2 * hf * diameter), diameter + (diameter + 1) % 2)
ihf = ihf + (ihf + 1) % 2 # index ihf must be odd to take the img part
ilf = max(int(2 * lf * diameter), 0)
ilf = ilf + ilf % 2 # index ilf must be even to fall in the real part
bdf = ihf - ilf + 1
# process lines
nxe = sino.get_xsize()
nye = sino.get_ysize()
prj = model_blank(bdf, 2*nye)
pp = model_blank(nxe, 2*nye)
for li in xrange(nye):
# get the line li
line = Util.window(sino, nxe, 1, 1, 0, li-nye//2, 0)
# u2 (not improve the results)
#line = filt_tanh(line, ou / float(nx), ou / float(nx))
# normalize this line
[mean_l, sigma_l, imin, imax] = Util.infomask(line, None, True)
line = (line - mean_l) / sigma_l
# fft
fftip(line)
# filter (cut part of coef) and create mirror line
Util.cml_prepare_line(prj, line, ilf, ihf, li, nye)
# store the projection
Prj.append(prj)
return Prj, Ori
def shiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1, oneDx=False, search_rng_y=-1):
from applications import MPI_start_end
from utilities import model_circle, model_blank, get_image, peak_search, get_im
from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
from statistics import varf2d_MPI
from fundamentals import fft, ccf, rot_shift3D, rot_shift2D
from utilities import get_params2D, set_params2D
from utilities import print_msg, print_begin_msg, print_end_msg
import os
import sys
from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
from EMAN2 import Processor
from time import time
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("shiftali_MPI")
max_iter=int(maxit)
if myid == main_node:
if ftp == "bdb":
from EMAN2db import db_open_dict
dummy = db_open_dict(stack, True)
nima = EMUtil.get_image_count(stack)
else:
nima = 0
nima = bcast_number_to_all(nima, source_node = main_node)
list_of_particles = range(nima)
image_start, image_end = MPI_start_end(nima, number_of_proc, myid)
list_of_particles = list_of_particles[image_start: image_end]
# read nx and ctf_app (if CTF) and broadcast to all nodes
if myid == main_node:
ima = EMData()
ima.read_image(stack, list_of_particles[0], True)
nx = ima.get_xsize()
ny = ima.get_ysize()
if CTF: ctf_app = ima.get_attr_default('ctf_applied', 2)
del ima
else:
nx = 0
ny = 0
if CTF: ctf_app = 0
nx = bcast_number_to_all(nx, source_node = main_node)
ny = bcast_number_to_all(ny, source_node = main_node)
if CTF:
ctf_app = bcast_number_to_all(ctf_app, source_node = main_node)
if ctf_app > 0: ERROR("data cannot be ctf-applied", "shiftali_MPI", 1, myid)
if maskfile == None:
mrad = min(nx, ny)
mask = model_circle(mrad//2-2, nx, ny)
else:
mask = get_im(maskfile)
if CTF:
from filter import filt_ctf
from morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
from global_def import CACHE_DISABLE
if CACHE_DISABLE:
data = EMData.read_images(stack, list_of_particles)
else:
for i in xrange(number_of_proc):
if myid == i:
data = EMData.read_images(stack, list_of_particles)
if ftp == "bdb": mpi_barrier(MPI_COMM_WORLD)
for im in xrange(len(data)):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
else: ctf_2_sum = None
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
for i in xrange(0,nx,2):
for j in xrange(ny):
temp.set_value_at(i,j,snr)
Util.add_img(ctf_2_sum, temp)
del temp
total_iter = 0
# apply initial xform.align2d parameters stored in header
init_params = []
for im in xrange(len(data)):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im], p['alpha'], sx=p['tx'], sy=p['ty'], mirror=p['mirror'], scale=p['scale'])
# fourier transform all images, and apply ctf if CTF
for im in xrange(len(data)):
if CTF:
ctf_params = data[im].get_attr("ctf")
data[im] = filt_ctf(fft(data[im]), ctf_params)
else:
data[im] = fft(data[im])
sx_sum=0
sy_sum=0
sx_sum_total=0
sy_sum_total=0
shift_x = [0.0]*len(data)
shift_y = [0.0]*len(data)
ishift_x = [0.0]*len(data)
ishift_y = [0.0]*len(data)
for Iter in xrange(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n"%(total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in data: Util.add_img(avg, im)
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF:
tavg = Util.divn_filter(avg, ctf_2_sum)
else: tavg = Util.mult_scalar(avg, 1.0/float(nima))
else:
tavg = EMData(nx, ny, 1, False)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask( tavg, mask, False )
tavg -= stat[0]
Util.mul_img(tavg, mask)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
tavg = fft(tavg)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
sx_sum=0
sy_sum=0
if search_rng > 0: nwx = 2*search_rng+1
else: nwx = nx
if search_rng_y > 0: nwy = 2*search_rng_y+1
else: nwy = ny
not_zero = 0
for im in xrange(len(data)):
if oneDx:
ctx = Util.window(ccf(data[im],tavg),nwx,1)
p1 = peak_search(ctx)
p1_x = -int(p1[0][3])
ishift_x[im] = p1_x
sx_sum += p1_x
else:
p1 = peak_search(Util.window(ccf(data[im],tavg), nwx,nwy))
p1_x = -int(p1[0][4])
p1_y = -int(p1[0][5])
ishift_x[im] = p1_x
ishift_y[im] = p1_y
sx_sum += p1_x
sy_sum += p1_y
if not_zero == 0:
if (not(ishift_x[im] == 0.0)) or (not(ishift_y[im] == 0.0)):
not_zero = 1
sx_sum = mpi_reduce(sx_sum, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if not oneDx:
sy_sum = mpi_reduce(sy_sum, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
sx_sum_total = int(sx_sum[0])
if not oneDx:
sy_sum_total = int(sy_sum[0])
else:
sx_sum_total = 0
sy_sum_total = 0
sx_sum_total = bcast_number_to_all(sx_sum_total, source_node = main_node)
if not oneDx:
sy_sum_total = bcast_number_to_all(sy_sum_total, source_node = main_node)
sx_ave = round(float(sx_sum_total)/nima)
sy_ave = round(float(sy_sum_total)/nima)
for im in xrange(len(data)):
p1_x = ishift_x[im] - sx_ave
p1_y = ishift_y[im] - sy_ave
params2 = {"filter_type" : Processor.fourier_filter_types.SHIFT, "x_shift" : p1_x, "y_shift" : p1_y, "z_shift" : 0.0}
data[im] = Processor.EMFourierFilter(data[im], params2)
shift_x[im] += p1_x
shift_y[im] += p1_y
# stop if all shifts are zero
not_zero = mpi_reduce(not_zero, 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
not_zero_all = int(not_zero[0])
else:
not_zero_all = 0
not_zero_all = bcast_number_to_all(not_zero_all, source_node = main_node)
if myid == main_node:
print_msg("Time of iteration = %12.2f\n"%(time()-start_time))
start_time = time()
if not_zero_all == 0: break
#for im in xrange(len(data)): data[im] = fft(data[im]) This should not be required as only header information is used
# combine shifts found with the original parameters
for im in xrange(len(data)):
t0 = init_params[im]
t1 = Transform()
t1.set_params({"type":"2D","alpha":0,"scale":t0.get_scale(),"mirror":0,"tx":shift_x[im],"ty":shift_y[im]})
# combine t0 and t1
tt = t1*t0
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from utilities import file_type
if(file_type(stack) == "bdb"):
from utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, image_start, image_end, number_of_proc)
else:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, image_start, image_end, number_of_proc)
else: send_attr_dict(main_node, data, par_str, image_start, image_end)
if myid == main_node: print_end_msg("shiftali_MPI")
def main():
import sys
import os
import math
import random
import pyemtbx.options
import time
from random import random, seed, randint
from optparse import OptionParser
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Generic 2-D image processing programs.
Functionality:
1. Phase flip a stack of images and write output to new file:
sxprocess.py input_stack.hdf output_stack.hdf --phase_flip
2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.
The window size will change accordingly.
sxprocess input.hdf output.hdf --changesize --ratio=0.5
3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes or a 3-D volume.
sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024]
4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs:
sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64
5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3):
sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid
6. Retrieve original image numbers of images listed in ISAC output stack of averages:
sxprocess.py select1.hdf ohk.txt
7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file.
Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack.
sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw
sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw
8. Generate a 1D rotationally averaged power spectrum of an image.
sxprocess.py vol.hdf --rotwp=rotpw.txt
# Output will contain three columns:
(1) rotationally averaged power spectrum
(2) logarithm of the rotationally averaged power spectrum
(3) integer line number (from zero to approximately to half the image size)
9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data.
sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt
The output file is then imported and 3D transformed volume computed:
sxheader.py bdb:p --params=xform.projection --import=output.txt
mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI
The reconstructed volume is in the position of the volume computed using the input.txt parameters and then
transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz)
10. Import ctf parameters from the output of sxcter into windowed particle headers.
There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph.
In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'.
Normally this is done by e2boxer.py during windowing.
Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created.
sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0
# Output will be a vritual stack bdb:Gdata
sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0
To concatenate output files:
cd directory
e2bdb.py . --makevstack=bdb:allparticles --filt=G
IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them.
11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor
sxprocess.py orientationparams.txt scaledparams.txt scale=0.5
12. Generate 3D mask from a given 3-D volume automatically or using threshold provided by user.
13. Postprocess 3-D or 2-D images:
for 3-D volumes: calculate FSC with provided mask; weight summed volume with FSC; estimate B-factor from FSC weighted summed two volumes; apply negative B-factor to the weighted volume.
for 2-D images: calculate B-factor and apply negative B-factor to 2-D images.
14. Winow stack file -reduce size of images without changing the pixel size.
"""
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--order", action="store_true", help="Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False)
parser.add_option("--order_lookup", action="store_true", help="Test/Debug.", default=False)
parser.add_option("--order_metropolis", action="store_true", help="Test/Debug.", default=False)
parser.add_option("--order_pca", action="store_true", help="Test/Debug.", default=False)
parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
parser.add_option("--circular", action="store_true", help="Select circular ordering (fisr image has to be similar to the last", default=False)
parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering")
parser.add_option("--changesize", action="store_true", help="resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False)
parser.add_option("--ratio", type="float", default=1.0, help="The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round")
parser.add_option("--pw", action="store_true", help="compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False)
parser.add_option("--wn", type="int", default=-1, help="Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))")
parser.add_option("--phase_flip", action="store_true", help="Phase flip the input stack", default=False)
parser.add_option("--makedb", metavar="param1=value1:param2=value2", type="string",
action="append", help="One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0")
parser.add_option("--generate_projections", metavar="param1=value1:param2=value2", type="string",
action="append", help="Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64.")
parser.add_option("--isacgroup", type="int", help="Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1)
parser.add_option("--isacselect", action="store_true", help="Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False)
parser.add_option("--params", type="string", default=None, help="Name of header of parameter, which one depends on specific option")
parser.add_option("--adjpw", action="store_true", help="Adjust rotationally averaged power spectrum of an image", default=False)
parser.add_option("--rotpw", type="string", default=None, help="Name of the text file to contain rotationally averaged power spectrum of the input image.")
parser.add_option("--transformparams", type="string", default=None, help="Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters.")
# import ctf estimates done using cter
parser.add_option("--input", type="string", default= None, help="Input particles.")
parser.add_option("--importctf", type="string", default= None, help="Name of the file containing CTF parameters produced by sxcter.")
parser.add_option("--defocuserror", type="float", default=1000000.0, help="Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%")
parser.add_option("--astigmatismerror", type="float", default=360.0, help="Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees.")
# import ctf estimates done using cter
parser.add_option("--scale", type="float", default=-1.0, help="Divide shifts in the input 3D orientation parameters text file by the scale factor.")
# generate adaptive mask from an given 3-D volume
parser.add_option("--adaptive_mask", action="store_true", help="create adavptive 3-D mask from a given volume", default=False)
parser.add_option("--nsigma", type="float", default= 1., help="number of times of sigma of the input volume to obtain the the large density cluster")
parser.add_option("--ndilation", type="int", default= 3, help="number of times of dilation applied to the largest cluster of density")
parser.add_option("--kernel_size", type="int", default= 11, help="convolution kernel for smoothing the edge of the mask")
parser.add_option("--gauss_standard_dev", type="int", default= 9, help="stanadard deviation value to generate Gaussian edge")
parser.add_option("--threshold", type="float", default= 9999., help="threshold provided by user to binarize input volume")
parser.add_option("--ne", type="int", default= 0, help="number of times to erode the binarized input image")
parser.add_option("--nd", type="int", default= 0, help="number of times to dilate the binarized input image")
parser.add_option("--postprocess", action="store_true", help="postprocess unfiltered odd, even 3-D volumes",default=False)
parser.add_option("--fsc_weighted", action="store_true", help="postprocess unfiltered odd, even 3-D volumes")
parser.add_option("--low_pass_filter", action="store_true", default=False, help="postprocess unfiltered odd, even 3-D volumes")
parser.add_option("--ff", type="float", default=.25, help="low pass filter stop band frequency in absolute unit")
parser.add_option("--aa", type="float", default=.1, help="low pass filter falloff" )
parser.add_option("--mask", type="string", help="input mask file", default=None)
parser.add_option("--output", type="string", help="output file name", default="postprocessed.hdf")
parser.add_option("--pixel_size", type="float", help="pixel size of the data", default=1.0)
parser.add_option("--B_start", type="float", help="starting frequency in Angstrom for B-factor estimation", default=10.)
parser.add_option("--FSC_cutoff", type="float", help="stop frequency in Angstrom for B-factor estimation", default=0.143)
parser.add_option("--2d", action="store_true", help="postprocess isac 2-D averaged images",default=False)
parser.add_option("--window_stack", action="store_true", help="window stack images using a smaller window size", default=False)
parser.add_option("--box", type="int", default= 0, help="the new window size ")
(options, args) = parser.parse_args()
global_def.BATCH = True
if options.phase_flip:
nargs = len(args)
if nargs != 2:
print "must provide name of input and output file!"
return
from EMAN2 import Processor
instack = args[0]
outstack = args[1]
nima = EMUtil.get_image_count(instack)
from filter import filt_ctf
for i in xrange(nima):
img = EMData()
img.read_image(instack, i)
try:
ctf = img.get_attr('ctf')
except:
print "no ctf information in input stack! Exiting..."
return
dopad = True
sign = 1
binary = 1 # phase flip
assert img.get_ysize() > 1
dict = ctf.to_dict()
dz = dict["defocus"]
cs = dict["cs"]
voltage = dict["voltage"]
pixel_size = dict["apix"]
b_factor = dict["bfactor"]
ampcont = dict["ampcont"]
dza = dict["dfdiff"]
azz = dict["dfang"]
if dopad and not img.is_complex(): ip = 1
else: ip = 0
params = {"filter_type": Processor.fourier_filter_types.CTF_,
"defocus" : dz,
"Cs": cs,
"voltage": voltage,
"Pixel_size": pixel_size,
"B_factor": b_factor,
"amp_contrast": ampcont,
"dopad": ip,
"binary": binary,
"sign": sign,
"dza": dza,
"azz":azz}
tmp = Processor.EMFourierFilter(img, params)
tmp.set_attr_dict({"ctf": ctf})
tmp.write_image(outstack, i)
elif options.changesize:
nargs = len(args)
if nargs != 2:
ERROR("must provide name of input and output file!", "change size", 1)
return
from utilities import get_im
instack = args[0]
outstack = args[1]
sub_rate = float(options.ratio)
nima = EMUtil.get_image_count(instack)
from fundamentals import resample
for i in xrange(nima):
resample(get_im(instack, i), sub_rate).write_image(outstack, i)
elif options.isacgroup>-1:
nargs = len(args)
if nargs != 3:
ERROR("Three files needed on input!", "isacgroup", 1)
return
from utilities import get_im
instack = args[0]
m=get_im(args[1],int(options.isacgroup)).get_attr("members")
l = []
for k in m:
l.append(int(get_im(args[0],k).get_attr(options.params)))
from utilities import write_text_file
write_text_file(l, args[2])
elif options.isacselect:
nargs = len(args)
if nargs != 2:
ERROR("Two files needed on input!", "isacgroup", 1)
return
from utilities import get_im
nima = EMUtil.get_image_count(args[0])
m = []
for k in xrange(nima):
m += get_im(args[0],k).get_attr("members")
m.sort()
from utilities import write_text_file
write_text_file(m, args[1])
elif options.pw:
nargs = len(args)
if nargs < 2:
ERROR("must provide name of input and output file!", "pw", 1)
return
from utilities import get_im, write_text_file
from fundamentals import rops_table
d = get_im(args[0])
ndim = d.get_ndim()
if ndim ==3:
pw = rops_table(d)
write_text_file(pw, args[1])
else:
nx = d.get_xsize()
ny = d.get_ysize()
if nargs ==3: mask = get_im(args[2])
wn = int(options.wn)
if wn == -1:
wn = max(nx, ny)
else:
if( (wn<nx) or (wn<ny) ): ERROR("window size cannot be smaller than the image size","pw",1)
n = EMUtil.get_image_count(args[0])
from utilities import model_blank, model_circle, pad
from EMAN2 import periodogram
p = model_blank(wn,wn)
for i in xrange(n):
d = get_im(args[0], i)
if nargs==3:
d *=mask
st = Util.infomask(d, None, True)
d -= st[0]
p += periodogram(pad(d, wn, wn, 1, 0.))
p /= n
p.write_image(args[1])
elif options.adjpw:
if len(args) < 3:
ERROR("filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)","adjpw",1)
return
img_stack = args[0]
from math import sqrt
from fundamentals import rops_table, fft
from utilities import read_text_file, get_im
from filter import filt_tanl, filt_table
if( args[1][-3:] == 'txt'):
rops_dst = read_text_file( args[1] )
else:
rops_dst = rops_table(get_im( args[1] ))
out_stack = args[2]
if(len(args) >4):
fl = float(args[3])
aa = float(args[4])
else:
fl = -1.0
aa = 0.0
nimage = EMUtil.get_image_count( img_stack )
for i in xrange(nimage):
img = fft(get_im(img_stack, i) )
rops_src = rops_table(img)
assert len(rops_dst) == len(rops_src)
table = [0.0]*len(rops_dst)
for j in xrange( len(rops_dst) ):
table[j] = sqrt( rops_dst[j]/rops_src[j] )
if( fl > 0.0):
img = filt_tanl(img, fl, aa)
img = fft(filt_table(img, table))
img.write_image(out_stack, i)
elif options.rotpw != None:
if len(args) != 1:
ERROR("Only one input permitted","rotpw",1)
return
from utilities import write_text_file, get_im
from fundamentals import rops_table
from math import log10
t = rops_table(get_im(args[0]))
x = range(len(t))
r = [0.0]*len(x)
for i in x: r[i] = log10(t[i])
write_text_file([t,r,x],options.rotpw)
elif options.transformparams != None:
if len(args) != 2:
ERROR("Please provide names of input and output files with orientation parameters","transformparams",1)
return
from utilities import read_text_row, write_text_row
transf = [0.0]*6
spl=options.transformparams.split(',')
for i in xrange(len(spl)): transf[i] = float(spl[i])
write_text_row( rotate_shift_params(read_text_row(args[0]), transf) , args[1])
elif options.makedb != None:
nargs = len(args)
if nargs != 1:
print "must provide exactly one argument denoting database key under which the input params will be stored"
return
dbkey = args[0]
print "database key under which params will be stored: ", dbkey
gbdb = js_open_dict("e2boxercache/gauss_box_DB.json")
parmstr = 'dummy:'+options.makedb[0]
(processorname, param_dict) = parsemodopt(parmstr)
dbdict = {}
for pkey in param_dict:
if (pkey == 'invert_contrast') or (pkey == 'use_variance'):
if param_dict[pkey] == 'True':
dbdict[pkey] = True
else:
dbdict[pkey] = False
else:
dbdict[pkey] = param_dict[pkey]
gbdb[dbkey] = dbdict
elif options.generate_projections:
nargs = len(args)
if nargs != 3:
ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\
"sxprocess - generate projections",1)
return
inpstr = args[0]
outstk = args[1]
micpref = args[2]
parmstr = 'dummy:'+options.generate_projections[0]
(processorname, param_dict) = parsemodopt(parmstr)
parm_CTF = False
parm_format = 'bdb'
parm_apix = 2.5
if 'CTF' in param_dict:
if param_dict['CTF'] == 'True':
parm_CTF = True
if 'format' in param_dict:
parm_format = param_dict['format']
if 'apix' in param_dict:
parm_apix = float(param_dict['apix'])
boxsize = 64
if 'boxsize' in param_dict:
boxsize = int(param_dict['boxsize'])
print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize
scale_mult = 2500
sigma_add = 1.5
sigma_proj = 30.0
sigma2_proj = 17.5
sigma_gauss = 0.3
sigma_mic = 30.0
sigma2_mic = 17.5
sigma_gauss_mic = 0.3
if 'scale_mult' in param_dict:
scale_mult = float(param_dict['scale_mult'])
if 'sigma_add' in param_dict:
sigma_add = float(param_dict['sigma_add'])
if 'sigma_proj' in param_dict:
sigma_proj = float(param_dict['sigma_proj'])
if 'sigma2_proj' in param_dict:
sigma2_proj = float(param_dict['sigma2_proj'])
if 'sigma_gauss' in param_dict:
sigma_gauss = float(param_dict['sigma_gauss'])
if 'sigma_mic' in param_dict:
sigma_mic = float(param_dict['sigma_mic'])
if 'sigma2_mic' in param_dict:
sigma2_mic = float(param_dict['sigma2_mic'])
if 'sigma_gauss_mic' in param_dict:
sigma_gauss_mic = float(param_dict['sigma_gauss_mic'])
from filter import filt_gaussl, filt_ctf
from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank,pad,model_gauss_noise,set_params2D, set_params_proj
from projection import prep_vol,prgs
seed(14567)
delta = 29
angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S")
nangle = len(angles)
modelvol = []
nvlms = EMUtil.get_image_count(inpstr)
from utilities import get_im
for k in xrange(nvlms): modelvol.append(get_im(inpstr,k))
nx = modelvol[0].get_xsize()
if nx != boxsize:
ERROR("Requested box dimension does not match dimension of the input model.", \
"sxprocess - generate projections",1)
nvol = 10
volfts = [[] for k in xrange(nvlms)]
for k in xrange(nvlms):
for i in xrange(nvol):
sigma = sigma_add + random() # 1.5-2.5
addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40 )
scale = scale_mult * (0.5+random())
vf, kb = prep_vol(modelvol[k] + scale*addon)
volfts[k].append(vf)
del vf, modelvol
if parm_format == "bdb":
stack_data = "bdb:"+outstk
delete_bdb(stack_data)
else:
stack_data = outstk + ".hdf"
Cs = 2.0
pixel = parm_apix
voltage = 120.0
ampcont = 10.0
ibd = 4096/2-boxsize
iprj = 0
width = 240
xstart = 8 + boxsize/2
ystart = 8 + boxsize/2
rowlen = 17
from random import randint
params = []
for idef in xrange(3, 8):
irow = 0
icol = 0
mic = model_blank(4096, 4096)
defocus = idef * 0.5#0.2
if parm_CTF:
astampl=defocus*0.15
astangl=50.0
ctf = generate_ctf([defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl])
for i in xrange(nangle):
for k in xrange(12):
dphi = 8.0*(random()-0.5)
dtht = 8.0*(random()-0.5)
psi = 360.0*random()
phi = angles[i][0]+dphi
tht = angles[i][1]+dtht
s2x = 4.0*(random()-0.5)
s2y = 4.0*(random()-0.5)
params.append([phi, tht, psi, s2x, s2y])
ivol = iprj % nvol
#imgsrc = randint(0,nvlms-1)
imgsrc = iprj % nvlms
proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y])
x = xstart + irow * width
y = ystart + icol * width
mic += pad(proj, 4096, 4096, 1, 0.0, x-2048, y-2048, 0)
proj = proj + model_gauss_noise( sigma_proj, nx, nx )
if parm_CTF:
proj = filt_ctf(proj, ctf)
proj.set_attr_dict({"ctf":ctf, "ctf_applied":0})
proj = proj + filt_gaussl(model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss)
proj.set_attr("origimgsrc",imgsrc)
proj.set_attr("test_id", iprj)
# flags describing the status of the image (1 = true, 0 = false)
set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0])
set_params_proj(proj, [phi, tht, psi, s2x, s2y])
proj.write_image(stack_data, iprj)
icol += 1
if icol == rowlen:
icol = 0
irow += 1
iprj += 1
mic += model_gauss_noise(sigma_mic,4096,4096)
if parm_CTF:
#apply CTF
mic = filt_ctf(mic, ctf)
mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic)
mic.write_image(micpref + "%1d.hdf" % (idef-3), 0)
drop_spider_doc("params.txt", params)
elif options.importctf != None:
print ' IMPORTCTF '
from utilities import read_text_row,write_text_row
from random import randint
import subprocess
grpfile = 'groupid%04d'%randint(1000,9999)
ctfpfile = 'ctfpfile%04d'%randint(1000,9999)
cterr = [options.defocuserror/100.0, options.astigmatismerror]
ctfs = read_text_row(options.importctf)
for kk in xrange(len(ctfs)):
root,name = os.path.split(ctfs[kk][-1])
ctfs[kk][-1] = name[:-4]
if(options.input[:4] != 'bdb:'):
ERROR('Sorry, only bdb files implemented','importctf',1)
d = options.input[4:]
#try: str = d.index('*')
#except: str = -1
from string import split
import glob
uu = os.path.split(d)
uu = os.path.join(uu[0],'EMAN2DB',uu[1]+'.bdb')
flist = glob.glob(uu)
for i in xrange(len(flist)):
root,name = os.path.split(flist[i])
root = root[:-7]
name = name[:-4]
fil = 'bdb:'+os.path.join(root,name)
sourcemic = EMUtil.get_all_attributes(fil,'ptcl_source_image')
nn = len(sourcemic)
gctfp = []
groupid = []
for kk in xrange(nn):
junk,name2 = os.path.split(sourcemic[kk])
name2 = name2[:-4]
ctfp = [-1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]
for ll in xrange(len(ctfs)):
if(name2 == ctfs[ll][-1]):
# found correct
if(ctfs[ll][8]/ctfs[ll][0] <= cterr[0]):
# acceptable defocus error
ctfp = ctfs[ll][:8]
if(ctfs[ll][10] > cterr[1] ):
# error of astigmatism exceed the threshold, set astigmatism to zero.
ctfp[6] = 0.0
ctfp[7] = 0.0
gctfp.append(ctfp)
groupid.append(kk)
break
if(len(groupid) > 0):
write_text_row(groupid, grpfile)
write_text_row(gctfp, ctfpfile)
cmd = "{} {} {} {}".format('e2bdb.py',fil,'--makevstack=bdb:'+root+'G'+name,'--list='+grpfile)
#print cmd
subprocess.call(cmd, shell=True)
cmd = "{} {} {} {}".format('sxheader.py','bdb:'+root+'G'+name,'--params=ctf','--import='+ctfpfile)
#print cmd
subprocess.call(cmd, shell=True)
else:
print ' >>> Group ',name,' skipped.'
cmd = "{} {} {}".format("rm -f",grpfile,ctfpfile)
subprocess.call(cmd, shell=True)
elif options.scale > 0.0:
from utilities import read_text_row,write_text_row
scale = options.scale
nargs = len(args)
if nargs != 2:
print "Please provide names of input and output file!"
return
p = read_text_row(args[0])
for i in xrange(len(p)):
p[i][3] /= scale
p[i][4] /= scale
write_text_row(p, args[1])
elif options.adaptive_mask:
from utilities import get_im
from morphology import adaptive_mask, binarize, erosion, dilation
nsigma = options.nsigma
ndilation = options.ndilation
kernel_size = options.kernel_size
gauss_standard_dev = options.gauss_standard_dev
nargs = len(args)
if nargs ==0:
print " Create 3D mask from a given volume, either automatically or from the user provided threshold."
elif nargs > 2:
print "Too many inputs are given, try again!"
return
else:
inputvol = get_im(args[0])
input_path, input_file_name = os.path.split(args[0])
input_file_name_root,ext=os.path.splitext(input_file_name)
if nargs == 2: mask_file_name = args[1]
else: mask_file_name = "adaptive_mask_for_"+input_file_name_root+".hdf" # Only hdf file is output.
if options.threshold !=9999.:
mask3d = binarize(inputvol, options.threshold)
for i in xrange(options.ne): mask3d = erosion(mask3d)
for i in xrange(options.nd): mask3d = dilation(mask3d)
else:
mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev)
mask3d.write_image(mask_file_name)
elif options.postprocess:
from utilities import get_im
from fundamentals import rot_avg_table
from morphology import compute_bfactor,power
from statistics import fsc
from filter import filt_table, filt_gaussinv
from EMAN2 import periodogram
e1 = get_im(args[0],0)
if e1.get_zsize()==1:
nimage = EMUtil.get_image_count(args[0])
if options.mask !=None: m = get_im(options.mask)
else: m = None
for i in xrange(nimage):
e1 = get_im(args[0],i)
if m: e1 *=m
guinerline = rot_avg_table(power(periodogram(e1),.5))
freq_max = 1/(2.*pixel_size)
freq_min = 1./options.B_start
b,junk=compute_bfactor(guinerline, freq_min, freq_max, pixel_size)
tmp = b/pixel_size**2
sigma_of_inverse=sqrt(2./tmp)
e1 = filt_gaussinv(e1,sigma_of_inverse)
if options.low_pass_filter:
from filter import filt_tanl
e1 =filt_tanl(e1,options.ff, options.aa)
e1.write_image(options.output)
else:
nargs = len(args)
e1 = get_im(args[0])
if nargs >1: e2 = get_im(args[1])
if options.mask !=None: m = get_im(options.mask)
else: m =None
pixel_size = options.pixel_size
from math import sqrt
if m !=None:
e1 *=m
if nargs >1 :e2 *=m
if options.fsc_weighted:
frc = fsc(e1,e2,1)
## FSC is done on masked two images
#### FSC weighting sqrt((2.*fsc)/(1+fsc));
fil = len(frc[1])*[None]
for i in xrange(len(fil)):
if frc[1][i]>=options.FSC_cutoff: tmp = frc[1][i]
else: tmp = 0.0
fil[i] = sqrt(2.*tmp/(1.+tmp))
if nargs>1: e1 +=e2
if options.fsc_weighted: e1=filt_table(e1,fil)
guinerline = rot_avg_table(power(periodogram(e1),.5))
freq_max = 1/(2.*pixel_size)
freq_min = 1./options.B_start
b,junk = compute_bfactor(guinerline, freq_min, freq_max, pixel_size)
tmp = b/pixel_size**2
sigma_of_inverse=sqrt(2./tmp)
e1 = filt_gaussinv(e1,sigma_of_inverse)
if options.low_pass_filter:
from filter import filt_tanl
e1 =filt_tanl(e1,options.ff, options.aa)
e1.write_image(options.output)
elif options.window_stack:
nargs = len(args)
if nargs ==0:
print " Reduce image size of a stack"
return
else:
output_stack_name = None
inputstack = args[0]
if nargs ==2:output_stack_name = args[1]
input_path,input_file_name=os.path.split(inputstack)
input_file_name_root,ext=os.path.splitext(input_file_name)
if input_file_name_root[0:3]=="bdb":stack_is_bdb= True
else: stack_is_bdb= False
if output_stack_name is None:
if stack_is_bdb: output_stack_name ="bdb:reduced_"+input_file_name_root[4:]
else:output_stack_name = "reduced_"+input_file_name_root+".hdf" # Only hdf file is output.
nimage = EMUtil.get_image_count(inputstack)
from fundamentals import window2d
for i in xrange(nimage):
image = EMData()
image.read_image(inputstack,i)
w = window2d(image,options.box,options.box)
w.write_image(output_stack_name,i)
else: ERROR("Please provide option name","sxprocess.py",1)
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ Input Output [options]
Generate three micrographs, each micrograph contains one projection of a long filament.
Input: Reference Volume, output directory
Output: Three micrographs stored in output directory
sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84
Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
Generate run through example script for helicon
sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
"""
parser = OptionParser(usage,version=SPARXVERSION)
# helicise the Atom coordinates
# generate micrographs of helical filament
parser.add_option("--generate_micrograph", action="store_true", default=False, help="Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction")
parser.add_option("--apix", type="float", default= -1, help="pixel size in Angstroms")
parser.add_option("--rand_seed", type="int", default=14567, help="the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs.")
parser.add_option("--Cs", type="float", default= 2.0, help="Microscope Cs (spherical aberation)")
parser.add_option("--voltage", type="float", default=200.0, help="Microscope voltage in KV")
parser.add_option("--ac", type="float", default=10.0, help="Amplitude contrast (percentage, default=10)")
parser.add_option("--nonoise", action="store_true", default=False, help="Do not add noise to the micrograph.")
# generate initial volume
parser.add_option("--generate_noisycyl", action="store_true", default=False, help="Generate initial volume of noisy cylinder.")
parser.add_option("--boxsize", type="string", default="100,100,200", help="String containing x , y, z dimensions (separated by comma) in pixels")
parser.add_option("--rad", type="int", default=35, help="Radius of initial volume in pixels")
# generate 2D mask
parser.add_option("--generate_mask", action="store_true", default=False, help="Generate 2D rectangular mask.")
parser.add_option("--masksize", type="string", default="200,200", help="String containing x and y dimensions (separated by comma) in pixels")
parser.add_option("--maskwidth", type="int", default=60, help="Width of rectangular mask")
# Apply 2D alignment parameters to input stack and output new images to output stack
parser.add_option("--applyparams", action="store_true", default=False, help="Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack")
# Generate run script
parser.add_option("--generate_script", action="store_true", default=False, help="Generate script for helicon run through example")
parser.add_option("--filename", type="string", default="runhelicon", help="Name of run script to generate")
parser.add_option("--seg_ny", type="int", default=180, help="y-dimension of segment used for refinement")
parser.add_option("--ptcl_dist", type="int", default=15, help="Distance in pixels between adjacent segments windowed from same filament")
parser.add_option("--fract", type="float", default=0.35, help="Fraction of the volume used for applying helical symmetry.")
(options, args) = parser.parse_args()
if len(args) > 3:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
else:
if options.generate_script:
generate_runscript(options.filename, options.seg_ny, options.ptcl_dist, options.fract)
if options.generate_micrograph:
if options.apix <= 0:
print "Please enter pixel size."
sys.exit()
generate_helimic(args[0], args[1], options.apix, options.CTF, options.Cs, options.voltage, options.ac, options.nonoise, options.rand_seed)
if options.generate_noisycyl:
from utilities import model_cylinder, model_gauss_noise
outvol = args[0]
boxdims = options.boxsize.split(',')
if len(boxdims) < 1 or len(boxdims) > 3:
print "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'"
sys.exit()
nx= int(boxdims[0])
if len(boxdims) == 1:
ny = nx
nz = nx
else:
ny = int(boxdims[1])
if len(boxdims) == 3:
nz = int(boxdims[2])
(model_cylinder(options.rad,nx, ny, nz)*model_gauss_noise(1.0, nx, ny, nz) ).write_image(outvol)
if options.generate_mask:
from utilities import model_blank, pad
outvol = args[0]
maskdims = options.masksize.split(',')
if len(maskdims) < 1 or len(maskdims) > 2:
print "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'"
sys.exit()
nx= int(maskdims[0])
if len(maskdims) == 1:
ny = nx
else:
ny = int(maskdims[1])
mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1, 0.0)
mask.write_image(outvol)
if options.applyparams:
from utilities import get_im, get_params2D, set_params2D
from fundamentals import cyclic_shift
stack = args[0]
newstack = args[1]
mask = get_im(args[2])
nima = EMUtil.get_image_count(stack)
for im in xrange(nima):
prj = get_im(stack,im)
alpha, sx, sy, mirror, scale = get_params2D(prj)
prj = cyclic_shift(prj, int(sx))
set_params2D(prj, [0.0,0.,0.0,0,1])
stat = Util.infomask(prj , mask, False )
prj= (prj-stat[0])/stat[1]
ctf_params = prj.get_attr("ctf")
prj.set_attr('ctf_applied', 0)
prj.write_image(newstack, im)
def isac_substack(args):
from utilities import get_im, write_text_file
from EMAN2db import db_open_dict
from e2bdb import makerelpath
# To make the execution exit upon fatal error by ERROR in global_def.py
global_def.BATCH = True
# Check error conditions
subcommand_name = 'isac_substack'
if not os.path.exists(args.input_isac_class_avgs_path):
ERROR('Input ISAC class average stack file does not exist. Please check the file path and restart the program.', subcommand_name) # action=1 - fatal error, exit
if os.path.exists(args.output_directory):
ERROR('Output directory exists. Please change the name and restart the program.', subcommand_name) # action=1 - fatal error, exit
assert(os.path.exists(args.input_isac_class_avgs_path))
assert(not os.path.exists(args.output_directory))
# Create output directory
os.mkdir(args.output_directory)
# Retrieve original particle IDs of member particles listed in ISAC class average stack
n_img_processed = EMUtil.get_image_count(args.input_isac_class_avgs_path)
isac_substack_particle_id_list = []
for i_img in xrange(n_img_processed):
isac_substack_particle_id_list += get_im(args.input_isac_class_avgs_path, i_img).get_attr('members')
isac_substack_particle_id_list.sort()
# Save the substack particle id list
isac_substack_particle_id_list_file_path = os.path.join(args.output_directory, 'isac_substack_particle_id_list.txt')
write_text_file(isac_substack_particle_id_list, isac_substack_particle_id_list_file_path)
# Open the output BDB dictionary
assert(args.output_directory != '')
output_virtual_bdb_stack_real_path = 'bdb:%s#isac_substack' % args.output_directory
output_virtual_bdb_stack = db_open_dict(output_virtual_bdb_stack_real_path)
# Convert an absolute path to the actual output data to a relative path by eliminating any symbolic links
output_virtual_bdb_stack_real_path=os.path.realpath(output_virtual_bdb_stack.path)+"/"
# Open the input BDB dictionary
input_bdb_stack = db_open_dict(args.input_bdb_stack_path, ro=True) # Read only
# Copy the header from input to output BDB dictionary
n_img_detected = len(isac_substack_particle_id_list)
print_progress("Detected %d ISAC validated particles in %s"%(n_img_detected, args.input_isac_class_avgs_path))
print(" ")
# Loop through all ISAC validated particles
n_img_processed = 0
for i_img_detected, isac_substack_particle_id in enumerate(isac_substack_particle_id_list):
# Print progress
if i_img_detected % 1000 == 0:
try:
print_progress("Progress %5.2f%%: Processing %6dth entry (Particle ID %6d)."%(float(i_img_detected)/n_img_detected*100.0, i_img_detected, isac_substack_particle_id))
sys.stdout.flush()
except:
pass
# Read a particle image header from input bdb stack
try:
img_header = input_bdb_stack.get(isac_substack_particle_id, nodata=1).get_attr_dict() # Need only header information
except:
ERROR('Failed to read image header of particle #%d from %s. Skipping this image...' % (isac_substack_particle_id, args.input_bdb_stack_path), subcommand_name, action = 0) # action = 0 - non-fatal, print a warning;
continue
# Convert an absolute path to the actual input data to a relative path by eliminating any symbolic links
try:
input_bdb_stack_real_path = os.path.realpath(input_bdb_stack.get_data_path(isac_substack_particle_id))
# Conver the path to OS specific format
if os.name == 'nt':
output_virtual_bdb_stack_real_path = output_virtual_bdb_stack_real_path.replace("\\", '/')
input_bdb_stack_real_path = input_bdb_stack_real_path.replace('\\', '/')
# Takes a pair of paths /a/b/c/d and /a/b/e/f/g and returns a relative path to b from a, ../../e/f/g
common_relative_path = makerelpath(output_virtual_bdb_stack_real_path, input_bdb_stack_real_path)
except:
ERROR('Failure to find common relative data path for particle image #%d. Skipping this image...' % (isac_substack_particle_id), subcommand_name, action = 0) # action = 0 - non-fatal, print a warning;
continue
assert(img_header["data_path"] != None)
# Update the image header for output
img_header["data_path"] = common_relative_path
img_header["data_n"] = isac_substack_particle_id
img_header["data_source"] = args.input_bdb_stack_path
# Register the image header to output virtual bdb stack
output_virtual_bdb_stack[n_img_processed] = img_header
# Increment process image counts
n_img_processed += 1
# Close input and output bdb stacks
output_virtual_bdb_stack.close()
input_bdb_stack.close()
# Print summary of processing
print(" ")
print_progress("Summary of processing...")
print_progress("Detected : %6d"%(n_img_detected))
print_progress("Processed : %6d"%(n_img_processed))
print(" ")
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
# the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 540.0-psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0, 360.0-psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl= --aa= --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--output_dir", type="string" , default="./", help="Output directory")
parser.add_option("--ave2D", type="string" , default=False, help="Write to the disk a stack of 2D averages")
parser.add_option("--var2D", type="string" , default=False, help="Write to the disk a stack of 2D variances")
parser.add_option("--ave3D", type="string" , default=False, help="Write to the disk reconstructed 3D average")
parser.add_option("--var3D", type="string" , default=False, help="Compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp", type="int" , default=100, help="Number of neighbouring projections.(Default is 100)")
parser.add_option("--no_norm", action="store_true", default=False, help="Do not use normalization.(Default is to apply normalization)")
#parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option("--npad", type="int" , default=2 , help="Number of time to pad the original images.(Default is 2 times padding)")
parser.add_option("--sym" , type="string" , default="c1", help="Symmetry. (Default is no symmetry)")
parser.add_option("--fl", type="float" , default=0.0, help="Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)")
parser.add_option("--aa", type="float" , default=0.02 , help="Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)")
parser.add_option("--CTF", action="store_true", default=False, help="Use CFT correction.(Default is no CTF correction)")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option("--VAR" , action="store_true", default=False, help="Stack of input consists of 2D variances (Default False)")
parser.add_option("--decimate", type ="float", default=0.25, help="Image decimate rate, a number less than 1. (Default is 0.25)")
parser.add_option("--window", type ="int", default=0, help="Target image size relative to original image size. (Default value is zero.)")
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
#parser.add_option("--nvec", type="int" , default=0 , help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)")
parser.add_option("--symmetrize", action="store_true", default=False, help="Prepare input stack for handling symmetry (Default False)")
parser.add_option("--overhead", type ="float", default=0.5, help="python overhead per CPU.")
(options,args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
#from mpi import *
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im, wrap_mpi_send, wrap_mpi_recv
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
# Set up global variables related to bdb cache
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
# Set up global variables related to ERROR function
global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
if RUNNING_UNDER_MPI: global_def.MPI = True
if options.output_dir =="./": current_output_dir = os.path.abspath(options.output_dir)
else: current_output_dir = options.output_dir
if options.symmetrize :
if RUNNING_UNDER_MPI:
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if( number_of_proc > 1 ):
ERROR("Cannot use more than one CPU for symmetry preparation","sx3dvariability",1)
except:
pass
except:
pass
if not os.path.exists(current_output_dir): os.mkdir(current_output_dir)
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
from logger import Logger,BaseLogger_Files
if os.path.exists(os.path.join(current_output_dir, "log.txt")): os.remove(os.path.join(current_output_dir, "log.txt"))
log_main=Logger(BaseLogger_Files())
log_main.prefix = os.path.join(current_output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if( sym == "c1" ):
ERROR("There is no need to symmetrize stack for C1 symmetry","sx3dvariability",1)
line =""
for a in sys.argv:
line +=" "+a
log_main.add(line)
if(instack[:4] !="bdb:"):
#if output_dir =="./": stack = "bdb:data"
stack = "bdb:"+current_output_dir+"/data"
delete_bdb(stack)
junk = cmdexecute("sxcpy.py "+instack+" "+stack)
else: stack = instack
qt = EMUtil.get_all_attributes(stack,'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None]*na
for k in range(ks):
#Qfile = "Q%1d"%k
#if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k)
Qfile = os.path.join(current_output_dir, "Q%1d"%k)
#delete_bdb("bdb:Q%1d"%k)
delete_bdb("bdb:"+Qfile)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:"+Qfile)
#DB = db_open_dict("bdb:Q%1d"%k)
DB = db_open_dict("bdb:"+Qfile)
for i in range(na):
ut = qt[i]*ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
#if options.output_dir =="./": delete_bdb("bdb:sdata")
delete_bdb("bdb:" + current_output_dir + "/"+"sdata")
#junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:"+current_output_dir+"/"+"sdata"
print(sdata)
junk = cmdexecute("e2bdb.py " + current_output_dir +" --makevstack="+sdata +" --filt=Q")
#junk = cmdexecute("ls EMAN2DB/sdata*")
#a = get_im("bdb:sdata")
a = get_im(sdata)
a.set_attr("variabilitysymmetry",sym)
#a.write_image("bdb:sdata")
a.write_image(sdata)
else:
from fundamentals import window2d
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
shared_comm = mpi_comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL)
myid_on_node = mpi_comm_rank(shared_comm)
no_of_processes_per_group = mpi_comm_size(shared_comm)
masters_from_groups_vs_everything_else_comm = mpi_comm_split(MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node)
color, no_of_groups, balanced_processor_load_on_nodes = get_colors_and_subsets(main_node, MPI_COMM_WORLD, myid, \
shared_comm, myid_on_node, masters_from_groups_vs_everything_else_comm)
overhead_loading = options.overhead*number_of_proc
#memory_per_node = options.memory_per_node
#if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group
keepgoing = 1
current_window = options.window
current_decimate = options.decimate
if len(args) == 1: stack = args[0]
else:
print(( "usage: " + usage))
print(( "Please run '" + progname + " -h' for detailed options"))
return 1
t0 = time()
# obsolete flags
options.MPI = True
#options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter", "sx3dvariability", 1, myid)
#if options.VAR and options.SND:
# ERROR("Only one of var and SND can be set!", "sx3dvariability", myid)
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR("When VAR is set, the program cannot output ave2D, ave3D or var2D", "sx3dvariability", 1, myid)
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
#if options.nvec > 0 :
# ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
#if options.nvec > 0 and options.ave3D == None:
# ERROR("When doing PCA analysis, one must set ave3D", "sx3dvariability", 1, myid)
if current_decimate>1.0 or current_decimate<0.0:
ERROR("Decimate rate should be a value between 0.0 and 1.0", "sx3dvariability", 1, myid)
if current_window < 0.0:
ERROR("Target window size should be always larger than zero", "sx3dvariability", 1, myid)
if myid == main_node:
img = get_image(stack, 0)
nx = img.get_xsize()
ny = img.get_ysize()
if(min(nx, ny) < current_window): keepgoing = 0
keepgoing = bcast_number_to_all(keepgoing, main_node, MPI_COMM_WORLD)
if keepgoing == 0: ERROR("The target window size cannot be larger than the size of decimated image", "sx3dvariability", 1, myid)
import string
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if not os.path.exists(current_output_dir): os.mkdir(current_output_dir)# Never delete output_dir in the program!
img_per_grp = options.img_per_grp
#nvec = options.nvec
radiuspca = options.radiuspca
from logger import Logger,BaseLogger_Files
#if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main=Logger(BaseLogger_Files())
log_main.prefix = os.path.join(current_output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv: line +=" "+a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("Symmetry : %s"%options.sym)
log_main.add("Input stack : %s"%stack)
log_main.add("Output_dir : %s"%current_output_dir)
if options.ave3D: log_main.add("Ave3d : %s"%options.ave3D)
if options.var3D: log_main.add("Var3d : %s"%options.var3D)
if options.ave2D: log_main.add("Ave2D : %s"%options.ave2D)
if options.var2D: log_main.add("Var2D : %s"%options.var2D)
if options.VAR: log_main.add("VAR : True")
else: log_main.add("VAR : False")
if options.CTF: log_main.add("CTF correction : True ")
else: log_main.add("CTF correction : False ")
log_main.add("Image per group : %5d"%options.img_per_grp)
log_main.add("Image decimate rate : %4.3f"%current_decimate)
log_main.add("Low pass filter : %4.3f"%options.fl)
current_fl = options.fl
if current_fl == 0.0: current_fl = 0.5
log_main.add("Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"%round((current_fl*current_decimate),3))
log_main.add("Window size : %5d "%current_window)
log_main.add("sx3dvariability begins")
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
nnxo = nx
nnyo = ny
if options.sym != "c1" :
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if(i != options.sym):
ERROR("The symmetry provided does not agree with the symmetry of the input stack", "sx3dvariability", 1, myid)
except:
ERROR("Input stack is not prepared for symmetry, please follow instructions", "sx3dvariability", 1, myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if((nima/i)*i != nima):
ERROR("The length of the input stack is incorrect for symmetry processing", "sx3dvariability", 1, myid)
symbaselen = nima/i
else: symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nnxo = 0
nnyo = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
nnxo = bcast_number_to_all(nnxo)
nnyo = bcast_number_to_all(nnyo)
if current_window > max(nx, ny):
ERROR("Window size is larger than the original image size", "sx3dvariability", 1)
if current_decimate == 1.:
if current_window !=0:
nx = current_window
ny = current_window
else:
if current_window == 0:
nx = int(nx*current_decimate+0.5)
ny = int(ny*current_decimate+0.5)
else:
nx = int(current_window*current_decimate+0.5)
ny = nx
symbaselen = bcast_number_to_all(symbaselen)
# check FFT prime number
from fundamentals import smallprime
is_fft_friendly = (nx == smallprime(nx))
if not is_fft_friendly:
if myid == main_node:
log_main.add("The target image size is not a product of small prime numbers")
log_main.add("Program adjusts the input settings!")
### two cases
if current_decimate == 1.:
nx = smallprime(nx)
ny = nx
current_window = nx # update
if myid == main_node:
log_main.add("The window size is updated to %d."%current_window)
else:
if current_window == 0:
nx = smallprime(int(nx*current_decimate+0.5))
current_decimate = float(nx)/nnxo
ny = nx
if (myid == main_node):
log_main.add("The decimate rate is updated to %f."%current_decimate)
else:
nx = smallprime(int(current_window*current_decimate+0.5))
ny = nx
current_window = int(nx/current_decimate+0.5)
if (myid == main_node):
log_main.add("The window size is updated to %d."%current_window)
if myid == main_node:
log_main.add("The target image size is %d"%nx)
if radiuspca == -1: radiuspca = nx/2-2
if myid == main_node: log_main.add("%-70s: %d\n"%("Number of projection", nima))
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR: # 2D variance images have no shifts
#varList = EMData.read_images(stack, range(img_begin, img_end))
from EMAN2 import Region
for index_of_particle in range(img_begin,img_end):
image = get_im(stack, index_of_proj)
if current_window > 0: varList.append(fdecimate(window2d(image,current_window,current_window), nx,ny))
else: varList.append(fdecimate(image, nx,ny))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle, write_text_row, wrap_mpi_gatherv
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
from utilities import wrap_mpi_recv, wrap_mpi_bcast, wrap_mpi_send
import numpy as np
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in range(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x*10000+psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
log_main.add( "%-70s: %d\n"%("Number of neighboring projections", img_per_grp))
log_main.add("...... Finding neighboring projections\n")
log_main.add( "Number of images per group: %d"%img_per_grp)
log_main.add( "Now grouping projections")
proj_angles.sort()
proj_angles_list = np.full((nima, 4), 0.0, dtype=np.float32)
for i in range(nima):
proj_angles_list[i][0] = proj_angles[i][1]
proj_angles_list[i][1] = proj_angles[i][2]
proj_angles_list[i][2] = proj_angles[i][3]
proj_angles_list[i][3] = proj_angles[i][4]
else: proj_angles_list = 0
proj_angles_list = wrap_mpi_bcast(proj_angles_list, main_node, MPI_COMM_WORLD)
proj_angles = []
for i in range(nima):
proj_angles.append([proj_angles_list[i][0], proj_angles_list[i][1], proj_angles_list[i][2], int(proj_angles_list[i][3])])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp, range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
index = {}
for i in range(len(all_proj)): index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("%-70s: %.2f\n"%("Finding neighboring projections lasted [s]", time()-t2))
log_main.add("%-70s: %d\n"%("Number of groups processed on the main node", len(proj_list)))
log_main.add("Grouping projections took: %12.1f [m]"%((time()-t2)/60.))
log_main.add("Number of groups on main node: ", len(proj_list))
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("...... Calculating the stack of 2D variances \n")
# Memory estimation. There are two memory consumption peaks
# peak 1. Compute ave, var;
# peak 2. Var volume reconstruction;
# proj_params = [0.0]*(nima*5)
aveList = []
varList = []
#if nvec > 0: eigList = [[] for i in range(nvec)]
dnumber = len(all_proj)# all neighborhood set for assigned to myid
pnumber = len(proj_list)*2. + img_per_grp # aveList and varList
tnumber = dnumber+pnumber
vol_size2 = nx**3*4.*8/1.e9
vol_size1 = 2.*nnxo**3*4.*8/1.e9
proj_size = nnxo*nnyo*len(proj_list)*4.*2./1.e9 # both aveList and varList
orig_data_size = nnxo*nnyo*4.*tnumber/1.e9
reduced_data_size = nx*nx*4.*tnumber/1.e9
full_data = np.full((number_of_proc, 2), -1., dtype=np.float16)
full_data[myid] = orig_data_size, reduced_data_size
if myid != main_node: wrap_mpi_send(full_data, main_node, MPI_COMM_WORLD)
if myid == main_node:
for iproc in range(number_of_proc):
if iproc != main_node:
dummy = wrap_mpi_recv(iproc, MPI_COMM_WORLD)
full_data[np.where(dummy>-1)] = dummy[np.where(dummy>-1)]
del dummy
mpi_barrier(MPI_COMM_WORLD)
full_data = wrap_mpi_bcast(full_data, main_node, MPI_COMM_WORLD)
# find the CPU with heaviest load
minindx = np.argsort(full_data, 0)
heavy_load_myid = minindx[-1][1]
total_mem = sum(full_data)
if myid == main_node:
if current_window == 0:
log_main.add("Nx: current image size = %d. Decimated by %f from %d"%(nx, current_decimate, nnxo))
else:
log_main.add("Nx: current image size = %d. Windowed to %d, and decimated by %f from %d"%(nx, current_window, current_decimate, nnxo))
log_main.add("Nproj: number of particle images.")
log_main.add("Navg: number of 2D average images.")
log_main.add("Nvar: number of 2D variance images.")
log_main.add("Img_per_grp: user defined image per group for averaging = %d"%img_per_grp)
log_main.add("Overhead: total python overhead memory consumption = %f"%overhead_loading)
log_main.add("Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"%\
(total_mem[1] + overhead_loading))
del full_data
mpi_barrier(MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("Begin reading and preprocessing images on processor. Wait... ")
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
imgdata = [ None for im in range(len(all_proj))]
for index_of_proj in range(len(all_proj)):
#image = get_im(stack, all_proj[index_of_proj])
if( current_window > 0): imgdata[index_of_proj] = fdecimate(window2d(get_im(stack, all_proj[index_of_proj]),current_window,current_window), nx, ny)
else: imgdata[index_of_proj] = fdecimate(get_im(stack, all_proj[index_of_proj]), nx, ny)
if (current_decimate> 0.0 and options.CTF):
ctf = imgdata[index_of_proj].get_attr("ctf")
ctf.apix = ctf.apix/current_decimate
imgdata[index_of_proj].set_attr("ctf", ctf)
if myid == heavy_load_myid and index_of_proj%100 == 0:
log_main.add(" ...... %6.2f%% "%(index_of_proj/float(len(all_proj))*100.))
mpi_barrier(MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("All_proj preprocessing cost %7.2f m"%((time()-ttt)/60.))
log_main.add("Wait untill reading on all CPUs done...")
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
from EMAN2 import Transform
if not options.no_norm:
mask = model_circle(nx/2-2, nx, nx)
if options.CTF:
from utilities import pad
from filter import filt_ctf
from filter import filt_tanl
if myid == heavy_load_myid:
log_main.add("Start computing 2D aveList and varList. Wait...")
ttt = time()
inner=nx//2-4
outer=inner+2
xform_proj_for_2D = [ None for i in range(len(proj_list))]
for i in range(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
dpar = Util.get_transform_params(imgdata[mi], "xform.projection", "spider")
phiM, thetaM, psiM, s2xM, s2yM = dpar["phi"],dpar["theta"],dpar["psi"],-dpar["tx"]*current_decimate,-dpar["ty"]*current_decimate
grp_imgdata = []
for j in range(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
cpar = Util.get_transform_params(imgdata[mj], "xform.projection", "spider")
alpha, sx, sy, mirror = params_3D_2D_NEW(cpar["phi"], cpar["theta"],cpar["psi"], -cpar["tx"]*current_decimate, -cpar["ty"]*current_decimate, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, 180-(phiM - cpar["phi"]), 0.0, 0.0, 1.0)
else:
if mirror == 0: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(phiM- cpar["phi"]), 0.0, 0.0, 1.0)
else: alpha, sx, sy, scale = compose_transform2(alpha, sx, sy, 1.0, -(180-(phiM - cpar["phi"])), 0.0, 0.0, 1.0)
imgdata[mj].set_attr("xform.align2d", Transform({"type":"2D","alpha":alpha,"tx":sx,"ty":sy,"mirror":mirror,"scale":1.0}))
grp_imgdata.append(imgdata[mj])
if not options.no_norm:
for k in range(img_per_grp):
ave, std, minn, maxx = Util.infomask(grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
if options.fl > 0.0:
for k in range(img_per_grp):
grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
# Because of background issues, only linear option works.
if options.CTF: ave, var = aves_wiener(grp_imgdata, SNR = 1.0e5, interpolation_method = "linear")
else: ave, var = ave_var(grp_imgdata)
# Switch to std dev
# threshold is not really needed,it is just in case due to numerical accuracy something turns out negative.
var = square_root(threshold(var))
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0]
'''
if nvec > 0:
eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
for k in range(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
'''
if (myid == heavy_load_myid) and (i%100 == 0):
log_main.add(" ......%6.2f%% "%(i/float(len(proj_list))*100.))
del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index
if not options.no_norm: del mask
if myid == main_node: del tab
# At this point, all averages and variances are computed
mpi_barrier(MPI_COMM_WORLD)
if (myid == heavy_load_myid):
log_main.add("Computing aveList and varList took %12.1f [m]"%((time()-ttt)/60.))
xform_proj_for_2D = wrap_mpi_gatherv(xform_proj_for_2D, main_node, MPI_COMM_WORLD)
if (myid == main_node):
write_text_row(xform_proj_for_2D, os.path.join(current_output_dir, "params.txt"))
del xform_proj_for_2D
mpi_barrier(MPI_COMM_WORLD)
if options.ave2D:
from fundamentals import fpol
from applications import header
if myid == main_node:
log_main.add("Compute ave2D ... ")
km = 0
for i in range(number_of_proc):
if i == main_node :
for im in range(len(aveList)):
aveList[im].write_image(os.path.join(current_output_dir, options.ave2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im+i+70000)
"""
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol=fpol(ave, nx, nx,1)
tmpvol.write_image(os.path.join(current_output_dir, options.ave2D), km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in range(len(aveList)):
send_EMData(aveList[im], main_node,im+myid+70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
"""
if myid == main_node:
header(os.path.join(current_output_dir, options.ave2D), params='xform.projection', fimport = os.path.join(current_output_dir, "params.txt"))
mpi_barrier(MPI_COMM_WORLD)
if options.ave3D:
from fundamentals import fpol
t5 = time()
if myid == main_node: log_main.add("Reconstruct ave3D ... ")
ave3D = recons3d_4nn_MPI(myid, aveList, symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
if current_decimate != 1.0: ave3D = resample(ave3D, 1./current_decimate)
ave3D = fpol(ave3D, nnxo, nnxo, nnxo) # always to the orignal image size
set_pixel_size(ave3D, 1.0)
ave3D.write_image(os.path.join(current_output_dir, options.ave3D))
log_main.add("Ave3D reconstruction took %12.1f [m]"%((time()-t5)/60.0))
log_main.add("%-70s: %s\n"%("The reconstructed ave3D is saved as ", options.ave3D))
mpi_barrier(MPI_COMM_WORLD)
del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList
'''
if nvec > 0:
for k in range(nvec):
if myid == main_node:log_main.add("Reconstruction eigenvolumes", k)
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image(os.path.join(options.outpout_dir, "eig3d_%03d.hdf"%(k, ITER)))
Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in range(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
u = int(u[0])
log_main.add(" Eigenvector: ",k," number changed ",int(icont[0]))
else: u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
'''
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
from applications import header
if myid == main_node:
log_main.add("Compute var2D...")
km = 0
for i in range(number_of_proc):
if i == main_node :
for im in range(len(varList)):
tmpvol=fpol(varList[im], nx, nx,1)
tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im+i+70000)
tmpvol=fpol(ave, nx, nx,1)
tmpvol.write_image(os.path.join(current_output_dir, options.var2D), km)
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in range(len(varList)):
send_EMData(varList[im], main_node, im+myid+70000)# What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
from applications import header
header(os.path.join(current_output_dir, options.var2D), params = 'xform.projection',fimport = os.path.join(current_output_dir, "params.txt"))
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node: log_main.add("Reconstruct var3D ...")
t6 = time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid, varList, symmetry = options.sym, npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
if current_decimate != 1.0: res = resample(res, 1./current_decimate)
res = fpol(res, nnxo, nnxo, nnxo)
set_pixel_size(res, 1.0)
res.write_image(os.path.join(current_output_dir, options.var3D))
log_main.add("%-70s: %s\n"%("The reconstructed var3D is saved as ", options.var3D))
log_main.add("Var3D reconstruction took %f12.1 [m]"%((time()-t6)/60.0))
log_main.add("Total computation time %f12.1 [m]"%((time()-t0)/60.0))
log_main.add("sx3dvariability finishes")
from mpi import mpi_finalize
mpi_finalize()
if RUNNING_UNDER_MPI: global_def.MPI = False
global_def.BATCH = False
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# ========================================================================================
# Imports
# ========================================================================================
# Python Standard Libraries
from __future__ import print_function
import sys
import os
import argparse
from utilities import get_im, write_text_file
# SPHIRE/EMAN2 Libraries
import global_def
from global_def import *
#=========================================================================================
input_isac_class_avgs_path = sys.argv[1] # only command line argument is saved class averages file
# Retrieve original particle IDs of member particles listed in ISAC class average stack
n_img_processed = EMUtil.get_image_count(input_isac_class_avgs_path)
isac_substack_particle_id_list = []
for i_img in xrange(n_img_processed):
isac_substack_particle_id_list += get_im(input_isac_class_avgs_path, i_img).get_attr('members')
isac_substack_particle_id_list.sort()
# Save the substack particle id list
isac_substack_particle_id_list_file_path = 'isac_substack_particle_id_list.txt'
write_text_file(isac_substack_particle_id_list, isac_substack_particle_id_list_file_path)
def do_volume_mrk02(ref_data):
"""
data - projections (scattered between cpus) or the volume. If volume, just do the volume processing
options - the same for all cpus
return - volume the same for all cpus
"""
from EMAN2 import Util
from mpi import mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD
from filter import filt_table
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import bcast_EMData_to_all, bcast_number_to_all, model_blank
from fundamentals import rops_table, fftip, fft
import types
# Retrieve the function specific input arguments from ref_data
data = ref_data[0]
Tracker = ref_data[1]
iter = ref_data[2]
mpi_comm = ref_data[3]
# # For DEBUG
# print "Type of data %s" % (type(data))
# print "Type of Tracker %s" % (type(Tracker))
# print "Type of iter %s" % (type(iter))
# print "Type of mpi_comm %s" % (type(mpi_comm))
if (mpi_comm == None): mpi_comm = MPI_COMM_WORLD
myid = mpi_comm_rank(mpi_comm)
nproc = mpi_comm_size(mpi_comm)
try:
local_filter = Tracker["local_filter"]
except:
local_filter = False
#=========================================================================
# volume reconstruction
if (type(data) == types.ListType):
if Tracker["constants"]["CTF"]:
vol = recons3d_4nn_ctf_MPI(myid, data, Tracker["constants"]["snr"], \
symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm, smearstep = Tracker["smearstep"])
else:
vol = recons3d_4nn_MPI (myid, data,\
symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm)
else:
vol = data
if myid == 0:
from morphology import threshold
from filter import filt_tanl, filt_btwl
from utilities import model_circle, get_im
import types
nx = vol.get_xsize()
if (Tracker["constants"]["mask3D"] == None):
mask3D = model_circle(
int(Tracker["constants"]["radius"] * float(nx) /
float(Tracker["constants"]["nnxo"]) + 0.5), nx, nx, nx)
elif (Tracker["constants"]["mask3D"] == "auto"):
from utilities import adaptive_mask
mask3D = adaptive_mask(vol)
else:
if (type(Tracker["constants"]["mask3D"]) == types.StringType):
mask3D = get_im(Tracker["constants"]["mask3D"])
else:
mask3D = (Tracker["constants"]["mask3D"]).copy()
nxm = mask3D.get_xsize()
if (nx != nxm):
from fundamentals import rot_shift3D
mask3D = Util.window(
rot_shift3D(mask3D, scale=float(nx) / float(nxm)), nx, nx,
nx)
nxm = mask3D.get_xsize()
assert (nx == nxm)
stat = Util.infomask(vol, mask3D, False)
vol -= stat[0]
Util.mul_scalar(vol, 1.0 / stat[1])
vol = threshold(vol)
Util.mul_img(vol, mask3D)
if (Tracker["PWadjustment"]):
from utilities import read_text_file, write_text_file
rt = read_text_file(Tracker["PWadjustment"])
fftip(vol)
ro = rops_table(vol)
# Here unless I am mistaken it is enough to take the beginning of the reference pw.
for i in xrange(1, len(ro)):
ro[i] = (rt[i] / ro[i])**Tracker["upscale"]
#write_text_file(rops_table(filt_table( vol, ro),1),"foo.txt")
if Tracker["constants"]["sausage"]:
ny = vol.get_ysize()
y = float(ny)
from math import exp
for i in xrange(len(ro)): ro[i] *= \
(1.0+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.10)/0.025)**2)+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.215)/0.025)**2))
if local_filter:
# skip low-pass filtration
vol = fft(filt_table(vol, ro))
else:
if (type(Tracker["lowpass"]) == types.ListType):
vol = fft(
filt_table(filt_table(vol, Tracker["lowpass"]), ro))
else:
vol = fft(
filt_table(
filt_tanl(vol, Tracker["lowpass"],
Tracker["falloff"]), ro))
del ro
else:
if Tracker["constants"]["sausage"]:
ny = vol.get_ysize()
y = float(ny)
ro = [0.0] * (ny // 2 + 2)
from math import exp
for i in xrange(len(ro)): ro[i] = \
(1.0+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.10)/0.025)**2)+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.215)/0.025)**2))
fftip(vol)
filt_table(vol, ro)
del ro
if not local_filter:
if (type(Tracker["lowpass"]) == types.ListType):
vol = filt_table(vol, Tracker["lowpass"])
else:
vol = filt_tanl(vol, Tracker["lowpass"],
Tracker["falloff"])
if Tracker["constants"]["sausage"]: vol = fft(vol)
if local_filter:
from morphology import binarize
if (myid == 0): nx = mask3D.get_xsize()
else: nx = 0
nx = bcast_number_to_all(nx, source_node=0)
# only main processor needs the two input volumes
if (myid == 0):
mask = binarize(mask3D, 0.5)
locres = get_im(Tracker["local_filter"])
lx = locres.get_xsize()
if (lx != nx):
if (lx < nx):
from fundamentals import fdecimate, rot_shift3D
mask = Util.window(
rot_shift3D(mask, scale=float(lx) / float(nx)), lx, lx,
lx)
vol = fdecimate(vol, lx, lx, lx)
else:
ERROR("local filter cannot be larger than input volume",
"user function", 1)
stat = Util.infomask(vol, mask, False)
vol -= stat[0]
Util.mul_scalar(vol, 1.0 / stat[1])
else:
lx = 0
locres = model_blank(1, 1, 1)
vol = model_blank(1, 1, 1)
lx = bcast_number_to_all(lx, source_node=0)
if (myid != 0): mask = model_blank(lx, lx, lx)
bcast_EMData_to_all(mask, myid, 0, comm=mpi_comm)
from filter import filterlocal
vol = filterlocal(locres, vol, mask, Tracker["falloff"], myid, 0,
nproc)
if myid == 0:
if (lx < nx):
from fundamentals import fpol
vol = fpol(vol, nx, nx, nx)
vol = threshold(vol)
vol = filt_btwl(vol, 0.38, 0.5) # This will have to be corrected.
Util.mul_img(vol, mask3D)
del mask3D
# vol.write_image('toto%03d.hdf'%iter)
else:
vol = model_blank(nx, nx, nx)
else:
if myid == 0:
#from utilities import write_text_file
#write_text_file(rops_table(vol,1),"goo.txt")
stat = Util.infomask(vol, mask3D, False)
vol -= stat[0]
Util.mul_scalar(vol, 1.0 / stat[1])
vol = threshold(vol)
vol = filt_btwl(vol, 0.38, 0.5) # This will have to be corrected.
Util.mul_img(vol, mask3D)
del mask3D
# vol.write_image('toto%03d.hdf'%iter)
# broadcast volume
bcast_EMData_to_all(vol, myid, 0, comm=mpi_comm)
#=========================================================================
return vol
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = progname + """ firstvolume secondvolume maskfile outputfile --wn --step --cutoff --radius --fsc --res_overall --out_ang_res --apix --MPI
Compute local resolution in real space within area outlined by the maskfile and within regions wn x wn x wn
"""
parser = optparse.OptionParser(usage, version=global_def.SPARXVERSION)
parser.add_option(
"--wn",
type="int",
default=7,
help=
"Size of window within which local real-space FSC is computed. (default 7)"
)
parser.add_option(
"--step",
type="float",
default=1.0,
help="Shell step in Fourier size in pixels. (default 1.0)")
parser.add_option("--cutoff",
type="float",
default=0.5,
help="Resolution cut-off for FSC. (default 0.5)")
parser.add_option(
"--radius",
type="int",
default=-1,
help=
"If there is no maskfile, sphere with r=radius will be used. By default, the radius is nx/2-wn (default -1)"
)
parser.add_option(
"--fsc",
type="string",
default=None,
help=
"Save overall FSC curve (might be truncated). By default, the program does not save the FSC curve. (default none)"
)
parser.add_option(
"--res_overall",
type="float",
default=-1.0,
help=
"Overall resolution at the cutoff level estimated by the user [abs units]. (default None)"
)
parser.add_option(
"--out_ang_res",
action="store_true",
default=False,
help=
"Additionally creates a local resolution file in Angstroms. (default False)"
)
parser.add_option(
"--apix",
type="float",
default=1.0,
help=
"Pixel size in Angstrom. Effective only with --out_ang_res options. (default 1.0)"
)
parser.add_option("--MPI",
action="store_true",
default=False,
help="Use MPI version.")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 3 or len(args) > 4:
print("See usage " + usage)
sys.exit()
if global_def.CACHE_DISABLE:
utilities.disable_bdb_cache()
res_overall = options.res_overall
if options.MPI:
sys.argv = mpi.mpi_init(len(sys.argv), sys.argv)
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
global_def.MPI = True
cutoff = options.cutoff
nk = int(options.wn)
if (myid == main_node):
#print sys.argv
vi = utilities.get_im(sys.argv[1])
ui = utilities.get_im(sys.argv[2])
nx = vi.get_xsize()
ny = vi.get_ysize()
nz = vi.get_zsize()
dis = [nx, ny, nz]
else:
dis = [0, 0, 0, 0]
global_def.BATCH = True
dis = utilities.bcast_list_to_all(dis, myid, source_node=main_node)
if (myid != main_node):
nx = int(dis[0])
ny = int(dis[1])
nz = int(dis[2])
vi = utilities.model_blank(nx, ny, nz)
ui = utilities.model_blank(nx, ny, nz)
if len(args) == 3:
m = utilities.model_circle((min(nx, ny, nz) - nk) // 2, nx, ny, nz)
outvol = args[2]
elif len(args) == 4:
if (myid == main_node):
m = morphology.binarize(utilities.get_im(args[2]), 0.5)
else:
m = utilities.model_blank(nx, ny, nz)
outvol = args[3]
utilities.bcast_EMData_to_all(m, myid, main_node)
"""Multiline Comment0"""
freqvol, resolut = statistics.locres(vi, ui, m, nk, cutoff,
options.step, myid, main_node,
number_of_proc)
if (myid == 0):
# Remove outliers based on the Interquartile range
output_volume(freqvol, resolut, options.apix, outvol, options.fsc,
options.out_ang_res, nx, ny, nz, res_overall)
mpi.mpi_finalize()
else:
cutoff = options.cutoff
vi = utilities.get_im(args[0])
ui = utilities.get_im(args[1])
nn = vi.get_xsize()
nk = int(options.wn)
if len(args) == 3:
m = utilities.model_circle((nn - nk) // 2, nn, nn, nn)
outvol = args[2]
elif len(args) == 4:
m = morphology.binarize(utilities.get_im(args[2]), 0.5)
outvol = args[3]
mc = utilities.model_blank(nn, nn, nn, 1.0) - m
vf = fundamentals.fft(vi)
uf = fundamentals.fft(ui)
"""Multiline Comment1"""
lp = int(nn / 2 / options.step + 0.5)
step = 0.5 / lp
freqvol = utilities.model_blank(nn, nn, nn)
resolut = []
for i in range(1, lp):
fl = step * i
fh = fl + step
#print(lp,i,step,fl,fh)
v = fundamentals.fft(filter.filt_tophatb(vf, fl, fh))
u = fundamentals.fft(filter.filt_tophatb(uf, fl, fh))
tmp1 = EMAN2_cppwrap.Util.muln_img(v, v)
tmp2 = EMAN2_cppwrap.Util.muln_img(u, u)
do = EMAN2_cppwrap.Util.infomask(
morphology.square_root(
morphology.threshold(
EMAN2_cppwrap.Util.muln_img(tmp1, tmp2))), m, True)[0]
tmp3 = EMAN2_cppwrap.Util.muln_img(u, v)
dp = EMAN2_cppwrap.Util.infomask(tmp3, m, True)[0]
resolut.append([i, (fl + fh) / 2.0, dp / do])
tmp1 = EMAN2_cppwrap.Util.box_convolution(tmp1, nk)
tmp2 = EMAN2_cppwrap.Util.box_convolution(tmp2, nk)
tmp3 = EMAN2_cppwrap.Util.box_convolution(tmp3, nk)
EMAN2_cppwrap.Util.mul_img(tmp1, tmp2)
tmp1 = morphology.square_root(morphology.threshold(tmp1))
EMAN2_cppwrap.Util.mul_img(tmp1, m)
EMAN2_cppwrap.Util.add_img(tmp1, mc)
EMAN2_cppwrap.Util.mul_img(tmp3, m)
EMAN2_cppwrap.Util.add_img(tmp3, mc)
EMAN2_cppwrap.Util.div_img(tmp3, tmp1)
EMAN2_cppwrap.Util.mul_img(tmp3, m)
freq = (fl + fh) / 2.0
bailout = True
for x in range(nn):
for y in range(nn):
for z in range(nn):
if (m.get_value_at(x, y, z) > 0.5):
if (freqvol.get_value_at(x, y, z) == 0.0):
if (tmp3.get_value_at(x, y, z) < cutoff):
freqvol.set_value_at(x, y, z, freq)
bailout = False
else:
bailout = False
if (bailout): break
#print(len(resolut))
# remove outliers
output_volume(freqvol, resolut, options.apix, outvol, options.fsc,
options.out_ang_res, nx, ny, nz, res_overall)
def helicalshiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1):
from applications import MPI_start_end
from utilities import model_circle, model_blank, get_image, peak_search, get_im, pad
from utilities import reduce_EMData_to_root, bcast_EMData_to_all, send_attr_dict, file_type, bcast_number_to_all, bcast_list_to_all
from statistics import varf2d_MPI
from fundamentals import fft, ccf, rot_shift3D, rot_shift2D, fshift
from utilities import get_params2D, set_params2D, chunks_distribution
from utilities import print_msg, print_begin_msg, print_end_msg
import os
import sys
from mpi import mpi_init, mpi_comm_size, mpi_comm_rank, MPI_COMM_WORLD
from mpi import mpi_reduce, mpi_bcast, mpi_barrier, mpi_gatherv
from mpi import MPI_SUM, MPI_FLOAT, MPI_INT
from time import time
from pixel_error import ordersegments
from math import sqrt, atan2, tan, pi
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("helical-shiftali_MPI")
max_iter=int(maxit)
if( myid == main_node):
infils = EMUtil.get_all_attributes(stack, "filament")
ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord')
filaments = ordersegments(infils, ptlcoords)
total_nfils = len(filaments)
inidl = [0]*total_nfils
for i in xrange(total_nfils): inidl[i] = len(filaments[i])
linidl = sum(inidl)
nima = linidl
tfilaments = []
for i in xrange(total_nfils): tfilaments += filaments[i]
del filaments
else:
total_nfils = 0
linidl = 0
total_nfils = bcast_number_to_all(total_nfils, source_node = main_node)
if myid != main_node:
inidl = [-1]*total_nfils
inidl = bcast_list_to_all(inidl, myid, source_node = main_node)
linidl = bcast_number_to_all(linidl, source_node = main_node)
if myid != main_node:
tfilaments = [-1]*linidl
tfilaments = bcast_list_to_all(tfilaments, myid, source_node = main_node)
filaments = []
iendi = 0
for i in xrange(total_nfils):
isti = iendi
iendi = isti+inidl[i]
filaments.append(tfilaments[isti:iendi])
del tfilaments,inidl
if myid == main_node:
print_msg( "total number of filaments: %d"%total_nfils)
if total_nfils< nproc:
ERROR('number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'%(nproc, total_nfils), "ehelix_MPI", 1,myid)
# balanced load
temp = chunks_distribution([[len(filaments[i]), i] for i in xrange(len(filaments))], nproc)[myid:myid+1][0]
filaments = [filaments[temp[i][1]] for i in xrange(len(temp))]
nfils = len(filaments)
#filaments = [[0,1]]
#print "filaments",filaments
list_of_particles = []
indcs = []
k = 0
for i in xrange(nfils):
list_of_particles += filaments[i]
k1 = k+len(filaments[i])
indcs.append([k,k1])
k = k1
data = EMData.read_images(stack, list_of_particles)
ldata = len(data)
print "ldata=", ldata
nx = data[0].get_xsize()
ny = data[0].get_ysize()
if maskfile == None:
mrad = min(nx, ny)//2-2
mask = pad( model_blank(2*mrad+1, ny, 1, 1.0), nx, ny, 1, 0.0)
else:
mask = get_im(maskfile)
# apply initial xform.align2d parameters stored in header
init_params = []
for im in xrange(ldata):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'], p['mirror'], p['scale'])
if CTF:
from filter import filt_ctf
from morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
ctf_abs_sum = None
from utilities import info
for im in xrange(ldata):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
qctf = data[im].get_attr("ctf_applied")
if qctf == 0:
data[im] = filt_ctf(fft(data[im]), ctf_params)
data[im].set_attr('ctf_applied', 1)
elif qctf != 1:
ERROR('Incorrectly set qctf flag', "helicalshiftali_MPI", 1,myid)
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
else: data[im] = fft(data[im])
del list_of_particles
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
tsnr = 1./snr
for i in xrange(0,nx+2,2):
for j in xrange(ny):
temp.set_value_at(i,j,tsnr)
temp.set_value_at(i+1,j,0.0)
#info(ctf_2_sum)
Util.add_img(ctf_2_sum, temp)
#info(ctf_2_sum)
del temp
total_iter = 0
shift_x = [0.0]*ldata
for Iter in xrange(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n"%(total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in xrange(ldata):
Util.add_img(avg, fshift(data[im], shift_x[im]))
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF: tavg = Util.divn_filter(avg, ctf_2_sum)
else: tavg = Util.mult_scalar(avg, 1.0/float(nima))
else:
tavg = model_blank(nx,ny)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask( tavg, mask, False )
tavg -= stat[0]
Util.mul_img(tavg, mask)
tavg.write_image("tavg.hdf",Iter)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
tavg = fft(tavg)
sx_sum = 0.0
nxc = nx//2
for ifil in xrange(nfils):
"""
# Calculate filament average
avg = EMData(nx, ny, 1, False)
filnima = 0
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
Util.add_img(avg, data[im])
filnima += 1
tavg = Util.mult_scalar(avg, 1.0/float(filnima))
"""
# Calculate 1D ccf between each segment and filament average
nsegms = indcs[ifil][1]-indcs[ifil][0]
ctx = [None]*nsegms
pcoords = [None]*nsegms
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
ctx[im-indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx, 1)
pcoords[im-indcs[ifil][0]] = data[im].get_attr('ptcl_source_coord')
#ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0])
#print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True)
# search for best x-shift
cents = nsegms//2
dst = sqrt(max((pcoords[cents][0] - pcoords[0][0])**2 + (pcoords[cents][1] - pcoords[0][1])**2, (pcoords[cents][0] - pcoords[-1][0])**2 + (pcoords[cents][1] - pcoords[-1][1])**2))
maxincline = atan2(ny//2-2-float(search_rng),dst)
kang = int(dst*tan(maxincline)+0.5)
#print " settings ",nsegms,cents,dst,search_rng,maxincline,kang
# ## C code for alignment. @ming
results = [0.0]*3;
results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline, kang, search_rng,nxc)
sib = int(results[0])
bang = results[1]
qm = results[2]
#print qm, sib, bang
# qm = -1.e23
#
# for six in xrange(-search_rng, search_rng+1,1):
# q0 = ctx[cents].get_value_at(six+nxc)
# for incline in xrange(kang+1):
# qt = q0
# qu = q0
# if(kang>0): tang = tan(maxincline/kang*incline)
# else: tang = 0.0
# for kim in xrange(cents+1,nsegms):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# #print " A ", ifil,six,incline,kim,xl,ixl,dxl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# for kim in xrange(cents):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# if( qt > qm ):
# qm = qt
# sib = six
# bang = tang
# if( qu > qm ):
# qm = qu
# sib = six
# bang = -tang
#if incline == 0: print "incline = 0 ",six,tang,qt,qu
#print qm,six,sib,bang
#print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True)
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
kim = im-indcs[ifil][0]
dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
if(kim < cents): xl = -dst*bang+sib
else: xl = dst*bang+sib
shift_x[im] = xl
# Average shift
sx_sum += shift_x[indcs[ifil][0]+cents]
# #print myid,sx_sum,total_nfils
sx_sum = mpi_reduce(sx_sum, 1, MPI_FLOAT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
sx_sum = float(sx_sum[0])/total_nfils
print_msg("Average shift %6.2f\n"%(sx_sum))
else:
sx_sum = 0.0
sx_sum = 0.0
sx_sum = bcast_number_to_all(sx_sum, source_node = main_node)
for im in xrange(ldata):
shift_x[im] -= sx_sum
#print " %3d %6.3f"%(im,shift_x[im])
#exit()
# combine shifts found with the original parameters
for im in xrange(ldata):
t1 = Transform()
##import random
##shix=random.randint(-10, 10)
##t1.set_params({"type":"2D","tx":shix})
t1.set_params({"type":"2D","tx":shift_x[im]})
# combine t0 and t1
tt = t1*init_params[im]
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi_barrier(MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from utilities import file_type
if(file_type(stack) == "bdb"):
from utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata, nproc)
else:
from utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc)
else: send_attr_dict(main_node, data, par_str, 0, ldata)
if myid == main_node: print_end_msg("helical-shiftali_MPI")
def do_volume_mrk03(ref_data):
"""
data - projections (scattered between cpus) or the volume. If volume, just do the volume processing
options - the same for all cpus
return - volume the same for all cpus
"""
from EMAN2 import Util
from mpi import mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD
from filter import filt_table
from reconstruction import recons3d_4nn_MPI, recons3d_4nnw_MPI # recons3d_4nn_ctf_MPI
from utilities import bcast_EMData_to_all, bcast_number_to_all, model_blank
from fundamentals import rops_table, fftip, fft
import types
# Retrieve the function specific input arguments from ref_data
data = ref_data[0]
Tracker = ref_data[1]
iter = ref_data[2]
mpi_comm = ref_data[3]
# # For DEBUG
# print "Type of data %s" % (type(data))
# print "Type of Tracker %s" % (type(Tracker))
# print "Type of iter %s" % (type(iter))
# print "Type of mpi_comm %s" % (type(mpi_comm))
if(mpi_comm == None): mpi_comm = MPI_COMM_WORLD
myid = mpi_comm_rank(mpi_comm)
nproc = mpi_comm_size(mpi_comm)
try: local_filter = Tracker["local_filter"]
except: local_filter = False
#=========================================================================
# volume reconstruction
if( type(data) == types.ListType ):
if Tracker["constants"]["CTF"]:
#vol = recons3d_4nn_ctf_MPI(myid, data, Tracker["constants"]["snr"], \
# symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm, smearstep = Tracker["smearstep"])
vol = recons3d_4nnw_MPI(myid, data, Tracker["bckgnoise"], Tracker["constants"]["snr"], \
symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm, smearstep = Tracker["smearstep"])
else:
vol = recons3d_4nn_MPI (myid, data,\
symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm)
else:
vol = data
if myid == 0:
from morphology import threshold
from filter import filt_tanl, filt_btwl
from utilities import model_circle, get_im
import types
nx = vol.get_xsize()
if(Tracker["constants"]["mask3D"] == None):
mask3D = model_circle(int(Tracker["constants"]["radius"]*float(nx)/float(Tracker["constants"]["nnxo"])+0.5), nx, nx, nx)
elif(Tracker["constants"]["mask3D"] == "auto"):
from utilities import adaptive_mask
mask3D = adaptive_mask(vol)
else:
if( type(Tracker["constants"]["mask3D"]) == types.StringType ): mask3D = get_im(Tracker["constants"]["mask3D"])
else: mask3D = (Tracker["constants"]["mask3D"]).copy()
nxm = mask3D.get_xsize()
if( nx != nxm):
from fundamentals import rot_shift3D
mask3D = Util.window(rot_shift3D(mask3D,scale=float(nx)/float(nxm)),nx,nx,nx)
nxm = mask3D.get_xsize()
assert(nx == nxm)
stat = Util.infomask(vol, mask3D, False)
vol -= stat[0]
Util.mul_scalar(vol, 1.0/stat[1])
vol = threshold(vol)
Util.mul_img(vol, mask3D)
if not local_filter:
if( type(Tracker["lowpass"]) == types.ListType ):
vol = filt_table(vol, Tracker["lowpass"])
else:
vol = filt_tanl(vol, Tracker["lowpass"], Tracker["falloff"])
if local_filter:
from morphology import binarize
if(myid == 0): nx = mask3D.get_xsize()
else: nx = 0
nx = bcast_number_to_all(nx, source_node = 0)
# only main processor needs the two input volumes
if(myid == 0):
mask = binarize(mask3D, 0.5)
locres = get_im(Tracker["local_filter"])
lx = locres.get_xsize()
if(lx != nx):
if(lx < nx):
from fundamentals import fdecimate, rot_shift3D
mask = Util.window(rot_shift3D(mask,scale=float(lx)/float(nx)),lx,lx,lx)
vol = fdecimate(vol, lx,lx,lx)
else: ERROR("local filter cannot be larger than input volume","user function",1)
stat = Util.infomask(vol, mask, False)
vol -= stat[0]
Util.mul_scalar(vol, 1.0/stat[1])
else:
lx = 0
locres = model_blank(1,1,1)
vol = model_blank(1,1,1)
lx = bcast_number_to_all(lx, source_node = 0)
if( myid != 0 ): mask = model_blank(lx,lx,lx)
bcast_EMData_to_all(mask, myid, 0, comm=mpi_comm)
from filter import filterlocal
vol = filterlocal( locres, vol, mask, Tracker["falloff"], myid, 0, nproc)
if myid == 0:
if(lx < nx):
from fundamentals import fpol
vol = fpol(vol, nx,nx,nx)
vol = threshold(vol)
Util.mul_img(vol, mask3D)
del mask3D
# vol.write_image('toto%03d.hdf'%iter)
else:
vol = model_blank(nx,nx,nx)
"""
else:
if myid == 0:
#from utilities import write_text_file
#write_text_file(rops_table(vol,1),"goo.txt")
stat = Util.infomask(vol, mask3D, False)
vol -= stat[0]
Util.mul_scalar(vol, 1.0/stat[1])
vol = threshold(vol)
Util.mul_img(vol, mask3D)
del mask3D
# vol.write_image('toto%03d.hdf'%iter)
"""
# broadcast volume
bcast_EMData_to_all(vol, myid, 0, comm=mpi_comm)
#=========================================================================
return vol
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ Input Output [options]
Generate three micrographs, each micrograph contains one projection of a long filament.
Input: Reference Volume, output directory
Output: Three micrographs stored in output directory
sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84
Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
Generate run through example script for helicon
sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
"""
parser = OptionParser(usage, version=SPARXVERSION)
# helicise the Atom coordinates
# generate micrographs of helical filament
parser.add_option(
"--generate_micrograph",
action="store_true",
default=False,
help=
"Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="Use CTF correction")
parser.add_option("--apix",
type="float",
default=-1,
help="pixel size in Angstroms")
parser.add_option(
"--rand_seed",
type="int",
default=14567,
help=
"the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs."
)
parser.add_option("--Cs",
type="float",
default=2.0,
help="Microscope Cs (spherical aberation)")
parser.add_option("--voltage",
type="float",
default=200.0,
help="Microscope voltage in KV")
parser.add_option("--ac",
type="float",
default=10.0,
help="Amplitude contrast (percentage, default=10)")
parser.add_option("--nonoise",
action="store_true",
default=False,
help="Do not add noise to the micrograph.")
# generate initial volume
parser.add_option("--generate_noisycyl",
action="store_true",
default=False,
help="Generate initial volume of noisy cylinder.")
parser.add_option(
"--boxsize",
type="string",
default="100,100,200",
help=
"String containing x , y, z dimensions (separated by comma) in pixels")
parser.add_option("--rad",
type="int",
default=35,
help="Radius of initial volume in pixels")
# generate 2D mask
parser.add_option("--generate_mask",
action="store_true",
default=False,
help="Generate 2D rectangular mask.")
parser.add_option(
"--masksize",
type="string",
default="200,200",
help=
"String containing x and y dimensions (separated by comma) in pixels")
parser.add_option("--maskwidth",
type="int",
default=60,
help="Width of rectangular mask")
# Apply 2D alignment parameters to input stack and output new images to output stack
parser.add_option(
"--applyparams",
action="store_true",
default=False,
help=
"Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack"
)
# Generate run script
parser.add_option("--generate_script",
action="store_true",
default=False,
help="Generate script for helicon run through example")
parser.add_option("--filename",
type="string",
default="runhelicon",
help="Name of run script to generate")
parser.add_option("--seg_ny",
type="int",
default=180,
help="y-dimension of segment used for refinement")
parser.add_option(
"--ptcl_dist",
type="int",
default=15,
help=
"Distance in pixels between adjacent segments windowed from same filament"
)
parser.add_option(
"--fract",
type="float",
default=0.35,
help="Fraction of the volume used for applying helical symmetry.")
(options, args) = parser.parse_args()
if len(args) > 3:
print("usage: " + usage)
print("Please run '" + progname + " -h' for detailed options")
else:
if options.generate_script:
generate_runscript(options.filename, options.seg_ny,
options.ptcl_dist, options.fract)
if options.generate_micrograph:
if options.apix <= 0:
print("Please enter pixel size.")
sys.exit()
generate_helimic(args[0], args[1], options.apix, options.CTF,
options.Cs, options.voltage, options.ac,
options.nonoise, options.rand_seed)
if options.generate_noisycyl:
from utilities import model_cylinder, model_gauss_noise
outvol = args[0]
boxdims = options.boxsize.split(',')
if len(boxdims) < 1 or len(boxdims) > 3:
print(
"Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'"
)
sys.exit()
nx = int(boxdims[0])
if len(boxdims) == 1:
ny = nx
nz = nx
else:
ny = int(boxdims[1])
if len(boxdims) == 3:
nz = int(boxdims[2])
(model_cylinder(options.rad, nx, ny, nz) *
model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol)
if options.generate_mask:
from utilities import model_blank, pad
outvol = args[0]
maskdims = options.masksize.split(',')
if len(maskdims) < 1 or len(maskdims) > 2:
print(
"Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'"
)
sys.exit()
nx = int(maskdims[0])
if len(maskdims) == 1:
ny = nx
else:
ny = int(maskdims[1])
mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1,
0.0)
mask.write_image(outvol)
if options.applyparams:
from utilities import get_im, get_params2D, set_params2D
from fundamentals import cyclic_shift
stack = args[0]
newstack = args[1]
mask = get_im(args[2])
nima = EMUtil.get_image_count(stack)
for im in range(nima):
prj = get_im(stack, im)
alpha, sx, sy, mirror, scale = get_params2D(prj)
prj = cyclic_shift(prj, int(sx))
set_params2D(prj, [0.0, 0., 0.0, 0, 1])
stat = Util.infomask(prj, mask, False)
prj = old_div((prj - stat[0]), stat[1])
ctf_params = prj.get_attr("ctf")
prj.set_attr('ctf_applied', 0)
prj.write_image(newstack, im)
def runcheck(classavgstack, recon, outdir, inangles=None, selectdoc=None, displayYN=False,
projstack='proj.hdf', outangles='angles.txt', outstack='comp-proj-reproj.hdf', normstack='comp-proj-reproj-norm.hdf'):
print
# Check if inputs exist
check(classavgstack)
check(recon)
# Create directory if it doesn't exist
if not os.path.isdir(outdir):
os.makedirs(outdir) # os.mkdir() can only operate one directory deep
print("mkdir -p %s" % outdir)
# Expand path for outputs
projstack = os.path.join(outdir, projstack)
outangles = os.path.join(outdir, outangles)
outstack = os.path.join(outdir, outstack)
normstack = os.path.join(outdir, normstack)
# Get number of images
nimg0 = EMUtil.get_image_count(classavgstack)
#print("nimg0: %s" % nimg0)
# In case class averages include discarded images, apply selection file
if selectdoc:
goodavgs, extension = os.path.splitext(classavgstack)
newclasses = goodavgs + "_kept" + extension
# e2proc2d appends to existing files, so rename existing output
if os.path.exists(newclasses):
renamefile = newclasses + '.bak'
os.rename(newclasses, renamefile)
print("mv %s %s" % (newclasses, renamefile))
cmd7="e2proc2d.py %s %s --list=%s" % (classavgstack, newclasses, selectdoc)
print cmd7
os.system(cmd7)
# Update class-averages
classavgstack = newclasses
# Import Euler angles
if inangles:
cmd6="sxheader.py %s --params=xform.projection --import=%s" % (classavgstack, inangles)
print cmd6
header(classavgstack, 'xform.projection', fimport=inangles)
cmd1="sxheader.py %s --params=xform.projection --export=%s" % (classavgstack, outangles)
print cmd1
#os.system(cmd1)
try:
header(classavgstack, 'xform.projection', fexport=outangles)
except RuntimeError:
print("\nERROR!! No projection angles found in class-average stack header!\n")
exit()
cmd2="sxproject3d.py %s %s --angles=%s" % (recon, projstack, outangles)
print cmd2
#os.system(cmd2)
project3d(recon, stack=projstack, listagls=outangles)
imgcounter = 0 # montage will have double the number of images as number of class-averages
result=[]
# Number of images may have changed
nimg1 = EMUtil.get_image_count(classavgstack)
for imgnum in xrange(nimg1):
#print imgnum
classimg = get_im(classavgstack, imgnum)
ccc1 = classimg.get_attr_default('cross-corr', -1.0)
prjimg = get_im(projstack,imgnum)
ccc1 = prjimg.get_attr_default('cross-corr', -1.0)
cccoeff = ccc(prjimg,classimg)
#print imgnum, cccoeff
classimg.set_attr_dict({'cross-corr':cccoeff})
prjimg.set_attr_dict({'cross-corr':cccoeff})
prjimg.write_image(outstack,imgcounter)
imgcounter += 1
classimg.write_image(outstack, imgcounter)
imgcounter += 1
result.append(cccoeff)
result1 = sum(result)
#print result1
nimg2 = EMUtil.get_image_count(outstack)
meanccc = result1/nimg1
print("Mean CCC is %s" % meanccc)
for imgnum in xrange(nimg2):
if (imgnum % 2 ==0):
prjimg = get_im(outstack,imgnum)
meanccc1 = prjimg.get_attr_default('mean-cross-corr', -1.0)
prjimg.set_attr_dict({'mean-cross-corr':meanccc})
write_header(outstack,prjimg,imgnum)
if (imgnum % 100) == 0:
print imgnum
# e2proc2d appends to existing files, so delete existing output
if os.path.exists(normstack):
os.remove(normstack)
print("rm %s" % normstack)
cmd5="e2proc2d.py %s %s --process=normalize" % (outstack, normstack)
print cmd5
os.system(cmd5)
# Optionally pop up e2display
if displayYN:
cmd8 = "e2display.py %s" % normstack
print cmd8
os.system(cmd8)
print("Done!")
def main():
import os
import sys
from optparse import OptionParser
from global_def import SPARXVERSION
import global_def
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage2 = progname + """ inputfile outputfile [options]
Functionalities:
1. Helicise input volume and save the result to output volume:
sxhelicon_utils.py input_vol.hdf output_vol.hdf --helicise --dp=27.6 --dphi=166.5 --fract=0.65 --rmax=70 --rmin=1 --apix=1.84 --sym=D1
2. Helicise pdb file and save the result to a new pdb file:
sxhelicon_utils.py input.pdb output.pdb --helicisepdb --dp=27.6 --dphi=166.5 --nrepeats --apix=1.84
3. Generate two lists of image indices used to split segment stack into halves for helical fsc calculation.
sxhelicon_utils.py bdb:big_stack --hfsc='flst' --filament_attr=filament
4. Map of filament distribution in the stack
sxhelicon_utils.py bdb:big_stack --filinfo=info.txt
The output file will contain four columns:
1 2 3 4
first image number last image number number of images in the filament name
5. Predict segments' orientation parameters based on distances between segments and known helical symmetry
sxhelicon_utils.py bdb:big_stack --predict_helical=helical_params.txt --dp=27.6 --dphi=166.5 --apix=1.84
6. Generate disks from filament based reconstructions:
sxheader.py stk.hdf --params=xform.projection --import=params.txt
mpirun -np 2 sxhelicon_utils.py stk.hdf --gendisk='bdb:disk' --ref_nx=100 --ref_ny=100 --ref_nz=200 --apix=1.84 --dp=27.6 --dphi=166.715 --fract=0.67 --rmin=0 --rmax=64 --function="[.,nofunc,helical3c]" --sym="c1" --MPI
7. Stack disks based on helical symmetry parameters
sxhelicon_utils.py disk_to_stack.hdf --stackdisk=stacked_disks.hdf --dphi=166.5 --dp=27.6 --ref_nx=160 --ref_ny=160 --ref_nz=225 --apix=1.84
8. Helical symmetry search:
mpirun -np 3 sxhelicon_utils.py volf0010.hdf outsymsearch --symsearch --dp=27.6 --dphi=166.715 --apix=1.84 --fract=0.65 --rmin=0 --rmax=92.0 --datasym=datasym.txt --dp_step=0.92 --ndp=3 --dphi_step=1.0 --ndphi=10 --MPI
"""
parser = OptionParser(usage2, version=SPARXVERSION)
#parser.add_option("--ir", type="float", default= -1, help="inner radius for rotational correlation > 0 (set to 1) (Angstroms)")
parser.add_option(
"--ou",
type="float",
default=-1,
help=
"outer radius for rotational 2D correlation < int(nx/2)-1 (set to the radius of the particle) (Angstroms)"
)
parser.add_option(
"--rs",
type="int",
default=1,
help="step between rings in rotational correlation >0 (set to 1)")
parser.add_option(
"--xr",
type="string",
default="4 2 1 1 1",
help=
"range for translation search in x direction, search is +/-xr (Angstroms) "
)
parser.add_option(
"--txs",
type="string",
default="1 1 1 0.5 0.25",
help=
"step size of the translation search in x directions, search is -xr, -xr+ts, 0, xr-ts, xr (Angstroms)"
)
parser.add_option("--delta",
type="string",
default="10 6 4 3 2",
help="angular step of reference projections")
parser.add_option("--an",
type="string",
default="-1",
help="angular neighborhood for local searches")
parser.add_option(
"--maxit",
type="int",
default=30,
help=
"maximum number of iterations performed for each angular step (set to 30) "
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="CTF correction")
parser.add_option("--snr",
type="float",
default=1.0,
help="Signal-to-Noise Ratio of the data")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version")
#parser.add_option("--fourvar", action="store_true", default=False, help="compute Fourier variance")
parser.add_option("--apix",
type="float",
default=-1.0,
help="pixel size in Angstroms")
parser.add_option("--dp",
type="float",
default=-1.0,
help="delta z - translation in Angstroms")
parser.add_option("--dphi",
type="float",
default=-1.0,
help="delta phi - rotation in degrees")
parser.add_option("--rmin",
type="float",
default=0.0,
help="minimal radius for hsearch (Angstroms)")
parser.add_option("--rmax",
type="float",
default=80.0,
help="maximal radius for hsearch (Angstroms)")
parser.add_option("--fract",
type="float",
default=0.7,
help="fraction of the volume used for helical search")
parser.add_option("--sym",
type="string",
default="c1",
help="symmetry of the structure")
parser.add_option("--function",
type="string",
default="helical",
help="name of the reference preparation function")
parser.add_option("--npad",
type="int",
default=2,
help="padding size for 3D reconstruction")
parser.add_option("--debug",
action="store_true",
default=False,
help="debug")
parser.add_option("--volalixshift",
action="store_true",
default=False,
help="Use volalixshift refinement")
parser.add_option(
"--searchxshift",
type="float",
default=0.0,
help=
"search range for x-shift determination: +/- searchxshift (Angstroms)")
parser.add_option(
"--nearby",
type="float",
default=6.0,
help=
"neighborhood within which to search for peaks in 1D ccf for x-shift search (Angstroms)"
)
# filinfo
parser.add_option(
"--filinfo",
type="string",
default="",
help=
"Store in an output text file infomration about distribution of filaments in the stack."
)
# diskali
parser.add_option("--diskali",
action="store_true",
default=False,
help="volume alignment")
parser.add_option(
"--zstep",
type="float",
default=1,
help="Step size for translational search along z (Angstroms)")
# helicise
parser.add_option(
"--helicise",
action="store_true",
default=False,
help="helicise input volume and save results to output volume")
parser.add_option(
"--hfsc",
type="string",
default="",
help=
"Generate two lists of image indices used to split segment stack into halves for helical fsc calculation. The lists will be stored in two text files named using file_prefix with '_even' and '_odd' suffixes, respectively."
)
parser.add_option(
"--filament_attr",
type="string",
default="filament",
help="attribute under which filament identification is stored")
parser.add_option(
"--predict_helical",
type="string",
default="",
help="Generate projection parameters consistent with helical symmetry")
# helicise pdb
parser.add_option(
"--helicisepdb",
action="store_true",
default=False,
help="Helicise pdb file and save the result to a new pdb file")
parser.add_option(
"--nrepeats",
type="int",
default=50,
help=
"Number of time the helical symmetry will be applied to the input file"
)
# input options for generating disks
parser.add_option(
"--gendisk",
type="string",
default="",
help="Name of file under which generated disks will be saved to")
parser.add_option("--ref_nx",
type="int",
default=-1,
help="nx=ny volume size")
parser.add_option(
"--ref_nz",
type="int",
default=-1,
help="nz volume size - computed disks will be nx x ny x rise/apix")
parser.add_option(
"--new_pixel_size",
type="float",
default=-1,
help=
"desired pixel size of the output disks. The default is -1, in which case there is no resampling (unless --match_pixel_rise flag is True)."
)
parser.add_option(
"--maxerror",
type="float",
default=0.1,
help=
"proportional to the maximum amount of error to tolerate between (dp/new_pixel_size) and int(dp/new_pixel_size ), where new_pixel_size is the pixel size calculated when the option --match_pixel_rise flag is True."
)
parser.add_option(
"--match_pixel_rise",
action="store_true",
default=False,
help=
"calculate new pixel size such that the rise is approximately integer number of pixels given the new pixel size. This will be the pixel size of the output disks."
)
# get consistency
parser.add_option(
"--consistency",
type="string",
default="",
help="Name of parameters to get consistency statistics for")
parser.add_option("--phithr",
type="float",
default=2.0,
help="phi threshold for consistency check")
parser.add_option("--ythr",
type="float",
default=2.0,
help="y threshold (in Angstroms) for consistency check")
parser.add_option(
"--segthr",
type="int",
default=3,
help="minimum number of segments/filament for consistency check")
# stack disks
parser.add_option(
"--stackdisk",
type="string",
default="",
help="Name of file under which output volume will be saved to.")
parser.add_option("--ref_ny",
type="int",
default=-1,
help="ny of output volume size. Default is ref_nx")
# symmetry search
parser.add_option("--symsearch",
action="store_true",
default=False,
help="Do helical symmetry search.")
parser.add_option(
"--ndp",
type="int",
default=12,
help=
"In symmetrization search, number of delta z steps equals to 2*ndp+1")
parser.add_option(
"--ndphi",
type="int",
default=12,
help=
"In symmetrization search, number of dphi steps equals to 2*ndphi+1")
parser.add_option(
"--dp_step",
type="float",
default=0.1,
help="delta z step for symmetrization [Angstroms] (default 0.1)")
parser.add_option(
"--dphi_step",
type="float",
default=0.1,
help="dphi step for symmetrization [degrees] (default 0.1)")
parser.add_option("--datasym",
type="string",
default="datasym.txt",
help="symdoc")
parser.add_option(
"--symdoc",
type="string",
default="",
help="text file containing helical symmetry parameters dp and dphi")
# filament statistics in the stack
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 1 or len(args) > 5:
print("Various helical reconstruction related functionalities: " +
usage2)
print("Please run '" + progname + " -h' for detailed options")
else:
if len(options.hfsc) > 0:
if len(args) != 1:
print("Incorrect number of parameters")
sys.exit()
from applications import imgstat_hfsc
imgstat_hfsc(args[0], options.hfsc, options.filament_attr)
sys.exit()
elif len(options.filinfo) > 0:
if len(args) != 1:
print("Incorrect number of parameters")
sys.exit()
from EMAN2 import EMUtil
filams = EMUtil.get_all_attributes(args[0], "filament")
ibeg = 0
filcur = filams[0]
n = len(filams)
inf = []
i = 1
while (i <= n):
if (i < n): fis = filams[i]
else: fis = ""
if (fis != filcur):
iend = i - 1
inf.append([ibeg, iend, iend - ibeg + 1, filcur])
ibeg = i
filcur = fis
i += 1
from utilities import write_text_row
write_text_row(inf, options.filinfo)
sys.exit()
if len(options.stackdisk) > 0:
if len(args) != 1:
print("Incorrect number of parameters")
sys.exit()
dpp = (float(options.dp) / options.apix)
rise = int(dpp)
if (abs(float(rise) - dpp) > 1.0e-3):
print(" dpp has to be integer multiplicity of the pixel size")
sys.exit()
from utilities import get_im
v = get_im(args[0])
from applications import stack_disks
ref_ny = options.ref_ny
if ref_ny < 0:
ref_ny = options.ref_nx
sv = stack_disks(v, options.ref_nx, ref_ny, options.ref_nz,
options.dphi, rise)
sv.write_image(options.stackdisk)
sys.exit()
if len(options.consistency) > 0:
if len(args) != 1:
print("Incorrect number of parameters")
sys.exit()
from development import consistency_params
consistency_params(args[0],
options.consistency,
options.dphi,
options.dp,
options.apix,
phithr=options.phithr,
ythr=options.ythr,
THR=options.segthr)
sys.exit()
rminp = int((float(options.rmin) / options.apix) + 0.5)
rmaxp = int((float(options.rmax) / options.apix) + 0.5)
from utilities import get_input_from_string, get_im
xr = get_input_from_string(options.xr)
txs = get_input_from_string(options.txs)
irp = 1
if options.ou < 0: oup = -1
else: oup = int((options.ou / options.apix) + 0.5)
xrp = ''
txsp = ''
for i in xrange(len(xr)):
xrp += " " + str(float(xr[i]) / options.apix)
for i in xrange(len(txs)):
txsp += " " + str(float(txs[i]) / options.apix)
searchxshiftp = int((options.searchxshift / options.apix) + 0.5)
nearbyp = int((options.nearby / options.apix) + 0.5)
zstepp = int((options.zstep / options.apix) + 0.5)
if options.MPI:
from mpi import mpi_init, mpi_finalize
sys.argv = mpi_init(len(sys.argv), sys.argv)
if len(options.predict_helical) > 0:
if len(args) != 1:
print("Incorrect number of parameters")
sys.exit()
if options.dp < 0:
print(
"Helical symmetry paramter rise --dp should not be negative"
)
sys.exit()
from applications import predict_helical_params
predict_helical_params(args[0], options.dp, options.dphi,
options.apix, options.predict_helical)
sys.exit()
if options.helicise:
if len(args) != 2:
print("Incorrect number of parameters")
sys.exit()
if options.dp < 0:
print(
"Helical symmetry paramter rise --dp should not be negative"
)
sys.exit()
from utilities import get_im, sym_vol
vol = get_im(args[0])
vol = sym_vol(vol, options.sym)
hvol = vol.helicise(options.apix, options.dp, options.dphi,
options.fract, rmaxp, rminp)
hvol = sym_vol(hvol, options.sym)
hvol.write_image(args[1])
sys.exit()
if options.helicisepdb:
if len(args) != 2:
print("Incorrect number of parameters")
sys.exit()
if options.dp < 0:
print(
"Helical symmetry paramter rise --dp should not be negative"
)
sys.exit()
from math import cos, sin, radians
from copy import deepcopy
import numpy
from numpy import zeros, dot, float32
dp = options.dp
dphi = options.dphi
nperiod = options.nrepeats
infile = open(args[0], "r")
pall = infile.readlines()
infile.close()
p = []
pos = []
lkl = -1
for i in xrange(len(pall)):
if ((pall[i])[:4] == 'ATOM'):
if (lkl == -1): lkl = i
p.append(pall[i])
pos.append(i)
n = len(p)
X = zeros((3, len(p)), dtype=float32)
X_new = zeros((3, len(p)), dtype=float32)
for i in xrange(len(p)):
element = deepcopy(p[i])
X[0, i] = float(element[30:38])
X[1, i] = float(element[38:46])
X[2, i] = float(element[46:54])
pnew = []
for j in xrange(-nperiod, nperiod + 1):
for i in xrange(n):
pnew.append(deepcopy(p[i]))
dphi = radians(dphi)
m = zeros((3, 3), dtype=float32)
t = zeros((3, 1), dtype=float32)
m[2][2] = 1.0
t[0, 0] = 0.0
t[1, 0] = 0.0
for j in xrange(-nperiod, nperiod + 1):
if j != 0:
rd = j * dphi
m[0][0] = cos(rd)
m[0][1] = sin(rd)
m[1][0] = -m[0][1]
m[1][1] = m[0][0]
t[2, 0] = j * dp
X_new = dot(m, X) + t
for i in xrange(n):
pnew[j * n +
i] = pnew[j * n + i][:30] + "%8.3f" % (float(
X_new[0, i])) + "%8.3f" % (float(
X_new[1, i])) + "%8.3f" % (float(
X_new[2, i])) + pnew[j * n + i][54:]
outfile = open(args[1], "w")
outfile.writelines(pall[0:lkl])
outfile.writelines(pnew)
outfile.writelines("END\n")
outfile.close()
sys.exit()
if options.volalixshift:
if options.maxit > 1:
print(
"Inner iteration for x-shift determinatin is restricted to 1"
)
sys.exit()
if len(args) < 4: mask = None
else: mask = args[3]
from applications import volalixshift_MPI
global_def.BATCH = True
volalixshift_MPI(args[0], args[1], args[2], searchxshiftp,
options.apix, options.dp, options.dphi,
options.fract, rmaxp, rminp, mask, options.maxit,
options.CTF, options.snr, options.sym,
options.function, options.npad, options.debug,
nearbyp)
global_def.BATCH = False
if options.diskali:
#if options.maxit > 1:
# print "Inner iteration for disk alignment is restricted to 1"
# sys.exit()
if len(args) < 4: mask = None
else: mask = args[3]
global_def.BATCH = True
if (options.sym[:1] == "d" or options.sym[:1] == "D"):
from development import diskaliD_MPI
diskaliD_MPI(args[0], args[1], args[2], mask, options.dp,
options.dphi, options.apix, options.function,
zstepp, options.fract, rmaxp, rminp, options.CTF,
options.maxit, options.sym)
else:
from applications import diskali_MPI
diskali_MPI(args[0], args[1], args[2], mask, options.dp,
options.dphi, options.apix, options.function,
zstepp, options.fract, rmaxp, rminp, options.CTF,
options.maxit, options.sym)
global_def.BATCH = False
if options.symsearch:
if len(options.symdoc) < 1:
if options.dp < 0 or options.dphi < 0:
print(
"Enter helical symmetry parameters either using --symdoc or --dp and --dphi"
)
sys.exit()
if options.dp < 0 or options.dphi < 0:
# read helical symmetry parameters from symdoc
from utilities import read_text_row
hparams = read_text_row(options.symdoc)
dp = hparams[0][0]
dphi = hparams[0][1]
else:
dp = options.dp
dphi = options.dphi
from applications import symsearch_MPI
if len(args) < 3:
mask = None
else:
mask = args[2]
global_def.BATCH = True
symsearch_MPI(args[0], args[1], mask, dp, options.ndp,
options.dp_step, dphi, options.ndphi,
options.dphi_step, rminp, rmaxp, options.fract,
options.sym, options.function, options.datasym,
options.apix, options.debug)
global_def.BATCH = False
elif len(options.gendisk) > 0:
from applications import gendisks_MPI
global_def.BATCH = True
if len(args) == 1: mask3d = None
else: mask3d = args[1]
if options.dp < 0:
print(
"Helical symmetry paramter rise --dp must be explictly set!"
)
sys.exit()
gendisks_MPI(args[0], mask3d, options.ref_nx, options.apix,
options.dp, options.dphi, options.fract, rmaxp, rminp,
options.CTF, options.function, options.sym,
options.gendisk, options.maxerror,
options.new_pixel_size, options.match_pixel_rise)
global_def.BATCH = False
if options.MPI:
from mpi import mpi_finalize
mpi_finalize()
def main():
from utilities import write_text_row, drop_image, model_gauss_noise, get_im, set_params_proj, wrap_mpi_bcast, model_circle
import user_functions
from applications import MPI_start_end
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from multi_shc import multi_shc, do_volume
from logger import Logger, BaseLogger_Files
import sys
import os
import time
import socket
progname = os.path.basename(sys.argv[0])
usage = progname + " stack [output_directory] initial_volume --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --CTF --fl --aa --ref_a=S --sym=c1"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--ir", type= "int", default= 1, help="inner radius for rotational correlation > 0 (set to 1)")
parser.add_option("--ou", type= "int", default= -1, help="outer radius for rotational correlation < int(nx/2)-1 (set to the radius of the particle)")
parser.add_option("--rs", type= "int", default= 1, help="step between rings in rotational correlation >0 (set to 1)" )
parser.add_option("--xr", type="string", default= "-1", help="range for translation search in x direction, search is +/xr (default 0)")
parser.add_option("--yr", type="string", default= "-1", help="range for translation search in y direction, search is +/yr (default = same as xr)")
parser.add_option("--ts", type="string", default= "1", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional")
parser.add_option("--delta", type="string", default= "-1", help="angular step of reference projections during initialization step (default automatically selected based on radius of the structure.)")
parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches (phi and theta) (Default exhaustive searches)")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF (Default no CTF correction)")
parser.add_option("--shrink", type="float", default= 1.0, help="Reduce data size by shrink factor (default 1.0)")
parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data (default 1.0)")
parser.add_option("--ref_a", type="string", default= "S", help="method for generating the quasi-uniformly distributed projection directions (default S)")
parser.add_option("--sym", type="string", default= "c1", help="symmetry of the refined structure")
parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default=2)")
#options introduced for the do_volume function
parser.add_option("--fl", type="float", default=0.12, help="cut-off frequency of hyperbolic tangent low-pass Fourier filte (default 0.12)")
parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass Fourier filter (default 0.1)")
parser.add_option("--pwreference", type="string", default="", help="text file with a reference power spectrum (default no power spectrum adjustment)")
parser.add_option("--mask3D", type="string", default=None, help="3D mask file (default a sphere WHAT RADIUS??)")
(options, args) = parser.parse_args(sys.argv[1:])
#print( " args ",args)
if( len(args) == 3):
volinit = args[2]
masterdir = args[1]
elif(len(args) == 2):
volinit = args[1]
masterdir = ""
else:
print( "usage: " + usage)
print( "Please run '" + progname + " -h' for detailed options")
return 1
stack = args[0]
# INPUT PARAMETERS
radi = options.ou
global_def.BATCH = True
ali3d_options.ir = options.ir
ali3d_options.rs = options.rs
ali3d_options.ou = options.ou
ali3d_options.xr = options.xr
ali3d_options.yr = options.yr
ali3d_options.ts = options.ts
ali3d_options.an = "-1"
ali3d_options.sym = options.sym
ali3d_options.delta = options.delta
ali3d_options.npad = options.npad
ali3d_options.CTF = options.CTF
ali3d_options.ref_a = options.ref_a
ali3d_options.snr = options.snr
ali3d_options.mask3D = options.mask3D
ali3d_options.pwreference = "" # It will have to be turned on after exhaustive done by setting to options.pwreference
ali3d_options.fl = 0.4
ali3d_options.initfl = 0.4
ali3d_options.aa = 0.1
mpi_init(0, [])
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
# Get the pixel size, if none set to 1.0, and the original image size
if(myid == main_node):
total_stack = EMUtil.get_image_count(stack)
a = get_im(stack)
nxinit = a.get_xsize()
if ali3d_options.CTF:
i = a.get_attr('ctf')
pixel_size = i.apix
fq = pixel_size/fq
else:
pixel_size = 1.0
# No pixel size, fusing computed as 5 Fourier pixels
fq = 5.0/nxinit
del a
else:
total_stack = 0
nxinit = 0
pixel_size = 1.0
total_stack = bcast_number_to_all(total_stack, source_node = main_node)
pixel_size = bcast_number_to_all(pixel_size, source_node = main_node)
nxinit = bcast_number_to_all(nxinit, source_node = main_node)
if(radi < 1): radi = nxinit//2-2
elif((2*radi+2)>nxinit): ERROR("Particle radius set too large!","sxcenter_projections",1,myid)
ali3d_options.ou = radi
shrink = options.shrink
nxshrink = int(nxinit*shrink+0.5)
angular_neighborhood = "-1"
# MASTER DIRECTORY
if(myid == main_node):
print( " masterdir ",masterdir)
if( masterdir == ""):
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "master"+timestring
li = len(masterdir)
cmd = "{} {}".format("mkdir", masterdir)
junk = cmdexecute(cmd)
else:
li = 0
li = mpi_bcast(li,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
if( li > 0 ):
masterdir = mpi_bcast(masterdir,li,MPI_CHAR,main_node,MPI_COMM_WORLD)
masterdir = string.join(masterdir,"")
nnxo = nxinit
# INITIALIZATION
initdir = masterdir
# This is initial setting, has to be initialized here, we do not want it to run too long.
# INITIALIZATION THAT FOLLOWS WILL HAVE TO BE CHANGED SO THE USER CAN PROVIDE INITIAL GUESS OF RESOLUTION
# If we new the initial resolution, it could be done more densely
if(options.xr == "-1"): xr = "%d"%((nnxo - (2*radi-1))//2)
else: xr = options.xr
if(options.yr == "-1"): yr = xr
else: yr = options.yr
delta = float(options.delta)
if(delta <= 0.0): delta = "%f"%round(degrees(atan(1.0/float(radi))), 2)
else: delta = "%f"%delta
paramsdict = { "stack":stack,"delta":delta, "ts":"1.0", "xr":xr, "an":angular_neighborhood, \
"center":"0", "maxit":1, "local":False,\
"lowpass":options.fl, "initialfl":0.4, "falloff":options.aa, "radius":radi, \
"nsoft":0, "delpreviousmax":True, "shrink":options.shrink, "saturatecrit":1.0, "pixercutoff":2.0,\
"refvol":volinit, "mask3D":options.mask3D}
partids = os.path.join(masterdir, "ids.txt")
partstack = os.path.join(masterdir, "paramszero.txt")
if( myid == main_node ):
write_text_file(range(total_stack), partids)
write_text_row([[0.0,0.0,0.0,0.0,0.0] for i in xrange(total_stack) ], partstack)
run3Dalignment(paramsdict, partids, partstack, initdir, 0, myid, main_node, nproc)
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def main():
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from logger import Logger, BaseLogger_Files
import sys, os, time
global Tracker, Blockdata
from global_def import ERROR
progname = os.path.basename(sys.argv[0])
usage = progname + " --output_dir=output_dir --isac_dir=output_dir_of_isac "
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--pw_adjustment", type ="string", default ='analytical_model', \
help="adjust power spectrum of 2-D averages to an analytic model. Other opions: no_adjustment; bfactor; a text file of 1D rotationally averaged PW")
#### Four options for --pw_adjustment:
# 1> analytical_model(default);
# 2> no_adjustment;
# 3> bfactor;
# 4> adjust_to_given_pw2(user has to provide a text file that contains 1D rotationally averaged PW)
# options in common
parser.add_option(
"--isac_dir",
type="string",
default='',
help="ISAC run output directory, input directory for this command")
parser.add_option(
"--output_dir",
type="string",
default='',
help="output directory where computed averages are saved")
parser.add_option(
"--pixel_size",
type="float",
default=-1.0,
help=
"pixel_size of raw images. one can put 1.0 in case of negative stain data"
)
parser.add_option(
"--fl",
type="float",
default=-1.0,
help=
"low pass filter, = -1.0, not applied; =0.0, using FH1 (initial resolution), = 1.0 using FH2 (resolution after local alignment), or user provided value in absolute freqency [0.0:0.5]"
)
parser.add_option("--stack",
type="string",
default="",
help="data stack used in ISAC")
parser.add_option("--radius", type="int", default=-1, help="radius")
parser.add_option("--xr",
type="float",
default=-1.0,
help="local alignment search range")
#parser.add_option("--ts", type ="float", default =1.0, help= "local alignment search step")
parser.add_option("--fh",
type="float",
default=-1.0,
help="local alignment high frequencies limit")
#parser.add_option("--maxit", type ="int", default =5, help= "local alignment iterations")
parser.add_option("--navg",
type="int",
default=1000000,
help="number of aveages")
parser.add_option("--local_alignment",
action="store_true",
default=False,
help="do local alignment")
parser.add_option(
"--noctf",
action="store_true",
default=False,
help=
"no ctf correction, useful for negative stained data. always ctf for cryo data"
)
parser.add_option(
"--B_start",
type="float",
default=45.0,
help=
"start frequency (Angstrom) of power spectrum for B_factor estimation")
parser.add_option(
"--Bfactor",
type="float",
default=-1.0,
help=
"User defined bactors (e.g. 25.0[A^2]). By default, the program automatically estimates B-factor. "
)
(options, args) = parser.parse_args(sys.argv[1:])
adjust_to_analytic_model = False
adjust_to_given_pw2 = False
B_enhance = False
no_adjustment = False
if options.pw_adjustment == 'analytical_model':
adjust_to_analytic_model = True
elif options.pw_adjustment == 'no_adjustment':
no_adjustment = True
elif options.pw_adjustment == 'bfactor':
B_enhance = True
else:
adjust_to_given_pw2 = True
from utilities import get_im, bcast_number_to_all, write_text_file, read_text_file, wrap_mpi_bcast, write_text_row
from utilities import cmdexecute
from filter import filt_tanl
from logger import Logger, BaseLogger_Files
import user_functions
import string
from string import split, atoi, atof
import json
mpi_init(0, [])
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
Blockdata = {}
# MPI stuff
Blockdata["nproc"] = nproc
Blockdata["myid"] = myid
Blockdata["main_node"] = 0
Blockdata["shared_comm"] = mpi_comm_split_type(MPI_COMM_WORLD,
MPI_COMM_TYPE_SHARED, 0,
MPI_INFO_NULL)
Blockdata["myid_on_node"] = mpi_comm_rank(Blockdata["shared_comm"])
Blockdata["no_of_processes_per_group"] = mpi_comm_size(
Blockdata["shared_comm"])
masters_from_groups_vs_everything_else_comm = mpi_comm_split(
MPI_COMM_WORLD, Blockdata["main_node"] == Blockdata["myid_on_node"],
Blockdata["myid_on_node"])
Blockdata["color"], Blockdata["no_of_groups"], balanced_processor_load_on_nodes = get_colors_and_subsets(Blockdata["main_node"], MPI_COMM_WORLD, Blockdata["myid"], \
Blockdata["shared_comm"], Blockdata["myid_on_node"], masters_from_groups_vs_everything_else_comm)
# We need two nodes for processing of volumes
Blockdata["node_volume"] = [
Blockdata["no_of_groups"] - 3, Blockdata["no_of_groups"] - 2,
Blockdata["no_of_groups"] - 1
] # For 3D stuff take three last nodes
# We need two CPUs for processing of volumes, they are taken to be main CPUs on each volume
# We have to send the two myids to all nodes so we can identify main nodes on two selected groups.
Blockdata["nodes"] = [Blockdata["node_volume"][0]*Blockdata["no_of_processes_per_group"],Blockdata["node_volume"][1]*Blockdata["no_of_processes_per_group"], \
Blockdata["node_volume"][2]*Blockdata["no_of_processes_per_group"]]
# End of Blockdata: sorting requires at least three nodes, and the used number of nodes be integer times of three
global_def.BATCH = True
global_def.MPI = True
if adjust_to_given_pw2:
checking_flag = 0
if (Blockdata["myid"] == Blockdata["main_node"]):
if not os.path.exists(options.pw_adjustment): checking_flag = 1
checking_flag = bcast_number_to_all(checking_flag,
Blockdata["main_node"],
MPI_COMM_WORLD)
if checking_flag == 1:
ERROR("User provided power spectrum does not exist",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
Tracker = {}
Constants = {}
Constants["isac_dir"] = options.isac_dir
Constants["masterdir"] = options.output_dir
Constants["pixel_size"] = options.pixel_size
Constants["orgstack"] = options.stack
Constants["radius"] = options.radius
Constants["xrange"] = options.xr
Constants["FH"] = options.fh
Constants["low_pass_filter"] = options.fl
#Constants["maxit"] = options.maxit
Constants["navg"] = options.navg
Constants["B_start"] = options.B_start
Constants["Bfactor"] = options.Bfactor
if adjust_to_given_pw2: Constants["modelpw"] = options.pw_adjustment
Tracker["constants"] = Constants
# -------------------------------------------------------------
#
# Create and initialize Tracker dictionary with input options # State Variables
#<<<---------------------->>>imported functions<<<---------------------------------------------
#x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1)
#y_range = x_range
####-----------------------------------------------------------
# Create Master directory and associated subdirectories
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]:
masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen")
else:
masterdir = Tracker["constants"]["masterdir"]
if (Blockdata["myid"] == Blockdata["main_node"]):
msg = "Postprocessing ISAC 2D averages starts"
print(line, "Postprocessing ISAC 2D averages starts")
if not masterdir:
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "sharpen_" + Tracker["constants"]["isac_dir"]
os.mkdir(masterdir)
else:
if os.path.exists(masterdir):
print("%s already exists" % masterdir)
else:
os.mkdir(masterdir)
subdir_path = os.path.join(masterdir, "ali2d_local_params_avg")
if not os.path.exists(subdir_path): os.mkdir(subdir_path)
subdir_path = os.path.join(masterdir, "params_avg")
if not os.path.exists(subdir_path): os.mkdir(subdir_path)
li = len(masterdir)
else:
li = 0
li = mpi_bcast(li, 1, MPI_INT, Blockdata["main_node"], MPI_COMM_WORLD)[0]
masterdir = mpi_bcast(masterdir, li, MPI_CHAR, Blockdata["main_node"],
MPI_COMM_WORLD)
masterdir = string.join(masterdir, "")
Tracker["constants"]["masterdir"] = masterdir
log_main = Logger(BaseLogger_Files())
log_main.prefix = Tracker["constants"]["masterdir"] + "/"
while not os.path.exists(Tracker["constants"]["masterdir"]):
print("Node ", Blockdata["myid"], " waiting...",
Tracker["constants"]["masterdir"])
sleep(1)
mpi_barrier(MPI_COMM_WORLD)
if (Blockdata["myid"] == Blockdata["main_node"]):
init_dict = {}
print(Tracker["constants"]["isac_dir"])
Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"],
"2dalignment")
core = read_text_row(
os.path.join(Tracker["directory"], "initial2Dparams.txt"))
for im in range(len(core)):
init_dict[im] = core[im]
del core
else:
init_dict = 0
init_dict = wrap_mpi_bcast(init_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
###
do_ctf = True
if options.noctf: do_ctf = False
if (Blockdata["myid"] == Blockdata["main_node"]):
if do_ctf: print("CTF correction is on")
else: print("CTF correction is off")
if options.local_alignment: print("local refinement is on")
else: print("local refinement is off")
if B_enhance: print("Bfactor is to be applied on averages")
elif adjust_to_given_pw2:
print("PW of averages is adjusted to a given 1D PW curve")
elif adjust_to_analytic_model:
print("PW of averages is adjusted to analytical model")
else:
print("PW of averages is not adjusted")
#Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack")
image = get_im(Tracker["constants"]["orgstack"], 0)
Tracker["constants"]["nnxo"] = image.get_xsize()
if Tracker["constants"]["pixel_size"] == -1.0:
print(
"Pixel size value is not provided by user. extracting it from ctf header entry of the original stack."
)
try:
ctf_params = image.get_attr("ctf")
Tracker["constants"]["pixel_size"] = ctf_params.apix
except:
ERROR(
"Pixel size could not be extracted from the original stack.",
"sxcompute_isac_avg.py", 1,
Blockdata["myid"]) # action=1 - fatal error, exit
## Now fill in low-pass filter
isac_shrink_path = os.path.join(Tracker["constants"]["isac_dir"],
"README_shrink_ratio.txt")
if not os.path.exists(isac_shrink_path):
ERROR(
"%s does not exist in the specified ISAC run output directory"
% (isac_shrink_path), "sxcompute_isac_avg.py", 1,
Blockdata["myid"]) # action=1 - fatal error, exit
isac_shrink_file = open(isac_shrink_path, "r")
isac_shrink_lines = isac_shrink_file.readlines()
isac_shrink_ratio = float(
isac_shrink_lines[5]
) # 6th line: shrink ratio (= [target particle radius]/[particle radius]) used in the ISAC run
isac_radius = float(
isac_shrink_lines[6]
) # 7th line: particle radius at original pixel size used in the ISAC run
isac_shrink_file.close()
print("Extracted parameter values")
print("ISAC shrink ratio : {0}".format(isac_shrink_ratio))
print("ISAC particle radius : {0}".format(isac_radius))
Tracker["ini_shrink"] = isac_shrink_ratio
else:
Tracker["ini_shrink"] = 0.0
Tracker = wrap_mpi_bcast(Tracker,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
#print(Tracker["constants"]["pixel_size"], "pixel_size")
x_range = max(Tracker["constants"]["xrange"],
int(1. / Tracker["ini_shrink"] + 0.99999))
a_range = y_range = x_range
if (Blockdata["myid"] == Blockdata["main_node"]):
parameters = read_text_row(
os.path.join(Tracker["constants"]["isac_dir"],
"all_parameters.txt"))
else:
parameters = 0
parameters = wrap_mpi_bcast(parameters,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
params_dict = {}
list_dict = {}
#parepare params_dict
#navg = min(Tracker["constants"]["navg"]*Blockdata["nproc"], EMUtil.get_image_count(os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf")))
navg = min(
Tracker["constants"]["navg"],
EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf")))
global_dict = {}
ptl_list = []
memlist = []
if (Blockdata["myid"] == Blockdata["main_node"]):
print("Number of averages computed in this run is %d" % navg)
for iavg in range(navg):
params_of_this_average = []
image = get_im(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"), iavg)
members = sorted(image.get_attr("members"))
memlist.append(members)
for im in range(len(members)):
abs_id = members[im]
global_dict[abs_id] = [iavg, im]
P = combine_params2( init_dict[abs_id][0], init_dict[abs_id][1], init_dict[abs_id][2], init_dict[abs_id][3], \
parameters[abs_id][0], parameters[abs_id][1]/Tracker["ini_shrink"], parameters[abs_id][2]/Tracker["ini_shrink"], parameters[abs_id][3])
if parameters[abs_id][3] == -1:
print(
"WARNING: Image #{0} is an unaccounted particle with invalid 2D alignment parameters and should not be the member of any classes. Please check the consitency of input dataset."
.format(abs_id)
) # How to check what is wrong about mirror = -1 (Toshio 2018/01/11)
params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0])
ptl_list.append(abs_id)
params_dict[iavg] = params_of_this_average
list_dict[iavg] = members
write_text_row(
params_of_this_average,
os.path.join(Tracker["constants"]["masterdir"], "params_avg",
"params_avg_%03d.txt" % iavg))
ptl_list.sort()
init_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
init_params[im] = [ptl_list[im]] + params_dict[global_dict[
ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
init_params,
os.path.join(Tracker["constants"]["masterdir"],
"init_isac_params.txt"))
else:
params_dict = 0
list_dict = 0
memlist = 0
params_dict = wrap_mpi_bcast(params_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
list_dict = wrap_mpi_bcast(list_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
memlist = wrap_mpi_bcast(memlist,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
# Now computing!
del init_dict
tag_sharpen_avg = 1000
## always apply low pass filter to B_enhanced images to suppress noise in high frequencies
enforced_to_H1 = False
if B_enhance:
if Tracker["constants"]["low_pass_filter"] == -1.0:
enforced_to_H1 = True
if navg < Blockdata["nproc"]: # Each CPU do one average
ERROR("number of nproc is larger than number of averages",
"sxcompute_isac_avg.py", 1, Blockdata["myid"])
else:
FH_list = [[0, 0.0, 0.0] for im in range(navg)]
image_start, image_end = MPI_start_end(navg, Blockdata["nproc"],
Blockdata["myid"])
if Blockdata["myid"] == Blockdata["main_node"]:
cpu_dict = {}
for iproc in range(Blockdata["nproc"]):
local_image_start, local_image_end = MPI_start_end(
navg, Blockdata["nproc"], iproc)
for im in range(local_image_start, local_image_end):
cpu_dict[im] = iproc
else:
cpu_dict = 0
cpu_dict = wrap_mpi_bcast(cpu_dict,
Blockdata["main_node"],
communicator=MPI_COMM_WORLD)
slist = [None for im in range(navg)]
ini_list = [None for im in range(navg)]
avg1_list = [None for im in range(navg)]
avg2_list = [None for im in range(navg)]
plist_dict = {}
data_list = [None for im in range(navg)]
if Blockdata["myid"] == Blockdata["main_node"]:
if B_enhance:
print(
"Avg ID B-factor FH1(Res before ali) FH2(Res after ali)"
)
else:
print("Avg ID FH1(Res before ali) FH2(Res after ali)")
for iavg in range(image_start, image_end):
mlist = EMData.read_images(Tracker["constants"]["orgstack"],
list_dict[iavg])
for im in range(len(mlist)):
#mlist[im]= get_im(Tracker["constants"]["orgstack"], list_dict[iavg][im])
set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
if options.local_alignment:
"""
new_average1 = within_group_refinement([mlist[kik] for kik in range(0,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou=Tracker["constants"]["radius"], rs=1.0, xrng=[x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH=max(Tracker["constants"]["FH"], FH1), FF=0.02, method="")
new_average2 = within_group_refinement([mlist[kik] for kik in range(1,len(mlist),2)], maskfile= None, randomize= False, ir=1.0, \
ou= Tracker["constants"]["radius"], rs=1.0, xrng=[ x_range], yrng=[y_range], step=[Tracker["constants"]["xstep"]], \
dst=0.0, maxit=Tracker["constants"]["maxit"], FH = max(Tracker["constants"]["FH"], FH1), FF=0.02, method="")
new_avg, frc, plist = compute_average(mlist, Tracker["constants"]["radius"], do_ctf)
"""
new_avg, plist, FH2 = refinement_2d_local(
mlist,
Tracker["constants"]["radius"],
a_range,
x_range,
y_range,
CTF=do_ctf,
SNR=1.0e10)
plist_dict[iavg] = plist
FH1 = -1.0
else:
new_avg, frc, plist = compute_average(
mlist, Tracker["constants"]["radius"], do_ctf)
FH1 = get_optimistic_res(frc)
FH2 = -1.0
#write_text_file(frc, os.path.join(Tracker["constants"]["masterdir"], "fsc%03d.txt"%iavg))
FH_list[iavg] = [iavg, FH1, FH2]
if B_enhance:
new_avg, gb = apply_enhancement(
new_avg, Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"])
print(" %6d %6.3f %4.3f %4.3f" % (iavg, gb, FH1, FH2))
elif adjust_to_given_pw2:
roo = read_text_file(Tracker["constants"]["modelpw"], -1)
roo = roo[0] # always on the first column
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], roo)
print(" %6d %4.3f %4.3f " % (iavg, FH1, FH2))
elif adjust_to_analytic_model:
new_avg = adjust_pw_to_model(
new_avg, Tracker["constants"]["pixel_size"], None)
print(" %6d %4.3f %4.3f " % (iavg, FH1, FH2))
elif no_adjustment:
pass
if Tracker["constants"]["low_pass_filter"] != -1.0:
if Tracker["constants"]["low_pass_filter"] == 0.0:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 1.0:
low_pass_filter = FH2
if not options.local_alignment: low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"]["low_pass_filter"]
if low_pass_filter >= 0.45: low_pass_filter = 0.45
new_avg = filt_tanl(new_avg, low_pass_filter, 0.02)
else: # No low pass filter but if enforced
if enforced_to_H1: new_avg = filt_tanl(new_avg, FH1, 0.02)
if B_enhance: new_avg = fft(new_avg)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
slist[iavg] = new_avg
print(
strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>",
"Refined average %7d" % iavg)
## send to main node to write
mpi_barrier(MPI_COMM_WORLD)
for im in range(navg):
# avg
if cpu_dict[im] == Blockdata[
"myid"] and Blockdata["myid"] != Blockdata["main_node"]:
send_EMData(slist[im], Blockdata["main_node"], tag_sharpen_avg)
elif cpu_dict[im] == Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
slist[im].set_attr("members", memlist[im])
slist[im].set_attr("n_objects", len(memlist[im]))
slist[im].write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
elif cpu_dict[im] != Blockdata["myid"] and Blockdata[
"myid"] == Blockdata["main_node"]:
new_avg_other_cpu = recv_EMData(cpu_dict[im], tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.set_attr("n_objects", len(memlist[im]))
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"), im)
if options.local_alignment:
if cpu_dict[im] == Blockdata["myid"]:
write_text_row(
plist_dict[im],
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params_avg",
"ali2d_local_params_avg_%03d.txt" % im))
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(plist_dict[im], Blockdata["main_node"],
MPI_COMM_WORLD)
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
plist_dict[im] = dummy
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
else:
if cpu_dict[im] == Blockdata[
"myid"] and cpu_dict[im] != Blockdata["main_node"]:
wrap_mpi_send(FH_list, Blockdata["main_node"],
MPI_COMM_WORLD)
elif cpu_dict[im] != Blockdata["main_node"] and Blockdata[
"myid"] == Blockdata["main_node"]:
dummy = wrap_mpi_recv(cpu_dict[im], MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi_barrier(MPI_COMM_WORLD)
mpi_barrier(MPI_COMM_WORLD)
if options.local_alignment:
if Blockdata["myid"] == Blockdata["main_node"]:
ali3d_local_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
ali3d_local_params[im] = [ptl_list[im]] + plist_dict[
global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
write_text_row(
ali3d_local_params,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params.txt"))
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
else:
if Blockdata["myid"] == Blockdata["main_node"]:
write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
mpi_barrier(MPI_COMM_WORLD)
target_xr = 3
target_yr = 3
if (Blockdata["myid"] == 0):
cmd = "{} {} {} {} {} {} {} {} {} {}".format("sxchains.py", os.path.join(Tracker["constants"]["masterdir"],"class_averages.hdf"),\
os.path.join(Tracker["constants"]["masterdir"],"junk.hdf"),os.path.join(Tracker["constants"]["masterdir"],"ordered_class_averages.hdf"),\
"--circular","--radius=%d"%Tracker["constants"]["radius"] , "--xr=%d"%(target_xr+1),"--yr=%d"%(target_yr+1),"--align", ">/dev/null")
junk = cmdexecute(cmd)
cmd = "{} {}".format(
"rm -rf",
os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"))
junk = cmdexecute(cmd)
from mpi import mpi_finalize
mpi_finalize()
exit()
def do_volume_mrk02(ref_data):
"""
data - projections (scattered between cpus) or the volume. If volume, just do the volume processing
options - the same for all cpus
return - volume the same for all cpus
"""
from EMAN2 import Util
from mpi import mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD
from filter import filt_table
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import bcast_EMData_to_all, bcast_number_to_all, model_blank
from fundamentals import rops_table, fftip, fft
import types
# Retrieve the function specific input arguments from ref_data
data = ref_data[0]
Tracker = ref_data[1]
iter = ref_data[2]
mpi_comm = ref_data[3]
# # For DEBUG
# print "Type of data %s" % (type(data))
# print "Type of Tracker %s" % (type(Tracker))
# print "Type of iter %s" % (type(iter))
# print "Type of mpi_comm %s" % (type(mpi_comm))
if(mpi_comm == None): mpi_comm = MPI_COMM_WORLD
myid = mpi_comm_rank(mpi_comm)
nproc = mpi_comm_size(mpi_comm)
try: local_filter = Tracker["local_filter"]
except: local_filter = False
#=========================================================================
# volume reconstruction
if( type(data) == types.ListType ):
if Tracker["constants"]["CTF"]:
vol = recons3d_4nn_ctf_MPI(myid, data, Tracker["constants"]["snr"], \
symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm, smearstep = Tracker["smearstep"])
else:
vol = recons3d_4nn_MPI (myid, data,\
symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm)
else:
vol = data
if myid == 0:
from morphology import threshold
from filter import filt_tanl, filt_btwl
from utilities import model_circle, get_im
import types
nx = vol.get_xsize()
if(Tracker["constants"]["mask3D"] == None):
mask3D = model_circle(int(Tracker["constants"]["radius"]*float(nx)/float(Tracker["constants"]["nnxo"])+0.5), nx, nx, nx)
elif(Tracker["constants"]["mask3D"] == "auto"):
from utilities import adaptive_mask
mask3D = adaptive_mask(vol)
else:
if( type(Tracker["constants"]["mask3D"]) == types.StringType ): mask3D = get_im(Tracker["constants"]["mask3D"])
else: mask3D = (Tracker["constants"]["mask3D"]).copy()
nxm = mask3D.get_xsize()
if( nx != nxm):
from fundamentals import rot_shift3D
mask3D = Util.window(rot_shift3D(mask3D,scale=float(nx)/float(nxm)),nx,nx,nx)
nxm = mask3D.get_xsize()
assert(nx == nxm)
stat = Util.infomask(vol, mask3D, False)
vol -= stat[0]
Util.mul_scalar(vol, 1.0/stat[1])
vol = threshold(vol)
Util.mul_img(vol, mask3D)
if( Tracker["PWadjustment"] ):
from utilities import read_text_file, write_text_file
rt = read_text_file( Tracker["PWadjustment"] )
fftip(vol)
ro = rops_table(vol)
# Here unless I am mistaken it is enough to take the beginning of the reference pw.
for i in xrange(1,len(ro)): ro[i] = (rt[i]/ro[i])**Tracker["upscale"]
#write_text_file(rops_table(filt_table( vol, ro),1),"foo.txt")
if Tracker["constants"]["sausage"]:
ny = vol.get_ysize()
y = float(ny)
from math import exp
for i in xrange(len(ro)): ro[i] *= \
(1.0+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.10)/0.025)**2)+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.215)/0.025)**2))
if local_filter:
# skip low-pass filtration
vol = fft( filt_table( vol, ro) )
else:
if( type(Tracker["lowpass"]) == types.ListType ):
vol = fft( filt_table( filt_table(vol, Tracker["lowpass"]), ro) )
else:
vol = fft( filt_table( filt_tanl(vol, Tracker["lowpass"], Tracker["falloff"]), ro) )
del ro
else:
if Tracker["constants"]["sausage"]:
ny = vol.get_ysize()
y = float(ny)
ro = [0.0]*(ny//2+2)
from math import exp
for i in xrange(len(ro)): ro[i] = \
(1.0+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.10)/0.025)**2)+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.215)/0.025)**2))
fftip(vol)
filt_table(vol, ro)
del ro
if not local_filter:
if( type(Tracker["lowpass"]) == types.ListType ):
vol = filt_table(vol, Tracker["lowpass"])
else:
vol = filt_tanl(vol, Tracker["lowpass"], Tracker["falloff"])
if Tracker["constants"]["sausage"]: vol = fft(vol)
if local_filter:
from morphology import binarize
if(myid == 0): nx = mask3D.get_xsize()
else: nx = 0
nx = bcast_number_to_all(nx, source_node = 0)
# only main processor needs the two input volumes
if(myid == 0):
mask = binarize(mask3D, 0.5)
locres = get_im(Tracker["local_filter"])
lx = locres.get_xsize()
if(lx != nx):
if(lx < nx):
from fundamentals import fdecimate, rot_shift3D
mask = Util.window(rot_shift3D(mask,scale=float(lx)/float(nx)),lx,lx,lx)
vol = fdecimate(vol, lx,lx,lx)
else: ERROR("local filter cannot be larger than input volume","user function",1)
stat = Util.infomask(vol, mask, False)
vol -= stat[0]
Util.mul_scalar(vol, 1.0/stat[1])
else:
lx = 0
locres = model_blank(1,1,1)
vol = model_blank(1,1,1)
lx = bcast_number_to_all(lx, source_node = 0)
if( myid != 0 ): mask = model_blank(lx,lx,lx)
bcast_EMData_to_all(mask, myid, 0, comm=mpi_comm)
from filter import filterlocal
vol = filterlocal( locres, vol, mask, Tracker["falloff"], myid, 0, nproc)
if myid == 0:
if(lx < nx):
from fundamentals import fpol
vol = fpol(vol, nx,nx,nx)
vol = threshold(vol)
vol = filt_btwl(vol, 0.38, 0.5)# This will have to be corrected.
Util.mul_img(vol, mask3D)
del mask3D
# vol.write_image('toto%03d.hdf'%iter)
else:
vol = model_blank(nx,nx,nx)
else:
if myid == 0:
#from utilities import write_text_file
#write_text_file(rops_table(vol,1),"goo.txt")
stat = Util.infomask(vol, mask3D, False)
vol -= stat[0]
Util.mul_scalar(vol, 1.0/stat[1])
vol = threshold(vol)
vol = filt_btwl(vol, 0.38, 0.5)# This will have to be corrected.
Util.mul_img(vol, mask3D)
del mask3D
# vol.write_image('toto%03d.hdf'%iter)
# broadcast volume
bcast_EMData_to_all(vol, myid, 0, comm=mpi_comm)
#=========================================================================
return vol
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
540.0 - psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
360.0 - psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl=15. --aa=0.01 --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--output_dir",
type="string",
default="./",
help="output directory")
parser.add_option("--ave2D",
type="string",
default=False,
help="write to the disk a stack of 2D averages")
parser.add_option("--var2D",
type="string",
default=False,
help="write to the disk a stack of 2D variances")
parser.add_option("--ave3D",
type="string",
default=False,
help="write to the disk reconstructed 3D average")
parser.add_option("--var3D",
type="string",
default=False,
help="compute 3D variability (time consuming!)")
parser.add_option("--img_per_grp",
type="int",
default=10,
help="number of neighbouring projections")
parser.add_option("--no_norm",
action="store_true",
default=False,
help="do not use normalization")
#parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option("--npad",
type="int",
default=2,
help="number of time to pad the original images")
parser.add_option("--sym", type="string", default="c1", help="symmetry")
parser.add_option(
"--fl",
type="float",
default=0.0,
help=
"cutoff freqency in absolute frequency (0.0-0.5). (Default - no filtration)"
)
parser.add_option(
"--aa",
type="float",
default=0.0,
help=
"fall off of the filter. Put 0.01 if user has no clue about falloff (Default - no filtration)"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="use CFT correction")
parser.add_option("--VERBOSE",
action="store_true",
default=False,
help="Long output for debugging")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option(
"--VAR",
action="store_true",
default=False,
help="stack on input consists of 2D variances (Default False)")
parser.add_option(
"--decimate",
type="float",
default=1.0,
help=
"image decimate rate, a number larger (expand image) or less (shrink image) than 1. default is 1"
)
parser.add_option(
"--window",
type="int",
default=0,
help=
"reduce images to a small image size without changing pixel_size. Default value is zero."
)
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
parser.add_option(
"--nvec",
type="int",
default=0,
help="number of eigenvectors, default = 0 meaning no PCA calculated")
parser.add_option(
"--symmetrize",
action="store_true",
default=False,
help="Prepare input stack for handling symmetry (Default False)")
(options, args) = parser.parse_args()
#####
from mpi import mpi_init, mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
from applications import MPI_start_end
from reconstruction import recons3d_em, recons3d_em_MPI
from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from utilities import print_begin_msg, print_end_msg, print_msg
from utilities import read_text_row, get_image, get_im
from utilities import bcast_EMData_to_all, bcast_number_to_all
from utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
# Set up global variables related to bdb cache
if global_def.CACHE_DISABLE:
from utilities import disable_bdb_cache
disable_bdb_cache()
# Set up global variables related to ERROR function
global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
if RUNNING_UNDER_MPI:
global_def.MPI = True
if options.symmetrize:
if RUNNING_UNDER_MPI:
try:
sys.argv = mpi_init(len(sys.argv), sys.argv)
try:
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
if (number_of_proc > 1):
ERROR(
"Cannot use more than one CPU for symmetry prepration",
"sx3dvariability", 1)
except:
pass
except:
pass
if options.output_dir != "./" and not os.path.exists(
options.output_dir):
os.mkdir(options.output_dir)
# Input
#instack = "Clean_NORM_CTF_start_wparams.hdf"
#instack = "bdb:data"
from logger import Logger, BaseLogger_Files
if os.path.exists(os.path.join(options.output_dir, "log.txt")):
os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(options.output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if (sym == "c1"):
ERROR("There is no need to symmetrize stack for C1 symmetry",
"sx3dvariability", 1)
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
if (instack[:4] != "bdb:"):
if output_dir == "./": stack = "bdb:data"
else: stack = "bdb:" + options.output_dir + "/data"
delete_bdb(stack)
junk = cmdexecute("sxcpy.py " + instack + " " + stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack, 'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None] * na
for k in xrange(ks):
#Qfile = "Q%1d"%k
if options.output_dir != "./":
Qfile = os.path.join(options.output_dir, "Q%1d" % k)
else:
Qfile = os.path.join(options.output_dir, "Q%1d" % k)
#delete_bdb("bdb:Q%1d"%k)
delete_bdb("bdb:" + Qfile)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = cmdexecute("e2bdb.py " + stack + " --makevstack=bdb:" +
Qfile)
#DB = db_open_dict("bdb:Q%1d"%k)
DB = db_open_dict("bdb:" + Qfile)
for i in xrange(na):
ut = qt[i] * ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
if options.output_dir == "./": delete_bdb("bdb:sdata")
else: delete_bdb("bdb:" + options.output_dir + "/" + "sdata")
#junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:" + options.output_dir + "/" + "sdata"
print(sdata)
junk = cmdexecute("e2bdb.py " + options.output_dir +
" --makevstack=" + sdata + " --filt=Q")
#junk = cmdexecute("ls EMAN2DB/sdata*")
#a = get_im("bdb:sdata")
a = get_im(sdata)
a.set_attr("variabilitysymmetry", sym)
#a.write_image("bdb:sdata")
a.write_image(sdata)
else:
sys.argv = mpi_init(len(sys.argv), sys.argv)
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 1:
stack = args[0]
else:
print(("usage: " + usage))
print(("Please run '" + progname + " -h' for detailed options"))
return 1
t0 = time()
# obsolete flags
options.MPI = True
options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter",
"sx3dvariability", 1, myid)
if options.VAR and options.SND:
ERROR("Only one of var and SND can be set!", "sx3dvariability",
myid)
exit()
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR(
"When VAR is set, the program cannot output ave2D, ave3D or var2D",
"sx3dvariability", 1, myid)
exit()
#if options.SND and (options.ave2D or options.ave3D):
# ERROR("When SND is set, the program cannot output ave2D or ave3D", "sx3dvariability", 1, myid)
# exit()
if options.nvec > 0:
ERROR("PCA option not implemented", "sx3dvariability", 1, myid)
exit()
if options.nvec > 0 and options.ave3D == None:
ERROR("When doing PCA analysis, one must set ave3D",
"sx3dvariability",
myid=myid)
exit()
import string
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if options.output_dir != "./" and not os.path.exists(
options.output_dir):
os.mkdir(options.output_dir)
img_per_grp = options.img_per_grp
nvec = options.nvec
radiuspca = options.radiuspca
from logger import Logger, BaseLogger_Files
#if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(options.output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("instack :" + stack)
log_main.add("output_dir :" + options.output_dir)
log_main.add("var3d :" + options.var3D)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
#print_begin_msg("sx3dvariability")
msg = "sx3dvariability"
log_main.add(msg)
print(line, msg)
msg = ("%-70s: %s\n" % ("Input stack", stack))
log_main.add(msg)
print(line, msg)
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
if options.sym != "c1":
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if (i != options.sym):
ERROR(
"The symmetry provided does not agree with the symmetry of the input stack",
"sx3dvariability",
myid=myid)
except:
ERROR(
"Input stack is not prepared for symmetry, please follow instructions",
"sx3dvariability",
myid=myid)
from utilities import get_symt
i = len(get_symt(options.sym))
if ((nima / i) * i != nima):
ERROR(
"The length of the input stack is incorrect for symmetry processing",
"sx3dvariability",
myid=myid)
symbaselen = nima / i
else:
symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
Tracker = {}
Tracker["total_stack"] = nima
if options.decimate == 1.:
if options.window != 0:
nx = options.window
ny = options.window
else:
if options.window == 0:
nx = int(nx * options.decimate)
ny = int(ny * options.decimate)
else:
nx = int(options.window * options.decimate)
ny = nx
Tracker["nx"] = nx
Tracker["ny"] = ny
Tracker["nz"] = nx
symbaselen = bcast_number_to_all(symbaselen)
if radiuspca == -1: radiuspca = nx / 2 - 2
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = "%-70s: %d\n" % ("Number of projection", nima)
log_main.add(msg)
print(line, msg)
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from projection import prep_vol, prgs
from statistics import im_diff
from utilities import get_im, model_circle, get_params_proj, set_params_proj
from utilities import get_ctf, generate_ctf
from filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR:
#varList = EMData.read_images(stack, range(img_begin, img_end))
varList = []
this_image = EMData()
for index_of_particle in xrange(img_begin, img_end):
this_image.read_image(stack, index_of_particle)
varList.append(
image_decimate_window_xform_ctf(this_image,
options.decimate,
options.window,
options.CTF))
else:
from utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from utilities import model_blank, nearest_proj, model_circle
from applications import pca
from statistics import avgvar, avgvar_ctf, ccc
from filter import filt_tanl
from morphology import threshold, square_root
from projection import project, prep_vol, prgs
from sets import Set
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in xrange(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x * 10000 + psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = "%-70s: %d\n" % ("Number of neighboring projections",
img_per_grp)
log_main.add(msg)
print(line, msg)
msg = "...... Finding neighboring projections\n"
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
msg = "Number of images per group: %d" % img_per_grp
log_main.add(msg)
print(line, msg)
msg = "Now grouping projections"
log_main.add(msg)
print(line, msg)
proj_angles.sort()
proj_angles_list = [0.0] * (nima * 4)
if myid == main_node:
for i in xrange(nima):
proj_angles_list[i * 4] = proj_angles[i][1]
proj_angles_list[i * 4 + 1] = proj_angles[i][2]
proj_angles_list[i * 4 + 2] = proj_angles[i][3]
proj_angles_list[i * 4 + 3] = proj_angles[i][4]
proj_angles_list = bcast_list_to_all(proj_angles_list, myid,
main_node)
proj_angles = []
for i in xrange(nima):
proj_angles.append([
proj_angles_list[i * 4], proj_angles_list[i * 4 + 1],
proj_angles_list[i * 4 + 2],
int(proj_angles_list[i * 4 + 3])
])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp,
range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
if options.VERBOSE:
print("On node %2d, number of images needed to be read = %5d" %
(myid, len(all_proj)))
index = {}
for i in xrange(len(all_proj)):
index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Finding neighboring projections lasted [s]",
time() - t2))
log_main.add(msg)
print(msg)
msg = ("%-70s: %d\n" %
("Number of groups processed on the main node",
len(proj_list)))
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Grouping projections took: ", (time() - t2) / 60,
"[min]")
print("Number of groups on main node: ", len(proj_list))
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("...... calculating the stack of 2D variances \n")
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Now calculating the stack of 2D variances")
proj_params = [0.0] * (nima * 5)
aveList = []
varList = []
if nvec > 0:
eigList = [[] for i in xrange(nvec)]
if options.VERBOSE:
print("Begin to read images on processor %d" % (myid))
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
imgdata = []
for index_of_proj in xrange(len(all_proj)):
#img = EMData()
#img.read_image(stack, all_proj[index_of_proj])
dmg = image_decimate_window_xform_ctf(
get_im(stack, all_proj[index_of_proj]), options.decimate,
options.window, options.CTF)
#print dmg.get_xsize(), "init"
imgdata.append(dmg)
if options.VERBOSE:
print("Reading images on processor %d done, time = %.2f" %
(myid, time() - ttt))
print("On processor %d, we got %d images" %
(myid, len(imgdata)))
mpi_barrier(MPI_COMM_WORLD)
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from applications import prepare_2d_forPCA
from utilities import model_blank
for i in xrange(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
phiM, thetaM, psiM, s2xM, s2yM = get_params_proj(imgdata[mi])
grp_imgdata = []
for j in xrange(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[mj])
alpha, sx, sy, mirror = params_3D_2D_NEW(
phi, theta, psi, s2x, s2y, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, phiM - phi, 0.0, 0.0, 1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, 180 - (phiM - phi), 0.0,
0.0, 1.0)
else:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(phiM - phi), 0.0, 0.0,
1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(180 - (phiM - phi)), 0.0,
0.0, 1.0)
set_params2D(imgdata[mj], [alpha, sx, sy, mirror, 1.0])
grp_imgdata.append(imgdata[mj])
#print grp_imgdata[j].get_xsize(), imgdata[mj].get_xsize()
if not options.no_norm:
#print grp_imgdata[j].get_xsize()
mask = model_circle(nx / 2 - 2, nx, nx)
for k in xrange(img_per_grp):
ave, std, minn, maxx = Util.infomask(
grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
del mask
if options.fl > 0.0:
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_tanl(
filt_ctf(
fft(
pad(grp_imgdata[k], nx2, ny2,
1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1), options.fl,
options.aa)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
for k in xrange(img_per_grp):
grp_imgdata[k] = filt_tanl(grp_imgdata[k],
options.fl, options.aa)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
else:
from utilities import pad, read_text_file
from filter import filt_ctf, filt_table
from fundamentals import fft, window2d
nx2 = 2 * nx
ny2 = 2 * ny
if options.CTF:
from utilities import pad
for k in xrange(img_per_grp):
grp_imgdata[k] = window2d(
fft(
filt_ctf(fft(
pad(grp_imgdata[k], nx2, ny2, 1, 0.0)),
grp_imgdata[k].get_attr("ctf"),
binary=1)), nx, ny)
#grp_imgdata[k] = window2d(fft( filt_table( filt_tanl( filt_ctf(fft(pad(grp_imgdata[k], nx2, ny2, 1,0.0)), grp_imgdata[k].get_attr("ctf"), binary=1), options.fl, options.aa), fifi) ),nx,ny)
#grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl, options.aa)
'''
if i < 10 and myid == main_node:
for k in xrange(10):
grp_imgdata[k].write_image("grp%03d.hdf"%i, k)
'''
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("pp.hdf", pp)
"""
ave, grp_imgdata = prepare_2d_forPCA(grp_imgdata)
"""
if myid == main_node and i==0:
for pp in xrange(len(grp_imgdata)):
grp_imgdata[pp].write_image("qq.hdf", pp)
"""
var = model_blank(nx, ny)
for q in grp_imgdata:
Util.add_img2(var, q)
Util.mul_scalar(var, 1.0 / (len(grp_imgdata) - 1))
# Switch to std dev
var = square_root(threshold(var))
#if options.CTF: ave, var = avgvar_ctf(grp_imgdata, mode="a")
#else: ave, var = avgvar(grp_imgdata, mode="a")
"""
if myid == main_node:
ave.write_image("avgv.hdf",i)
var.write_image("varv.hdf",i)
"""
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
if options.VERBOSE:
print("%5.2f%% done on processor %d" %
(i * 100.0 / len(proj_list), myid))
if nvec > 0:
eig = pca(input_stacks=grp_imgdata,
subavg="",
mask_radius=radiuspca,
nvec=nvec,
incore=True,
shuffle=False,
genbuf=True)
for k in xrange(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
del imgdata
# To this point, all averages, variances, and eigenvectors are computed
if options.ave2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(aveList)):
aveList[im].write_image(
os.path.join(options.output_dir,
options.ave2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
"""
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol = fpol(ave, Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.ave2D), km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in xrange(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
"""
if options.ave3D:
from fundamentals import fpol
if options.VERBOSE:
print("Reconstructing 3D average volume")
ave3D = recons3d_4nn_MPI(myid,
aveList,
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
ave3D = fpol(ave3D, Tracker["nx"], Tracker["nx"],
Tracker["nx"])
ave3D.write_image(
os.path.join(options.output_dir, options.ave3D))
msg = ("%-70s: %s\n" % (
"Writing to the disk volume reconstructed from averages as",
options.ave3D))
log_main.add(msg)
print(line, msg)
del ave, var, proj_list, stack, phi, theta, psi, s2x, s2y, alpha, sx, sy, mirror, aveList
if nvec > 0:
for k in xrange(nvec):
if options.VERBOSE:
print("Reconstruction eigenvolumes", k)
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid,
eigList[k],
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image(
os.path.join(options.outpout_dir,
"eig3d_%03d.hdf" % (k, ITER)))
Util.mul_img(eig3D,
model_circle(radiuspca, nx, nx, nx))
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in xrange(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(
eigList[k][l])
proj = prgs(eig3Df, kb,
[phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node,
MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM,
main_node, MPI_COMM_WORLD)
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S",
localtime()) + " =>"
u = int(u[0])
msg = (" Eigenvector: ", k, " number changed ",
int(icont[0]))
log_main.add(msg)
print(line, msg)
else:
u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
if options.ave3D: del ave3D
if options.var2D:
from fundamentals import fpol
if myid == main_node:
km = 0
for i in xrange(number_of_proc):
if i == main_node:
for im in xrange(len(varList)):
tmpvol = fpol(varList[im], Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.var2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in xrange(nl):
ave = recv_EMData(i, im + i + 70000)
tmpvol = fpol(ave, Tracker["nx"],
Tracker["nx"], 1)
tmpvol.write_image(
os.path.join(options.output_dir,
options.var2D, km))
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in xrange(len(varList)):
send_EMData(varList[im], main_node, im + myid +
70000) # What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node and options.VERBOSE:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("Reconstructing 3D variability volume")
log_main.add(msg)
print(line, msg)
t6 = time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid,
varList,
symmetry=options.sym,
npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from fundamentals import fpol
res = fpol(res, Tracker["nx"], Tracker["nx"], Tracker["nx"])
res.write_image(os.path.join(options.output_dir,
options.var3D))
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Reconstructing 3D variability took [s]", time() - t6))
log_main.add(msg)
print(line, msg)
if options.VERBOSE:
print("Reconstruction took: %.2f [min]" %
((time() - t6) / 60))
if myid == main_node:
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("%-70s: %.2f\n" %
("Total time for these computations [s]", time() - t0))
print(line, msg)
log_main.add(msg)
if options.VERBOSE:
print("Total time for these computations: %.2f [min]" %
((time() - t0) / 60))
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
msg = ("sx3dvariability")
print(line, msg)
log_main.add(msg)
from mpi import mpi_finalize
mpi_finalize()
if RUNNING_UNDER_MPI:
global_def.MPI = False
global_def.BATCH = False
def main():
from utilities import write_text_row, drop_image, model_gauss_noise, get_im, set_params_proj, wrap_mpi_bcast, model_circle
import user_functions
from applications import MPI_start_end
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from multi_shc import multi_shc, do_volume
from logger import Logger, BaseLogger_Files
import sys
import os
import time
import socket
progname = os.path.basename(sys.argv[0])
usage = progname + " stack [output_directory] initial_volume --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --CTF --fl --aa --ref_a=S --sym=c1"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--ir", type= "int", default= 1, help="inner radius for rotational correlation > 0 (set to 1)")
parser.add_option("--ou", type= "int", default= -1, help="outer radius for rotational correlation < int(nx/2)-1 (set to the radius of the particle)")
parser.add_option("--rs", type= "int", default= 1, help="step between rings in rotational correlation >0 (set to 1)" )
parser.add_option("--xr", type="string", default= "-1", help="range for translation search in x direction, search is +/xr (default 0)")
parser.add_option("--yr", type="string", default= "-1", help="range for translation search in y direction, search is +/yr (default = same as xr)")
parser.add_option("--ts", type="string", default= "1", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional")
parser.add_option("--delta", type="string", default= "-1", help="angular step of reference projections during initialization step (default automatically selected based on radius of the structure.)")
parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches (phi and theta) (Default exhaustive searches)")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF (Default no CTF correction)")
parser.add_option("--shrink", type="float", default= 1.0, help="Reduce data size by shrink factor (default 1.0)")
parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data (default 1.0)")
parser.add_option("--ref_a", type="string", default= "S", help="method for generating the quasi-uniformly distributed projection directions (default S)")
parser.add_option("--sym", type="string", default= "c1", help="symmetry of the refined structure")
parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default=2)")
#options introduced for the do_volume function
parser.add_option("--fl", type="float", default=0.12, help="cut-off frequency of hyperbolic tangent low-pass Fourier filte (default 0.12)")
parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass Fourier filter (default 0.1)")
parser.add_option("--pwreference", type="string", default="", help="text file with a reference power spectrum (default no power spectrum adjustment)")
parser.add_option("--mask3D", type="string", default=None, help="3D mask file (default a sphere WHAT RADIUS??)")
(options, args) = parser.parse_args(sys.argv[1:])
#print( " args ",args)
if( len(args) == 3):
volinit = args[2]
masterdir = args[1]
elif(len(args) == 2):
volinit = args[1]
masterdir = ""
else:
print( "usage: " + usage)
print( "Please run '" + progname + " -h' for detailed options")
return 1
stack = args[0]
# INPUT PARAMETERS
radi = options.ou
global_def.BATCH = True
ali3d_options.ir = options.ir
ali3d_options.rs = options.rs
ali3d_options.ou = options.ou
ali3d_options.xr = options.xr
ali3d_options.yr = options.yr
ali3d_options.ts = options.ts
ali3d_options.an = "-1"
ali3d_options.sym = options.sym
ali3d_options.delta = options.delta
ali3d_options.npad = options.npad
ali3d_options.CTF = options.CTF
ali3d_options.ref_a = options.ref_a
ali3d_options.snr = options.snr
ali3d_options.mask3D = options.mask3D
ali3d_options.pwreference = "" # It will have to be turned on after exhaustive done by setting to options.pwreference
ali3d_options.fl = 0.4
ali3d_options.initfl = 0.4
ali3d_options.aa = 0.1
mpi_init(0, [])
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
# Get the pixel size, if none set to 1.0, and the original image size
if(myid == main_node):
total_stack = EMUtil.get_image_count(stack)
a = get_im(stack)
nxinit = a.get_xsize()
if ali3d_options.CTF:
i = a.get_attr('ctf')
pixel_size = i.apix
fq = pixel_size/fq
else:
pixel_size = 1.0
# No pixel size, fusing computed as 5 Fourier pixels
fq = 5.0/nxinit
del a
else:
total_stack = 0
nxinit = 0
pixel_size = 1.0
total_stack = bcast_number_to_all(total_stack, source_node = main_node)
pixel_size = bcast_number_to_all(pixel_size, source_node = main_node)
nxinit = bcast_number_to_all(nxinit, source_node = main_node)
if(radi < 1): radi = nxinit//2-2
elif((2*radi+2)>nxinit): ERROR("Particle radius set too large!","sxcenter_projections",1,myid)
ali3d_options.ou = radi
shrink = options.shrink
nxshrink = int(nxinit*shrink+0.5)
angular_neighborhood = "-1"
# MASTER DIRECTORY
if(myid == main_node):
print( " masterdir ",masterdir)
if( masterdir == ""):
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "master"+timestring
li = len(masterdir)
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
else:
li = 0
li = mpi_bcast(li,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
if( li > 0 ):
masterdir = mpi_bcast(masterdir,li,MPI_CHAR,main_node,MPI_COMM_WORLD)
masterdir = string.join(masterdir,"")
nnxo = nxinit
# INITIALIZATION
initdir = masterdir
# This is initial setting, has to be initialized here, we do not want it to run too long.
# INITIALIZATION THAT FOLLOWS WILL HAVE TO BE CHANGED SO THE USER CAN PROVIDE INITIAL GUESS OF RESOLUTION
# If we new the initial resolution, it could be done more densely
if(options.xr == "-1"): xr = "%d"%((nnxo - (2*radi-1))//2)
else: xr = options.xr
if(options.yr == "-1"): yr = xr
else: yr = options.yr
delta = float(options.delta)
if(delta <= 0.0): delta = "%f"%round(degrees(atan(1.0/float(radi))), 2)
else: delta = "%f"%delta
paramsdict = { "stack":stack,"delta":delta, "ts":"1.0", "xr":xr, "an":angular_neighborhood, \
"center":"0", "maxit":1, "local":False,\
"lowpass":options.fl, "initialfl":0.4, "falloff":options.aa, "radius":radi, \
"nsoft":0, "delpreviousmax":True, "shrink":options.shrink, "saturatecrit":1.0, "pixercutoff":2.0,\
"refvol":volinit, "mask3D":options.mask3D}
partids = os.path.join(masterdir, "ids.txt")
partstack = os.path.join(masterdir, "paramszero.txt")
if( myid == main_node ):
write_text_file(range(total_stack), partids)
write_text_row([[0.0,0.0,0.0,0.0,0.0] for i in xrange(total_stack) ], partstack)
run3Dalignment(paramsdict, partids, partstack, initdir, 0, myid, main_node, nproc)
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def runcheck(classavgstack, reconfile, outdir, inangles=None, selectdoc=None, prjmethod='trilinear', displayYN=False,
projstack='proj.hdf', outangles='angles.txt', outstack='comp-proj-reproj.hdf', normstack='comp-proj-reproj-norm.hdf'):
print("\n%s, Modified 2018-12-07\n" % __file__)
# Check if inputs exist
check(classavgstack)
check(reconfile)
# Create directory if it doesn't exist
if not os.path.isdir(outdir):
os.makedirs(outdir) # os.mkdir() can only operate one directory deep
print("mkdir -p %s" % outdir)
# Expand path for outputs
projstack = os.path.join(outdir, projstack)
outangles = os.path.join(outdir, outangles)
outstack = os.path.join(outdir, outstack)
normstack = os.path.join(outdir, normstack)
# Get number of images
nimg0 = EMAN2_cppwrap.EMUtil.get_image_count(classavgstack)
recon = EMAN2_cppwrap.EMData(reconfile)
nx = recon.get_xsize()
# In case class averages include discarded images, apply selection file
if selectdoc:
goodavgs, extension = os.path.splitext(classavgstack)
newclasses = goodavgs + "_kept" + extension
# e2proc2d appends to existing files, so rename existing output
if os.path.exists(newclasses):
renamefile = newclasses + '.bak'
os.rename(newclasses, renamefile)
print("mv %s %s" % (newclasses, renamefile))
cmd7="e2proc2d.py %s %s --list=%s" % (classavgstack, newclasses, selectdoc)
print(cmd7)
os.system(cmd7)
# Update class-averages
classavgstack = newclasses
# Import Euler angles
if inangles:
cmd6 = "sxheader.py %s --params=xform.projection --import=%s" % (classavgstack, inangles)
print(cmd6)
header(classavgstack, 'xform.projection', fimport=inangles)
try:
header(classavgstack, 'xform.projection', fexport=outangles)
cmd1 = "sxheader.py %s --params=xform.projection --export=%s" % (classavgstack, outangles)
print(cmd1)
except RuntimeError:
print("\nERROR!! No projection angles found in class-average stack header!\n")
print('Usage:', USAGE)
exit()
#cmd2="sxproject3d.py %s %s --angles=%s" % (recon, projstack, outangles)
#print(cmd2)
#os.system(cmd2)
# Here if you want to be fancy, there should be an option to chose the projection method,
# the mechanism can be copied from sxproject3d.py PAP
if prjmethod=='trilinear':
method_num = 1
elif prjmethod=='gridding':
method_num = -1
elif prjmethod=='nn':
method_num = 0
else:
print("\nERROR!! Valid projection methods are: trilinear (default), gridding, and nn (nearest neighbor).")
print('Usage:', USAGE)
exit()
#project3d(recon, stack=projstack, listagls=outangles)
recon = prep_vol(recon, npad = 2, interpolation_method = 1)
result=[]
# Here you need actual radius to compute proper ccc's, but if you do, you have to deal with translations, PAP
mask = model_circle(nx//2-2,nx,nx)
# Number of images may have changed
nimg1 = EMAN2_cppwrap.EMUtil.get_image_count(classavgstack)
outangles = read_text_row(outangles)
for imgnum in range(nimg1):
# get class average
classimg = get_im(classavgstack, imgnum)
# compute re-projection
prjimg = prgl(recon, outangles[imgnum], 1, False)
# calculate 1D power spectra
rops_dst = rops_table(classimg*mask)
rops_src = rops_table(prjimg)
# Set power spectrum of reprojection to the data.
# Since data has an envelope, it would make more sense to set data to reconstruction,
# but to do it one would have to know the actual resolution of the data.
# you can check sxprocess.py --adjpw to see how this is done properly PAP
table = [0.0]*len(rops_dst) # initialize table
for j in range( len(rops_dst) ):
table[j] = sqrt( old_div(rops_dst[j],rops_src[j]) )
prjimg = fft(filt_table(prjimg, table)) # match FFT amplitdes of re-projection and class average
cccoeff = ccc(prjimg, classimg, mask)
#print(imgnum, cccoeff)
classimg.set_attr_dict({'cross-corr':cccoeff})
prjimg.set_attr_dict({'cross-corr':cccoeff})
prjimg.write_image(outstack,2*imgnum)
classimg.write_image(outstack, 2*imgnum+1)
result.append(cccoeff)
del outangles
meanccc = old_div(sum(result),nimg1)
print("Average CCC is %s" % meanccc)
nimg2 = EMAN2_cppwrap.EMUtil.get_image_count(outstack)
for imgnum in xrange(nimg2):
if (imgnum % 2 ==0):
prjimg = get_im(outstack,imgnum)
meanccc1 = prjimg.get_attr_default('mean-cross-corr', -1.0)
prjimg.set_attr_dict({'mean-cross-corr':meanccc})
write_header(outstack,prjimg,imgnum)
if (imgnum % 100) == 0:
print(imgnum)
# e2proc2d appends to existing files, so delete existing output
if os.path.exists(normstack):
os.remove(normstack)
print("rm %s" % normstack)
# Why would you want to do it? If you do, it should have been done during ccc calculations,
# otherwise what is see is not corresponding to actual data, thus misleading. PAP
#cmd5="e2proc2d.py %s %s --process=normalize" % (outstack, normstack)
#print(cmd5)
#os.system(cmd5)
# Optionally pop up e2display
if displayYN:
cmd8 = "e2display.py %s" % outstack
print(cmd8)
os.system(cmd8)
print("Done!")
def main():
"""
Main function.
Arguments:
None
Returns:
None
"""
command_args = parse_command_line()
# Import volume
print('Import volume.')
input_vol = utilities.get_im(command_args.input_volume)
# Sanity checks
sanity_checks(command_args, input_vol)
try:
os.makedirs(command_args.output_dir)
except OSError:
print('Output directory already exists. No need to create it.')
else:
print('Created output directory.')
output_prefix = os.path.join(command_args.output_dir, command_args.prefix)
# Filter volume if specified
if command_args.low_pass_filter_resolution is not None:
print('Filter volume to {0}A.'.format(
command_args.low_pass_filter_resolution))
input_vol = sparx_filter.filt_tanl(
input_vol,
command_args.pixel_size / command_args.low_pass_filter_resolution,
command_args.low_pass_filter_falloff)
input_vol.write_image(output_prefix + '_filtered_volume.hdf')
else:
print('Skip filter volume.')
# Create a mask based on the filtered volume
print('Create mask')
density_threshold = -9999.0
nsigma = 1.0
if command_args.mol_mass:
density_threshold = input_vol.find_3d_threshold(
command_args.mol_mass, command_args.pixel_size)
elif command_args.threshold:
density_threshold = command_args.threshold
elif command_args.nsigma:
nsigma = command_args.nsigma
else:
assert False
if command_args.edge_type == 'cosine':
mode = 'C'
elif command_args.edge_type == 'gaussian':
mode = 'G'
else:
assert False
mask_first = morphology.adaptive_mask_scipy(
input_vol,
nsigma=nsigma,
threshold=density_threshold,
ndilation=command_args.ndilation,
nerosion=command_args.nerosion,
edge_width=command_args.edge_width,
allow_disconnected=command_args.allow_disconnected,
mode=mode,
do_approx=command_args.do_old,
)
# Create a second mask based on the filtered volume
s_mask = None
s_density_threshold = 1
s_nsigma = 1.0
if command_args.second_mask is not None:
s_mask = utilities.get_im(command_args.second_mask)
density_threshold = -9999.0
nsigma = 1.0
if command_args.s_mol_mass:
s_density_threshold = input_vol.find_3d_threshold(
command_args.s_mol_mass, command_args.s_pixel_size)
elif command_args.s_threshold:
s_density_threshold = command_args.s_threshold
elif command_args.s_nsigma:
s_nsigma = command_args.s_nsigma
else:
assert False
elif command_args.second_mask_shape is not None:
nx = mask_first.get_xsize()
ny = mask_first.get_ysize()
nz = mask_first.get_zsize()
if command_args.second_mask_shape == 'cube':
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
s_mask = utilities.model_blank(s_nx, s_ny, s_nz, 1)
elif command_args.second_mask_shape == 'cylinder':
s_radius = command_args.s_radius
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
s_mask = utilities.model_cylinder(s_radius, s_nx, s_ny, s_nz)
elif command_args.second_mask_shape == 'sphere':
s_radius = command_args.s_radius
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
s_mask = utilities.model_circle(s_radius, s_nx, s_ny, s_nz)
else:
assert False
s_mask = utilities.pad(s_mask, nx, ny, nz, 0)
if s_mask is not None:
print('Create second mask')
if command_args.s_edge_type == 'cosine':
mode = 'C'
elif command_args.s_edge_type == 'gaussian':
mode = 'G'
else:
assert False
s_mask = morphology.adaptive_mask_scipy(
s_mask,
nsigma=s_nsigma,
threshold=s_density_threshold,
ndilation=command_args.s_ndilation,
nerosion=command_args.s_nerosion,
edge_width=command_args.s_edge_width,
allow_disconnected=command_args.s_allow_disconnected,
mode=mode,
do_approx=command_args.s_do_old)
if command_args.s_invert:
s_mask = 1 - s_mask
mask_first.write_image(output_prefix + '_mask_first.hdf')
s_mask.write_image(output_prefix + '_mask_second.hdf')
masked_combined = mask_first * s_mask
masked_combined.write_image(output_prefix + '_mask.hdf')
else:
mask_first.write_image(output_prefix + '_mask.hdf')
def calculate_volumes_after_rotation_and_save_them(ali3d_options, rviper_iter, masterdir, bdb_stack_location, mpi_rank, mpi_size,
no_of_viper_runs_analyzed_together, no_of_viper_runs_analyzed_together_from_user_options, mpi_comm = -1):
# This function takes into account the case in which there are more processors than images
if mpi_comm == -1:
mpi_comm = MPI_COMM_WORLD
# some arguments are for debugging purposes
mainoutputdir = masterdir + DIR_DELIM + NAME_OF_MAIN_DIR + ("%03d" + DIR_DELIM) %(rviper_iter)
# list_of_projection_indices_used_for_outlier_elimination = map(int, read_text_file(mainoutputdir + DIR_DELIM + "list_of_viper_runs_included_in_outlier_elimination.txt"))
import json; f = open(mainoutputdir + "list_of_viper_runs_included_in_outlier_elimination.json", 'r')
list_of_independent_viper_run_indices_used_for_outlier_elimination = json.load(f); f.close()
if len(list_of_independent_viper_run_indices_used_for_outlier_elimination)==0:
print "Error: len(list_of_independent_viper_run_indices_used_for_outlier_elimination)==0"
mpi_finalize()
sys.exit()
# if this data analysis step was already performed in the past then return
# for future changes make sure that the file checked is the last one to be processed !!!
# if(os.path.exists(mainoutputdir + DIR_DELIM + NAME_OF_RUN_DIR + "%03d"%(no_of_viper_runs_analyzed_together - 1) + DIR_DELIM + "rotated_volume.hdf")):
# check_last_run = max(get_latest_directory_increment_value(mainoutputdir, NAME_OF_RUN_DIR, start_value=0), no_of_viper_runs_analyzed_together_from_user_options)
# if(os.path.exists(mainoutputdir + DIR_DELIM + NAME_OF_RUN_DIR + "%03d"%(check_last_run) + DIR_DELIM + "rotated_volume.hdf")):
# return
# if this data analysis step was already performed in the past then return
for check_run in list_of_independent_viper_run_indices_used_for_outlier_elimination:
if not (os.path.exists(mainoutputdir + DIR_DELIM + NAME_OF_RUN_DIR + "%03d"%(check_run) + DIR_DELIM + "rotated_volume.hdf")):
break
else:
return
partstack = []
# for i1 in range(0,no_of_viper_runs_analyzed_together):
for i1 in list_of_independent_viper_run_indices_used_for_outlier_elimination:
partstack.append(mainoutputdir + NAME_OF_RUN_DIR + "%03d"%(i1) + DIR_DELIM + "rotated_reduced_params.txt")
partids_file_name = mainoutputdir + "this_iteration_index_keep_images.txt"
lpartids = map(int, read_text_file(partids_file_name) )
n_projs = len(lpartids)
if (mpi_size > n_projs):
# if there are more processors than images
working = int(not(mpi_rank < n_projs))
mpi_subcomm = mpi_comm_split(mpi_comm, working, mpi_rank - working*n_projs)
mpi_subsize = mpi_comm_size(mpi_subcomm)
mpi_subrank = mpi_comm_rank(mpi_subcomm)
if (mpi_rank < n_projs):
# for i in xrange(no_of_viper_runs_analyzed_together):
for idx, i in enumerate(list_of_independent_viper_run_indices_used_for_outlier_elimination):
projdata = getindexdata(bdb_stack_location + "_%03d"%(rviper_iter - 1), partids_file_name, partstack[idx], mpi_rank, mpi_subsize)
vol = do_volume(projdata, ali3d_options, 0, mpi_comm = mpi_subcomm)
del projdata
if( mpi_rank == 0):
vol.write_image(mainoutputdir + DIR_DELIM + NAME_OF_RUN_DIR + "%03d"%(i) + DIR_DELIM + "rotated_volume.hdf")
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " => "
print line + "Generated rec_ref_volume_run #%01d \n"%i
del vol
mpi_barrier(mpi_comm)
else:
for idx, i in enumerate(list_of_independent_viper_run_indices_used_for_outlier_elimination):
projdata = getindexdata(bdb_stack_location + "_%03d"%(rviper_iter - 1), partids_file_name, partstack[idx], mpi_rank, mpi_size)
vol = do_volume(projdata, ali3d_options, 0, mpi_comm = mpi_comm)
del projdata
if( mpi_rank == 0):
vol.write_image(mainoutputdir + DIR_DELIM + NAME_OF_RUN_DIR + "%03d"%(i) + DIR_DELIM + "rotated_volume.hdf")
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " => "
print line + "Generated rec_ref_volume_run #%01d"%i
del vol
if( mpi_rank == 0):
# Align all rotated volumes, calculate their average and save as an overall result
from utilities import get_params3D, set_params3D, get_im, model_circle
from statistics import ave_var
from applications import ali_vol
# vls = [None]*no_of_viper_runs_analyzed_together
vls = [None]*len(list_of_independent_viper_run_indices_used_for_outlier_elimination)
# for i in xrange(no_of_viper_runs_analyzed_together):
for idx, i in enumerate(list_of_independent_viper_run_indices_used_for_outlier_elimination):
vls[idx] = get_im(mainoutputdir + DIR_DELIM + NAME_OF_RUN_DIR + "%03d"%(i) + DIR_DELIM + "rotated_volume.hdf")
set_params3D(vls[idx],[0.,0.,0.,0.,0.,0.,0,1.0])
asa,sas = ave_var(vls)
# do the alignment
nx = asa.get_xsize()
radius = nx/2 - .5
st = Util.infomask(asa*asa, model_circle(radius,nx,nx,nx), True)
goal = st[0]
going = True
while(going):
set_params3D(asa,[0.,0.,0.,0.,0.,0.,0,1.0])
# for i in xrange(no_of_viper_runs_analyzed_together):
for idx, i in enumerate(list_of_independent_viper_run_indices_used_for_outlier_elimination):
o = ali_vol(vls[idx],asa,7.0,5.,radius) # range of angles and shifts, maybe should be adjusted
p = get_params3D(o)
del o
set_params3D(vls[idx],p)
asa,sas = ave_var(vls)
st = Util.infomask(asa*asa, model_circle(radius,nx,nx,nx), True)
if(st[0] > goal): goal = st[0]
else: going = False
# over and out
asa.write_image(mainoutputdir + DIR_DELIM + "average_volume.hdf")
sas.write_image(mainoutputdir + DIR_DELIM + "variance_volume.hdf")
return
def generate_helimic(refvol,
outdir,
pixel,
CTF=False,
Cs=2.0,
voltage=200.0,
ampcont=10.0,
nonoise=False,
rand_seed=14567):
from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im
from random import random
from projection import prgs, prep_vol
from filter import filt_gaussl, filt_ctf
from EMAN2 import EMAN2Ctf
if os.path.exists(outdir):
ERROR(
'Output directory exists, please change the name and restart the program',
"sxhelical_demo", 1)
os.mkdir(outdir)
seed(rand_seed)
Util.set_randnum_seed(rand_seed)
angles = []
for i in range(3):
angles.append([0.0 + 60.0 * i, 90.0 - i * 5, 0.0, 0.0, 0.0])
nangle = len(angles)
volfts = get_im(refvol)
nx = volfts.get_xsize()
ny = volfts.get_ysize()
nz = volfts.get_zsize()
volfts, kbx, kby, kbz = prep_vol(volfts)
iprj = 0
width = 500
xstart = 0
ystart = 0
for idef in range(3, 6):
mic = model_blank(2048, 2048)
#defocus = idef*0.2
defocus = idef * 0.6 ##@ming
if CTF:
#ctf = EMAN2Ctf()
#ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
from utilities import generate_ctf
ctf = generate_ctf(
[defocus, 2, 200, 1.84, 0.0, ampcont, defocus * 0.2, 80]
) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
i = idef - 4
for k in range(1):
psi = 90 + 10 * i
proj = prgs(
volfts, kbz,
[angles[idef - 3][0], angles[idef - 3][1], psi, 0.0, 0.0], kbx,
kby)
proj = Util.window(proj, 320, nz)
mic += pad(proj, 2048, 2048, 1, 0.0, 750 * i, 20 * i, 0)
if not nonoise: mic += model_gauss_noise(30.0, 2048, 2048)
if CTF:
#apply CTF
mic = filt_ctf(mic, ctf)
if not nonoise:
mic += filt_gaussl(model_gauss_noise(17.5, 2048, 2048), 0.3)
mic.write_image("%s/mic%1d.hdf" % (outdir, idef - 3), 0)
def main():
from logger import Logger, BaseLogger_Files
arglist = []
i = 0
while( i < len(sys.argv) ):
if sys.argv[i]=='-p4pg':
i = i+2
elif sys.argv[i]=='-p4wd':
i = i+2
else:
arglist.append( sys.argv[i] )
i = i+1
progname = os.path.basename(arglist[0])
usage = progname + " stack outdir <mask> --focus=3Dmask --radius=outer_radius --delta=angular_step" +\
"--an=angular_neighborhood --maxit=max_iter --CTF --sym=c1 --function=user_function --independent=indenpendent_runs --number_of_images_per_group=number_of_images_per_group --low_pass_frequency=.25 --seed=random_seed"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--focus", type ="string", default ='', help="bineary 3D mask for focused clustering ")
parser.add_option("--ir", type = "int", default =1, help="inner radius for rotational correlation > 0 (set to 1)")
parser.add_option("--radius", type = "int", default =-1, help="particle radius in pixel for rotational correlation <nx-1 (set to the radius of the particle)")
parser.add_option("--maxit", type = "int", default =25, help="maximum number of iteration")
parser.add_option("--rs", type = "int", default =1, help="step between rings in rotational correlation >0 (set to 1)" )
parser.add_option("--xr", type ="string", default ='1', help="range for translation search in x direction, search is +/-xr ")
parser.add_option("--yr", type ="string", default ='-1', help="range for translation search in y direction, search is +/-yr (default = same as xr)")
parser.add_option("--ts", type ="string", default ='0.25', help="step size of the translation search in both directions direction, search is -xr, -xr+ts, 0, xr-ts, xr ")
parser.add_option("--delta", type ="string", default ='2', help="angular step of reference projections")
parser.add_option("--an", type ="string", default ='-1', help="angular neighborhood for local searches")
parser.add_option("--center", type ="int", default =0, help="0 - if you do not want the volume to be centered, 1 - center the volume using cog (default=0)")
parser.add_option("--nassign", type ="int", default =1, help="number of reassignment iterations performed for each angular step (set to 3) ")
parser.add_option("--nrefine", type ="int", default =0, help="number of alignment iterations performed for each angular step (set to 0)")
parser.add_option("--CTF", action ="store_true", default =False, help="do CTF correction during clustring")
parser.add_option("--stoprnct", type ="float", default =3.0, help="Minimum percentage of assignment change to stop the program")
parser.add_option("--sym", type ="string", default ='c1', help="symmetry of the structure ")
parser.add_option("--function", type ="string", default ='do_volume_mrk05', help="name of the reference preparation function")
parser.add_option("--independent", type ="int", default = 3, help="number of independent run")
parser.add_option("--number_of_images_per_group", type ="int", default =1000, help="number of groups")
parser.add_option("--low_pass_filter", type ="float", default =-1.0, help="absolute frequency of low-pass filter for 3d sorting on the original image size" )
parser.add_option("--nxinit", type ="int", default =64, help="initial image size for sorting" )
parser.add_option("--unaccounted", action ="store_true", default =False, help="reconstruct the unaccounted images")
parser.add_option("--seed", type ="int", default =-1, help="random seed for create initial random assignment for EQ Kmeans")
parser.add_option("--smallest_group", type ="int", default =500, help="minimum members for identified group")
parser.add_option("--sausage", action ="store_true", default =False, help="way of filter volume")
parser.add_option("--chunkdir", type ="string", default ='', help="chunkdir for computing margin of error")
parser.add_option("--PWadjustment", type ="string", default ='', help="1-D power spectrum of PDB file used for EM volume power spectrum correction")
parser.add_option("--protein_shape", type ="string", default ='g', help="protein shape. It defines protein preferred orientation angles. Currently it has g and f two types ")
parser.add_option("--upscale", type ="float", default =0.5, help=" scaling parameter to adjust the power spectrum of EM volumes")
parser.add_option("--wn", type ="int", default =0, help="optimal window size for data processing")
parser.add_option("--interpolation", type ="string", default ="4nn", help="3-d reconstruction interpolation method, two options trl and 4nn")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 1 or len(args) > 4:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
else:
if len(args)>2:
mask_file = args[2]
else:
mask_file = None
orgstack =args[0]
masterdir =args[1]
global_def.BATCH = True
#---initialize MPI related variables
from mpi import mpi_init, mpi_comm_size, MPI_COMM_WORLD, mpi_comm_rank,mpi_barrier,mpi_bcast, mpi_bcast, MPI_INT,MPI_CHAR
sys.argv = mpi_init(len(sys.argv),sys.argv)
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
mpi_comm = MPI_COMM_WORLD
main_node= 0
# import some utilities
from utilities import get_im,bcast_number_to_all,cmdexecute,write_text_file,read_text_file,wrap_mpi_bcast, get_params_proj, write_text_row
from applications import recons3d_n_MPI, mref_ali3d_MPI, Kmref_ali3d_MPI
from statistics import k_means_match_clusters_asg_new,k_means_stab_bbenum
from applications import mref_ali3d_EQ_Kmeans, ali3d_mref_Kmeans_MPI
# Create the main log file
from logger import Logger,BaseLogger_Files
if myid ==main_node:
log_main=Logger(BaseLogger_Files())
log_main.prefix = masterdir+"/"
else:
log_main =None
#--- fill input parameters into dictionary named after Constants
Constants ={}
Constants["stack"] = args[0]
Constants["masterdir"] = masterdir
Constants["mask3D"] = mask_file
Constants["focus3Dmask"] = options.focus
Constants["indep_runs"] = options.independent
Constants["stoprnct"] = options.stoprnct
Constants["number_of_images_per_group"] = options.number_of_images_per_group
Constants["CTF"] = options.CTF
Constants["maxit"] = options.maxit
Constants["ir"] = options.ir
Constants["radius"] = options.radius
Constants["nassign"] = options.nassign
Constants["rs"] = options.rs
Constants["xr"] = options.xr
Constants["yr"] = options.yr
Constants["ts"] = options.ts
Constants["delta"] = options.delta
Constants["an"] = options.an
Constants["sym"] = options.sym
Constants["center"] = options.center
Constants["nrefine"] = options.nrefine
#Constants["fourvar"] = options.fourvar
Constants["user_func"] = options.function
Constants["low_pass_filter"] = options.low_pass_filter # enforced low_pass_filter
#Constants["debug"] = options.debug
Constants["main_log_prefix"] = args[1]
#Constants["importali3d"] = options.importali3d
Constants["myid"] = myid
Constants["main_node"] = main_node
Constants["nproc"] = nproc
Constants["log_main"] = log_main
Constants["nxinit"] = options.nxinit
Constants["unaccounted"] = options.unaccounted
Constants["seed"] = options.seed
Constants["smallest_group"] = options.smallest_group
Constants["sausage"] = options.sausage
Constants["chunkdir"] = options.chunkdir
Constants["PWadjustment"] = options.PWadjustment
Constants["upscale"] = options.upscale
Constants["wn"] = options.wn
Constants["3d-interpolation"] = options.interpolation
Constants["protein_shape"] = options.protein_shape
# -----------------------------------------------------
#
# Create and initialize Tracker dictionary with input options
Tracker = {}
Tracker["constants"] = Constants
Tracker["maxit"] = Tracker["constants"]["maxit"]
Tracker["radius"] = Tracker["constants"]["radius"]
#Tracker["xr"] = ""
#Tracker["yr"] = "-1" # Do not change!
#Tracker["ts"] = 1
#Tracker["an"] = "-1"
#Tracker["delta"] = "2.0"
#Tracker["zoom"] = True
#Tracker["nsoft"] = 0
#Tracker["local"] = False
#Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
Tracker["upscale"] = Tracker["constants"]["upscale"]
#Tracker["upscale"] = 0.5
Tracker["applyctf"] = False # Should the data be premultiplied by the CTF. Set to False for local continuous.
#Tracker["refvol"] = None
Tracker["nxinit"] = Tracker["constants"]["nxinit"]
#Tracker["nxstep"] = 32
Tracker["icurrentres"] = -1
#Tracker["ireachedres"] = -1
#Tracker["lowpass"] = 0.4
#Tracker["falloff"] = 0.2
#Tracker["inires"] = options.inires # Now in A, convert to absolute before using
Tracker["fuse_freq"] = 50 # Now in A, convert to absolute before using
#Tracker["delpreviousmax"] = False
#Tracker["anger"] = -1.0
#Tracker["shifter"] = -1.0
#Tracker["saturatecrit"] = 0.95
#Tracker["pixercutoff"] = 2.0
#Tracker["directory"] = ""
#Tracker["previousoutputdir"] = ""
#Tracker["eliminated-outliers"] = False
#Tracker["mainiteration"] = 0
#Tracker["movedback"] = False
#Tracker["state"] = Tracker["constants"]["states"][0]
#Tracker["global_resolution"] =0.0
Tracker["orgstack"] = orgstack
#--------------------------------------------------------------------
# import from utilities
from utilities import sample_down_1D_curve,get_initial_ID,remove_small_groups,print_upper_triangular_matrix,print_a_line_with_timestamp
from utilities import print_dict,get_resolution_mrk01,partition_to_groups,partition_independent_runs,get_outliers
from utilities import merge_groups, save_alist, margin_of_error, get_margin_of_error, do_two_way_comparison, select_two_runs, get_ali3d_params
from utilities import counting_projections, unload_dict, load_dict, get_stat_proj, create_random_list, get_number_of_groups, recons_mref
from utilities import apply_low_pass_filter, get_groups_from_partition, get_number_of_groups, get_complementary_elements_total, update_full_dict
from utilities import count_chunk_members, set_filter_parameters_from_adjusted_fsc, adjust_fsc_down, get_two_chunks_from_stack
####------------------------------------------------------------------
#
# Get the pixel size; if none, set to 1.0, and the original image size
from utilities import get_shrink_data_huang
if(myid == main_node):
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line+"Initialization of 3-D sorting")
a = get_im(orgstack)
nnxo = a.get_xsize()
if( Tracker["nxinit"] > nnxo ):
ERROR("Image size less than minimum permitted $d"%Tracker["nxinit"],"sxsort3d.py",1)
nnxo = -1
else:
if Tracker["constants"]["CTF"]:
i = a.get_attr('ctf')
pixel_size = i.apix
fq = pixel_size/Tracker["fuse_freq"]
else:
pixel_size = 1.0
# No pixel size, fusing computed as 5 Fourier pixels
fq = 5.0/nnxo
del a
else:
nnxo = 0
fq = 0.0
pixel_size = 1.0
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
if( nnxo < 0 ):
mpi_finalize()
exit()
pixel_size = bcast_number_to_all(pixel_size, source_node = main_node)
fq = bcast_number_to_all(fq, source_node = main_node)
if Tracker["constants"]["wn"]==0:
Tracker["constants"]["nnxo"] = nnxo
else:
Tracker["constants"]["nnxo"] = Tracker["constants"]["wn"]
nnxo = Tracker["constants"]["nnxo"]
Tracker["constants"]["pixel_size"] = pixel_size
Tracker["fuse_freq"] = fq
del fq, nnxo, pixel_size
if(Tracker["constants"]["radius"] < 1):
Tracker["constants"]["radius"] = Tracker["constants"]["nnxo"]//2-2
elif((2*Tracker["constants"]["radius"] +2) > Tracker["constants"]["nnxo"]):
ERROR("Particle radius set too large!","sxsort3d.py",1,myid)
####-----------------------------------------------------------------------------------------
# Master directory
if myid == main_node:
if masterdir =="":
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir ="master_sort3d"+timestring
li =len(masterdir)
cmd="{} {}".format("mkdir", masterdir)
os.system(cmd)
else:
li=0
li = mpi_bcast(li,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
if li>0:
masterdir = mpi_bcast(masterdir,li,MPI_CHAR,main_node,MPI_COMM_WORLD)
import string
masterdir = string.join(masterdir,"")
if myid ==main_node:
print_dict(Tracker["constants"],"Permanent settings of 3-D sorting program")
######### create a vstack from input stack to the local stack in masterdir
# stack name set to default
Tracker["constants"]["stack"] = "bdb:"+masterdir+"/rdata"
Tracker["constants"]["ali3d"] = os.path.join(masterdir, "ali3d_init.txt")
Tracker["constants"]["ctf_params"] = os.path.join(masterdir, "ctf_params.txt")
Tracker["constants"]["partstack"] = Tracker["constants"]["ali3d"] # also serves for refinement
if myid == main_node:
total_stack = EMUtil.get_image_count(Tracker["orgstack"])
else:
total_stack = 0
total_stack = bcast_number_to_all(total_stack, source_node = main_node)
mpi_barrier(MPI_COMM_WORLD)
from time import sleep
while not os.path.exists(masterdir):
print "Node ",myid," waiting..."
sleep(5)
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("Sphire sort3d ")
log_main.add("the sort3d master directory is "+masterdir)
#####
###----------------------------------------------------------------------------------
# Initial data analysis and handle two chunk files
from random import shuffle
# Compute the resolution
#### make chunkdir dictionary for computing margin of error
import user_functions
user_func = user_functions.factory[Tracker["constants"]["user_func"]]
chunk_dict = {}
chunk_list = []
if myid == main_node:
chunk_one = read_text_file(os.path.join(Tracker["constants"]["chunkdir"],"chunk0.txt"))
chunk_two = read_text_file(os.path.join(Tracker["constants"]["chunkdir"],"chunk1.txt"))
else:
chunk_one = 0
chunk_two = 0
chunk_one = wrap_mpi_bcast(chunk_one, main_node)
chunk_two = wrap_mpi_bcast(chunk_two, main_node)
mpi_barrier(MPI_COMM_WORLD)
######################## Read/write bdb: data on main node ############################
if myid==main_node:
if(orgstack[:4] == "bdb:"): cmd = "{} {} {}".format("e2bdb.py", orgstack,"--makevstack="+Tracker["constants"]["stack"])
else: cmd = "{} {} {}".format("sxcpy.py", orgstack, Tracker["constants"]["stack"])
cmdexecute(cmd)
cmd = "{} {} {}".format("sxheader.py --params=xform.projection", "--export="+Tracker["constants"]["ali3d"],orgstack)
cmdexecute(cmd)
cmd = "{} {} {}".format("sxheader.py --params=ctf", "--export="+Tracker["constants"]["ctf_params"],orgstack)
cmdexecute(cmd)
mpi_barrier(MPI_COMM_WORLD)
########-----------------------------------------------------------------------------
Tracker["total_stack"] = total_stack
Tracker["constants"]["total_stack"] = total_stack
Tracker["shrinkage"] = float(Tracker["nxinit"])/Tracker["constants"]["nnxo"]
Tracker["radius"] = Tracker["constants"]["radius"]*Tracker["shrinkage"]
if Tracker["constants"]["mask3D"]:
Tracker["mask3D"] = os.path.join(masterdir,"smask.hdf")
else:
Tracker["mask3D"] = None
if Tracker["constants"]["focus3Dmask"]:
Tracker["focus3D"] = os.path.join(masterdir,"sfocus.hdf")
else:
Tracker["focus3D"] = None
if myid == main_node:
if Tracker["constants"]["mask3D"]:
mask_3D = get_shrink_3dmask(Tracker["nxinit"],Tracker["constants"]["mask3D"])
mask_3D.write_image(Tracker["mask3D"])
if Tracker["constants"]["focus3Dmask"]:
mask_3D = get_shrink_3dmask(Tracker["nxinit"],Tracker["constants"]["focus3Dmask"])
st = Util.infomask(mask_3D, None, True)
if( st[0] == 0.0 ): ERROR("sxrsort3d","incorrect focused mask, after binarize all values zero",1)
mask_3D.write_image(Tracker["focus3D"])
del mask_3D
if Tracker["constants"]["PWadjustment"] !='':
PW_dict = {}
nxinit_pwsp = sample_down_1D_curve(Tracker["constants"]["nxinit"],Tracker["constants"]["nnxo"],Tracker["constants"]["PWadjustment"])
Tracker["nxinit_PW"] = os.path.join(masterdir,"spwp.txt")
if myid == main_node: write_text_file(nxinit_pwsp,Tracker["nxinit_PW"])
PW_dict[Tracker["constants"]["nnxo"]] = Tracker["constants"]["PWadjustment"]
PW_dict[Tracker["constants"]["nxinit"]] = Tracker["nxinit_PW"]
Tracker["PW_dict"] = PW_dict
mpi_barrier(MPI_COMM_WORLD)
#-----------------------From two chunks to FSC, and low pass filter-----------------------------------------###
for element in chunk_one: chunk_dict[element] = 0
for element in chunk_two: chunk_dict[element] = 1
chunk_list =[chunk_one, chunk_two]
Tracker["chunk_dict"] = chunk_dict
Tracker["P_chunk0"] = len(chunk_one)/float(total_stack)
Tracker["P_chunk1"] = len(chunk_two)/float(total_stack)
### create two volumes to estimate resolution
if myid == main_node:
for index in xrange(2): write_text_file(chunk_list[index],os.path.join(masterdir,"chunk%01d.txt"%index))
mpi_barrier(MPI_COMM_WORLD)
vols = []
for index in xrange(2):
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nxinit"], os.path.join(masterdir,"chunk%01d.txt"%index), Tracker["constants"]["partstack"],myid,main_node,nproc,preshift=True)
vol = recons3d_4nn_ctf_MPI(myid=myid, prjlist=data,symmetry=Tracker["constants"]["sym"], finfo=None)
if myid == main_node:
vol.write_image(os.path.join(masterdir, "vol%d.hdf"%index))
vols.append(vol)
mpi_barrier(MPI_COMM_WORLD)
if myid ==main_node:
low_pass, falloff,currentres = get_resolution_mrk01(vols,Tracker["constants"]["radius"],Tracker["constants"]["nxinit"],masterdir,Tracker["mask3D"])
if low_pass >Tracker["constants"]["low_pass_filter"]: low_pass= Tracker["constants"]["low_pass_filter"]
else:
low_pass =0.0
falloff =0.0
currentres =0.0
bcast_number_to_all(currentres,source_node = main_node)
bcast_number_to_all(low_pass,source_node = main_node)
bcast_number_to_all(falloff,source_node = main_node)
Tracker["currentres"] = currentres
Tracker["falloff"] = falloff
if Tracker["constants"]["low_pass_filter"] ==-1.0:
Tracker["low_pass_filter"] = min(.45,low_pass/Tracker["shrinkage"]) # no better than .45
else:
Tracker["low_pass_filter"] = min(.45,Tracker["constants"]["low_pass_filter"]/Tracker["shrinkage"])
Tracker["lowpass"] = Tracker["low_pass_filter"]
Tracker["falloff"] =.1
Tracker["global_fsc"] = os.path.join(masterdir, "fsc.txt")
############################################################################################
if myid == main_node:
log_main.add("The command-line inputs are as following:")
log_main.add("**********************************************************")
for a in sys.argv:
if myid == main_node:log_main.add(a)
if myid == main_node:
log_main.add("number of cpus used in this run is %d"%Tracker["constants"]["nproc"])
log_main.add("**********************************************************")
from filter import filt_tanl
### START 3-D sorting
if myid ==main_node:
log_main.add("----------3-D sorting program------- ")
log_main.add("current resolution %6.3f for images of original size in terms of absolute frequency"%Tracker["currentres"])
log_main.add("equivalent to %f Angstrom resolution"%(Tracker["constants"]["pixel_size"]/Tracker["currentres"]/Tracker["shrinkage"]))
log_main.add("the user provided enforced low_pass_filter is %f"%Tracker["constants"]["low_pass_filter"])
#log_main.add("equivalent to %f Angstrom resolution"%(Tracker["constants"]["pixel_size"]/Tracker["constants"]["low_pass_filter"]))
for index in xrange(2):
filt_tanl(get_im(os.path.join(masterdir,"vol%01d.hdf"%index)), Tracker["low_pass_filter"],Tracker["falloff"]).write_image(os.path.join(masterdir, "volf%01d.hdf"%index))
mpi_barrier(MPI_COMM_WORLD)
from utilities import get_input_from_string
delta = get_input_from_string(Tracker["constants"]["delta"])
delta = delta[0]
from utilities import even_angles
n_angles = even_angles(delta, 0, 180)
this_ali3d = Tracker["constants"]["ali3d"]
sampled = get_stat_proj(Tracker,delta,this_ali3d)
if myid ==main_node:
nc = 0
for a in sampled:
if len(sampled[a])>0:
nc += 1
log_main.add("total sampled direction %10d at angle step %6.3f"%(len(n_angles), delta))
log_main.add("captured sampled directions %10d percentage covered by data %6.3f"%(nc,float(nc)/len(n_angles)*100))
number_of_images_per_group = Tracker["constants"]["number_of_images_per_group"]
if myid ==main_node: log_main.add("user provided number_of_images_per_group %d"%number_of_images_per_group)
Tracker["number_of_images_per_group"] = number_of_images_per_group
number_of_groups = get_number_of_groups(total_stack,number_of_images_per_group)
Tracker["number_of_groups"] = number_of_groups
generation =0
partition_dict ={}
full_dict ={}
workdir =os.path.join(masterdir,"generation%03d"%generation)
Tracker["this_dir"] = workdir
if myid ==main_node:
log_main.add("---- generation %5d"%generation)
log_main.add("number of images per group is set as %d"%number_of_images_per_group)
log_main.add("the initial number of groups is %10d "%number_of_groups)
cmd="{} {}".format("mkdir",workdir)
os.system(cmd)
mpi_barrier(MPI_COMM_WORLD)
list_to_be_processed = range(Tracker["constants"]["total_stack"])
Tracker["this_data_list"] = list_to_be_processed
create_random_list(Tracker)
#################################
full_dict ={}
for iptl in xrange(Tracker["constants"]["total_stack"]):
full_dict[iptl] = iptl
Tracker["full_ID_dict"] = full_dict
#################################
for indep_run in xrange(Tracker["constants"]["indep_runs"]):
Tracker["this_particle_list"] = Tracker["this_indep_list"][indep_run]
ref_vol = recons_mref(Tracker)
if myid == main_node: log_main.add("independent run %10d"%indep_run)
mpi_barrier(MPI_COMM_WORLD)
Tracker["this_data_list"] = list_to_be_processed
Tracker["total_stack"] = len(Tracker["this_data_list"])
Tracker["this_particle_text_file"] = os.path.join(workdir,"independent_list_%03d.txt"%indep_run) # for get_shrink_data
if myid == main_node: write_text_file(Tracker["this_data_list"], Tracker["this_particle_text_file"])
mpi_barrier(MPI_COMM_WORLD)
outdir = os.path.join(workdir, "EQ_Kmeans%03d"%indep_run)
ref_vol = apply_low_pass_filter(ref_vol,Tracker)
mref_ali3d_EQ_Kmeans(ref_vol, outdir, Tracker["this_particle_text_file"], Tracker)
partition_dict[indep_run]=Tracker["this_partition"]
Tracker["partition_dict"] = partition_dict
Tracker["total_stack"] = len(Tracker["this_data_list"])
Tracker["this_total_stack"] = Tracker["total_stack"]
###############################
do_two_way_comparison(Tracker)
###############################
ref_vol_list = []
from time import sleep
number_of_ref_class = []
for igrp in xrange(len(Tracker["two_way_stable_member"])):
Tracker["this_data_list"] = Tracker["two_way_stable_member"][igrp]
Tracker["this_data_list_file"] = os.path.join(workdir,"stable_class%d.txt"%igrp)
if myid == main_node:
write_text_file(Tracker["this_data_list"], Tracker["this_data_list_file"])
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["nxinit"], Tracker["this_data_list_file"], Tracker["constants"]["partstack"], myid, main_node, nproc, preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"], finfo = None)
ref_vol_list.append(volref)
number_of_ref_class.append(len(Tracker["this_data_list"]))
if myid == main_node:
log_main.add("group %d members %d "%(igrp,len(Tracker["this_data_list"])))
Tracker["number_of_ref_class"] = number_of_ref_class
nx_of_image = ref_vol_list[0].get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"] # no PW adjustment
if myid == main_node:
for iref in xrange(len(ref_vol_list)):
refdata = [None]*4
refdata[0] = ref_vol_list[iref]
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
volref.write_image(os.path.join(workdir,"volf_stable.hdf"),iref)
mpi_barrier(MPI_COMM_WORLD)
Tracker["this_data_list"] = Tracker["this_accounted_list"]
outdir = os.path.join(workdir,"Kmref")
empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI(ref_vol_list,outdir,Tracker["this_accounted_text"],Tracker)
Tracker["this_unaccounted_list"] = get_complementary_elements(list_to_be_processed,final_list)
if myid == main_node:
log_main.add("the number of particles not processed is %d"%len(Tracker["this_unaccounted_list"]))
write_text_file(Tracker["this_unaccounted_list"],Tracker["this_unaccounted_text"])
update_full_dict(Tracker["this_unaccounted_list"], Tracker)
#######################################
number_of_groups = len(res_groups)
vol_list = []
number_of_ref_class = []
for igrp in xrange(number_of_groups):
data,old_shifts = get_shrink_data_huang(Tracker, Tracker["constants"]["nnxo"], os.path.join(outdir,"Class%d.txt"%igrp), Tracker["constants"]["partstack"],myid,main_node,nproc,preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"], finfo=None)
vol_list.append(volref)
if( myid == main_node ): npergroup = len(read_text_file(os.path.join(outdir,"Class%d.txt"%igrp)))
else: npergroup = 0
npergroup = bcast_number_to_all(npergroup, main_node )
number_of_ref_class.append(npergroup)
Tracker["number_of_ref_class"] = number_of_ref_class
mpi_barrier(MPI_COMM_WORLD)
nx_of_image = vol_list[0].get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"]=Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"]=Tracker["constants"]["PWadjustment"]
if myid == main_node:
for ivol in xrange(len(vol_list)):
refdata =[None]*4
refdata[0] = vol_list[ivol]
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
volref.write_image(os.path.join(workdir,"volf_of_Classes.hdf"),ivol)
log_main.add("number of unaccounted particles %10d"%len(Tracker["this_unaccounted_list"]))
log_main.add("number of accounted particles %10d"%len(Tracker["this_accounted_list"]))
Tracker["this_data_list"] = Tracker["this_unaccounted_list"] # reset parameters for the next round calculation
Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
Tracker["this_total_stack"] = Tracker["total_stack"]
number_of_groups = get_number_of_groups(len(Tracker["this_unaccounted_list"]),number_of_images_per_group)
Tracker["number_of_groups"] = number_of_groups
while number_of_groups >= 2 :
generation +=1
partition_dict ={}
workdir =os.path.join(masterdir,"generation%03d"%generation)
Tracker["this_dir"] = workdir
if myid ==main_node:
log_main.add("*********************************************")
log_main.add("----- generation %5d "%generation)
log_main.add("number of images per group is set as %10d "%number_of_images_per_group)
log_main.add("the number of groups is %10d "%number_of_groups)
log_main.add(" number of particles for clustering is %10d"%Tracker["total_stack"])
cmd ="{} {}".format("mkdir",workdir)
os.system(cmd)
mpi_barrier(MPI_COMM_WORLD)
create_random_list(Tracker)
for indep_run in xrange(Tracker["constants"]["indep_runs"]):
Tracker["this_particle_list"] = Tracker["this_indep_list"][indep_run]
ref_vol = recons_mref(Tracker)
if myid == main_node:
log_main.add("independent run %10d"%indep_run)
outdir = os.path.join(workdir, "EQ_Kmeans%03d"%indep_run)
Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
#ref_vol=apply_low_pass_filter(ref_vol,Tracker)
mref_ali3d_EQ_Kmeans(ref_vol,outdir,Tracker["this_unaccounted_text"],Tracker)
partition_dict[indep_run] = Tracker["this_partition"]
Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
Tracker["partition_dict"] = partition_dict
Tracker["this_total_stack"] = Tracker["total_stack"]
total_list_of_this_run = Tracker["this_unaccounted_list"]
###############################
do_two_way_comparison(Tracker)
###############################
ref_vol_list = []
number_of_ref_class = []
for igrp in xrange(len(Tracker["two_way_stable_member"])):
Tracker["this_data_list"] = Tracker["two_way_stable_member"][igrp]
Tracker["this_data_list_file"] = os.path.join(workdir,"stable_class%d.txt"%igrp)
if myid == main_node: write_text_file(Tracker["this_data_list"], Tracker["this_data_list_file"])
mpi_barrier(MPI_COMM_WORLD)
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nxinit"],Tracker["this_data_list_file"],Tracker["constants"]["partstack"],myid,main_node,nproc,preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"],finfo= None)
#volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1)
if myid == main_node:volref.write_image(os.path.join(workdir,"vol_stable.hdf"),iref)
#volref = resample(volref,Tracker["shrinkage"])
ref_vol_list.append(volref)
number_of_ref_class.append(len(Tracker["this_data_list"]))
mpi_barrier(MPI_COMM_WORLD)
Tracker["number_of_ref_class"] = number_of_ref_class
Tracker["this_data_list"] = Tracker["this_accounted_list"]
outdir = os.path.join(workdir,"Kmref")
empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI(ref_vol_list,outdir,Tracker["this_accounted_text"],Tracker)
# calculate the 3-D structure of original image size for each group
number_of_groups = len(res_groups)
Tracker["this_unaccounted_list"] = get_complementary_elements(total_list_of_this_run,final_list)
if myid == main_node:
log_main.add("the number of particles not processed is %d"%len(Tracker["this_unaccounted_list"]))
write_text_file(Tracker["this_unaccounted_list"],Tracker["this_unaccounted_text"])
mpi_barrier(MPI_COMM_WORLD)
update_full_dict(Tracker["this_unaccounted_list"],Tracker)
vol_list = []
for igrp in xrange(number_of_groups):
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nnxo"], os.path.join(outdir,"Class%d.txt"%igrp), Tracker["constants"]["partstack"], myid, main_node, nproc,preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"],finfo= None)
vol_list.append(volref)
mpi_barrier(MPI_COMM_WORLD)
nx_of_image=ref_vol_list[0].get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
if myid == main_node:
for ivol in xrange(len(vol_list)):
refdata = [None]*4
refdata[0] = vol_list[ivol]
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
volref.write_image(os.path.join(workdir, "volf_of_Classes.hdf"),ivol)
log_main.add("number of unaccounted particles %10d"%len(Tracker["this_unaccounted_list"]))
log_main.add("number of accounted particles %10d"%len(Tracker["this_accounted_list"]))
del vol_list
mpi_barrier(MPI_COMM_WORLD)
number_of_groups = get_number_of_groups(len(Tracker["this_unaccounted_list"]),number_of_images_per_group)
Tracker["number_of_groups"] = number_of_groups
Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
if Tracker["constants"]["unaccounted"]:
data,old_shifts = get_shrink_data_huang(Tracker,Tracker["constants"]["nnxo"],Tracker["this_unaccounted_text"],Tracker["constants"]["partstack"],myid,main_node,nproc,preshift = True)
volref = recons3d_4nn_ctf_MPI(myid=myid, prjlist = data, symmetry=Tracker["constants"]["sym"],finfo= None)
nx_of_image = volref.get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"]=Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"]=Tracker["constants"]["PWadjustment"]
if( myid == main_node ):
refdata = [None]*4
refdata[0] = volref
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
#volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1)
volref.write_image(os.path.join(workdir,"volf_unaccounted.hdf"))
# Finish program
if myid ==main_node: log_main.add("sxsort3d finishes")
mpi_barrier(MPI_COMM_WORLD)
from mpi import mpi_finalize
mpi_finalize()
exit()
def cml_open_proj(stack, ir, ou, lf, hf, dpsi=1):
from projection import cml_sinogram
from utilities import model_circle, get_params_proj, model_blank, get_im
from fundamentals import fftip
from filter import filt_tanh
# number of projections
if type(stack) == type(""): nprj = EMUtil.get_image_count(stack)
else: nprj = len(stack)
Prj = [] # list of projections
Ori = [
-1
] * 4 * nprj # orientation intial (phi, theta, psi, index) for each projection
for i in xrange(nprj):
image = get_im(stack, i)
# read initial angles if given
try:
Ori[4 * i], Ori[4 * i +
1], Ori[4 * i +
2], s2x, s2y = get_params_proj(image)
except:
pass
if (i == 0):
nx = image.get_xsize()
if (ou < 1): ou = nx // 2 - 1
diameter = int(2 * ou)
mask2D = model_circle(ou, nx, nx)
if ir > 0: mask2D -= model_circle(ir, nx, nx)
# normalize under the mask
[mean_a, sigma, imin, imax] = Util.infomask(image, mask2D, True)
image -= mean_a
Util.mul_scalar(image, 1.0 / sigma)
Util.mul_img(image, mask2D)
# sinogram
sino = cml_sinogram(image, diameter, dpsi)
# prepare the cut positions in order to filter (lf: low freq; hf: high freq)
ihf = min(int(2 * hf * diameter), diameter + (diameter + 1) % 2)
ihf = ihf + (ihf + 1) % 2 # index ihf must be odd to take the img part
ilf = max(int(2 * lf * diameter), 0)
ilf = ilf + ilf % 2 # index ilf must be even to fall in the real part
bdf = ihf - ilf + 1
# process lines
nxe = sino.get_xsize()
nye = sino.get_ysize()
prj = model_blank(bdf, 2 * nye)
pp = model_blank(nxe, 2 * nye)
for li in xrange(nye):
# get the line li
line = Util.window(sino, nxe, 1, 1, 0, li - nye // 2, 0)
# u2 (not improve the results)
#line = filt_tanh(line, ou / float(nx), ou / float(nx))
# normalize this line
[mean_l, sigma_l, imin, imax] = Util.infomask(line, None, True)
line = (line - mean_l) / sigma_l
# fft
fftip(line)
# filter (cut part of coef) and create mirror line
Util.cml_prepare_line(prj, line, ilf, ihf, li, nye)
# store the projection
Prj.append(prj)
return Prj, Ori
def main():
import os
import sys
from optparse import OptionParser
from global_def import SPARXVERSION
import global_def
arglist = []
for arg in sys.argv:
arglist.append( arg )
progname = os.path.basename(arglist[0])
usage2 = progname + """ inputfile outputfile [options]
Functionalities:
1. Helicise input volume and save the result to output volume:
sxhelicon_utils.py input_vol.hdf output_vol.hdf --helicise --dp=27.6 --dphi=166.5 --fract=0.65 --rmax=70 --rmin=1 --apix=1.84 --sym=D1
2. Helicise pdb file and save the result to a new pdb file:
sxhelicon_utils.py input.pdb output.pdb --helicisepdb --dp=27.6 --dphi=166.5 --nrepeats --apix=1.84
3. Generate two lists of image indices used to split segment stack into halves for helical fsc calculation.
sxhelicon_utils.py bdb:big_stack --hfsc='flst' --filament_attr=filament
4. Map of filament distribution in the stack
sxhelicon_utils.py bdb:big_stack --filinfo=info.txt
The output file will contain four columns:
1 2 3 4
first image number last image number number of images in the filament name
5. Predict segments' orientation parameters based on distances between segments and known helical symmetry
sxhelicon_utils.py bdb:big_stack --predict_helical=helical_params.txt --dp=27.6 --dphi=166.5 --apix=1.84
6. Generate disks from filament based reconstructions:
sxheader.py stk.hdf --params=xform.projection --import=params.txt
# horatio active_refactoring Jy51i1EwmLD4tWZ9_00000_1
# sxheader.py stk.hdf --params=active --one
mpirun -np 2 sxhelicon_utils.py stk.hdf --gendisk='bdb:disk' --ref_nx=100 --ref_ny=100 --ref_nz=200 --apix=1.84 --dp=27.6 --dphi=166.715 --fract=0.67 --rmin=0 --rmax=64 --function="[.,nofunc,helical3c]" --sym="c1" --MPI
7. Stack disks based on helical symmetry parameters
sxhelicon_utils.py disk_to_stack.hdf --stackdisk=stacked_disks.hdf --dphi=166.5 --dp=27.6 --ref_nx=160 --ref_ny=160 --ref_nz=225 --apix=1.84
8. Helical symmetry search:
mpirun -np 3 sxhelicon_utils.py volf0010.hdf outsymsearch --symsearch --dp=27.6 --dphi=166.715 --apix=1.84 --fract=0.65 --rmin=0 --rmax=92.0 --datasym=datasym.txt --dp_step=0.92 --ndp=3 --dphi_step=1.0 --ndphi=10 --MPI
"""
parser = OptionParser(usage2,version=SPARXVERSION)
#parser.add_option("--ir", type="float", default= -1, help="inner radius for rotational correlation > 0 (set to 1) (Angstroms)")
parser.add_option("--ou", type="float", default= -1, help="outer radius for rotational 2D correlation < int(nx/2)-1 (set to the radius of the particle) (Angstroms)")
parser.add_option("--rs", type="int", default= 1, help="step between rings in rotational correlation >0 (set to 1)" )
parser.add_option("--xr", type="string", default= "4 2 1 1 1", help="range for translation search in x direction, search is +/-xr (Angstroms) ")
parser.add_option("--txs", type="string", default= "1 1 1 0.5 0.25", help="step size of the translation search in x directions, search is -xr, -xr+ts, 0, xr-ts, xr (Angstroms)")
parser.add_option("--delta", type="string", default= "10 6 4 3 2", help="angular step of reference projections")
parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches")
parser.add_option("--maxit", type="int", default= 30, help="maximum number of iterations performed for each angular step (set to 30) ")
parser.add_option("--CTF", action="store_true", default=False, help="CTF correction")
parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data")
parser.add_option("--MPI", action="store_true", default=False, help="use MPI version")
#parser.add_option("--fourvar", action="store_true", default=False, help="compute Fourier variance")
parser.add_option("--apix", type="float", default= -1.0, help="pixel size in Angstroms")
parser.add_option("--dp", type="float", default= -1.0, help="delta z - translation in Angstroms")
parser.add_option("--dphi", type="float", default= -1.0, help="delta phi - rotation in degrees")
parser.add_option("--rmin", type="float", default= 0.0, help="minimal radius for hsearch (Angstroms)")
parser.add_option("--rmax", type="float", default= 80.0, help="maximal radius for hsearch (Angstroms)")
parser.add_option("--fract", type="float", default= 0.7, help="fraction of the volume used for helical search")
parser.add_option("--sym", type="string", default= "c1", help="symmetry of the structure")
parser.add_option("--function", type="string", default="helical", help="name of the reference preparation function")
parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction")
parser.add_option("--debug", action="store_true", default=False, help="debug")
parser.add_option("--volalixshift", action="store_true", default=False, help="Use volalixshift refinement")
parser.add_option("--searchxshift", type="float", default= 0.0, help="search range for x-shift determination: +/- searchxshift (Angstroms)")
parser.add_option("--nearby", type="float", default= 6.0, help="neighborhood within which to search for peaks in 1D ccf for x-shift search (Angstroms)")
# filinfo
parser.add_option( "--filinfo", type="string", default="", help="Store in an output text file infomration about distribution of filaments in the stack." )
# diskali
parser.add_option("--diskali", action="store_true", default=False, help="volume alignment")
parser.add_option("--zstep", type="float", default= 1, help="Step size for translational search along z (Angstroms)")
# helicise
parser.add_option("--helicise", action="store_true", default=False, help="helicise input volume and save results to output volume")
parser.add_option("--hfsc", type="string", default="", help="Generate two lists of image indices used to split segment stack into halves for helical fsc calculation. The lists will be stored in two text files named using file_prefix with '_even' and '_odd' suffixes, respectively." )
parser.add_option("--filament_attr", type="string", default="filament", help="attribute under which filament identification is stored" )
parser.add_option("--predict_helical", type="string", default="", help="Generate projection parameters consistent with helical symmetry")
# helicise pdb
parser.add_option("--helicisepdb", action="store_true", default=False, help="Helicise pdb file and save the result to a new pdb file")
parser.add_option("--nrepeats", type="int", default= 50, help="Number of time the helical symmetry will be applied to the input file")
# input options for generating disks
parser.add_option("--gendisk", type="string", default="", help="Name of file under which generated disks will be saved to")
parser.add_option("--ref_nx", type="int", default= -1, help="nx=ny volume size" )
parser.add_option("--ref_nz", type="int", default= -1, help="nz volume size - computed disks will be nx x ny x rise/apix" )
parser.add_option("--new_pixel_size", type="float", default= -1, help="desired pixel size of the output disks. The default is -1, in which case there is no resampling (unless --match_pixel_rise flag is True).")
parser.add_option("--maxerror", type="float", default= 0.1, help="proportional to the maximum amount of error to tolerate between (dp/new_pixel_size) and int(dp/new_pixel_size ), where new_pixel_size is the pixel size calculated when the option --match_pixel_rise flag is True.")
parser.add_option("--match_pixel_rise", action="store_true", default=False, help="calculate new pixel size such that the rise is approximately integer number of pixels given the new pixel size. This will be the pixel size of the output disks.")
# get consistency
parser.add_option("--consistency", type="string", default="", help="Name of parameters to get consistency statistics for")
parser.add_option("--phithr", type="float", default= 2.0, help="phi threshold for consistency check")
parser.add_option("--ythr", type="float", default= 2.0, help="y threshold (in Angstroms) for consistency check")
parser.add_option("--segthr", type="int", default= 3, help="minimum number of segments/filament for consistency check")
# stack disks
parser.add_option("--stackdisk", type="string", default="", help="Name of file under which output volume will be saved to.")
parser.add_option("--ref_ny", type="int", default=-1, help="ny of output volume size. Default is ref_nx" )
# symmetry search
parser.add_option("--symsearch", action="store_true", default=False, help="Do helical symmetry search." )
parser.add_option("--ndp", type="int", default= 12, help="In symmetrization search, number of delta z steps equals to 2*ndp+1")
parser.add_option("--ndphi", type="int", default= 12, help="In symmetrization search, number of dphi steps equals to 2*ndphi+1")
parser.add_option("--dp_step", type="float", default= 0.1, help="delta z step for symmetrization [Angstroms] (default 0.1)")
parser.add_option("--dphi_step", type="float", default= 0.1, help="dphi step for symmetrization [degrees] (default 0.1)")
parser.add_option("--datasym", type="string", default="datasym.txt", help="symdoc")
parser.add_option("--symdoc", type="string", default="", help="text file containing helical symmetry parameters dp and dphi")
# filament statistics in the stack
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 1 or len(args) > 5:
print "Various helical reconstruction related functionalities: " + usage2
print "Please run '" + progname + " -h' for detailed options"
else:
if len(options.hfsc) > 0:
if len(args) != 1:
print "Incorrect number of parameters"
sys.exit()
from applications import imgstat_hfsc
imgstat_hfsc( args[0], options.hfsc, options.filament_attr)
sys.exit()
elif len(options.filinfo) > 0:
if len(args) != 1:
print "Incorrect number of parameters"
sys.exit()
from EMAN2 import EMUtil
filams = EMUtil.get_all_attributes(args[0], "filament")
ibeg = 0
filcur = filams[0]
n = len(filams)
inf = []
i = 1
while( i <= n):
if(i < n): fis = filams[i]
else: fis = ""
if( fis != filcur ):
iend = i-1
inf.append([ibeg,iend,iend-ibeg+1,filcur])
ibeg = i
filcur = fis
i += 1
from utilities import write_text_row
write_text_row(inf, options.filinfo)
sys.exit()
if len(options.stackdisk) > 0:
if len(args) != 1:
print "Incorrect number of parameters"
sys.exit()
dpp = (float(options.dp)/options.apix)
rise = int(dpp)
if(abs(float(rise) - dpp)>1.0e-3):
print " dpp has to be integer multiplicity of the pixel size"
sys.exit()
from utilities import get_im
v = get_im(args[0])
from applications import stack_disks
ref_ny = options.ref_ny
if ref_ny < 0:
ref_ny = options.ref_nx
sv = stack_disks(v, options.ref_nx, ref_ny, options.ref_nz, options.dphi, rise)
sv.write_image(options.stackdisk)
sys.exit()
if len(options.consistency) > 0:
if len(args) != 1:
print "Incorrect number of parameters"
sys.exit()
from development import consistency_params
consistency_params(args[0], options.consistency, options.dphi, options.dp, options.apix,phithr=options.phithr, ythr=options.ythr, THR=options.segthr)
sys.exit()
rminp = int((float(options.rmin)/options.apix) + 0.5)
rmaxp = int((float(options.rmax)/options.apix) + 0.5)
from utilities import get_input_from_string, get_im
xr = get_input_from_string(options.xr)
txs = get_input_from_string(options.txs)
irp = 1
if options.ou < 0: oup = -1
else: oup = int( (options.ou/options.apix) + 0.5)
xrp = ''
txsp = ''
for i in xrange(len(xr)):
xrp += " "+str(float(xr[i])/options.apix)
for i in xrange(len(txs)):
txsp += " "+str(float(txs[i])/options.apix)
searchxshiftp = int( (options.searchxshift/options.apix) + 0.5)
nearbyp = int( (options.nearby/options.apix) + 0.5)
zstepp = int( (options.zstep/options.apix) + 0.5)
if options.MPI:
from mpi import mpi_init, mpi_finalize
sys.argv = mpi_init(len(sys.argv), sys.argv)
if len(options.predict_helical) > 0:
if len(args) != 1:
print "Incorrect number of parameters"
sys.exit()
if options.dp < 0:
print "Helical symmetry paramter rise --dp should not be negative"
sys.exit()
from applications import predict_helical_params
predict_helical_params(args[0], options.dp, options.dphi, options.apix, options.predict_helical)
sys.exit()
if options.helicise:
if len(args) != 2:
print "Incorrect number of parameters"
sys.exit()
if options.dp < 0:
print "Helical symmetry paramter rise --dp should not be negative"
sys.exit()
from utilities import get_im, sym_vol
vol = get_im(args[0])
vol = sym_vol(vol, options.sym)
hvol = vol.helicise(options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp)
hvol = sym_vol(hvol, options.sym)
hvol.write_image(args[1])
sys.exit()
if options.helicisepdb:
if len(args) != 2:
print "Incorrect number of parameters"
sys.exit()
if options.dp < 0:
print "Helical symmetry paramter rise --dp should not be negative"
sys.exit()
from math import cos, sin, radians
from copy import deepcopy
import numpy
from numpy import zeros,dot,float32
dp = options.dp
dphi = options.dphi
nperiod = options.nrepeats
infile =open(args[0],"r")
pall = infile.readlines()
infile.close()
p = []
pos = []
lkl = -1
for i in xrange( len(pall) ):
if( (pall[i])[:4] == 'ATOM'):
if( lkl == -1 ): lkl = i
p.append( pall[i] )
pos.append(i)
n = len(p)
X = zeros( (3,len(p) ), dtype=float32 )
X_new = zeros( (3,len(p) ), dtype=float32 )
for i in xrange( len(p) ):
element = deepcopy( p[i] )
X[0,i]=float(element[30:38])
X[1,i]=float(element[38:46])
X[2,i]=float(element[46:54])
pnew = []
for j in xrange(-nperiod, nperiod+1):
for i in xrange( n ):
pnew.append( deepcopy(p[i]) )
dphi = radians(dphi)
m = zeros( (3,3 ), dtype=float32 )
t = zeros( (3,1 ), dtype=float32 )
m[2][2] = 1.0
t[0,0] = 0.0
t[1,0] = 0.0
for j in xrange(-nperiod, nperiod+1):
if j != 0:
rd = j*dphi
m[0][0] = cos(rd)
m[0][1] = sin(rd)
m[1][0] = -m[0][1]
m[1][1] = m[0][0]
t[2,0] = j*dp
X_new = dot(m, X) + t
for i in xrange( n ):
pnew[j*n+i] = pnew[j*n+i][:30] + "%8.3f"%( float(X_new[0,i]) )+"%8.3f"%( float(X_new[1,i]) )+"%8.3f"%( float(X_new[2,i]) ) + pnew[j*n+i][54:]
outfile=open(args[1],"w")
outfile.writelines(pall[0:lkl])
outfile.writelines(pnew)
outfile.writelines("END\n")
outfile.close()
sys.exit()
if options.volalixshift:
if options.maxit > 1:
print "Inner iteration for x-shift determinatin is restricted to 1"
sys.exit()
if len(args) < 4: mask = None
else: mask = args[3]
from applications import volalixshift_MPI
global_def.BATCH = True
volalixshift_MPI(args[0], args[1], args[2], searchxshiftp, options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp, mask, options.maxit, options.CTF, options.snr, options.sym, options.function, options.npad, options.debug, nearbyp)
global_def.BATCH = False
if options.diskali:
#if options.maxit > 1:
# print "Inner iteration for disk alignment is restricted to 1"
# sys.exit()
if len(args) < 4: mask = None
else: mask = args[3]
global_def.BATCH = True
if(options.sym[:1] == "d" or options.sym[:1] == "D" ):
from development import diskaliD_MPI
diskaliD_MPI(args[0], args[1], args[2], mask, options.dp, options.dphi, options.apix, options.function, zstepp, options.fract, rmaxp, rminp, options.CTF, options.maxit, options.sym)
else:
from applications import diskali_MPI
diskali_MPI(args[0], args[1], args[2], mask, options.dp, options.dphi, options.apix, options.function, zstepp, options.fract, rmaxp, rminp, options.CTF, options.maxit, options.sym)
global_def.BATCH = False
if options.symsearch:
if len(options.symdoc) < 1:
if options.dp < 0 or options.dphi < 0:
print "Enter helical symmetry parameters either using --symdoc or --dp and --dphi"
sys.exit()
if options.dp < 0 or options.dphi < 0:
# read helical symmetry parameters from symdoc
from utilities import read_text_row
hparams=read_text_row(options.symdoc)
dp = hparams[0][0]
dphi = hparams[0][1]
else:
dp = options.dp
dphi = options.dphi
from applications import symsearch_MPI
if len(args) < 3:
mask = None
else:
mask= args[2]
global_def.BATCH = True
symsearch_MPI(args[0], args[1], mask, dp, options.ndp, options.dp_step, dphi, options.ndphi, options.dphi_step, rminp, rmaxp, options.fract, options.sym, options.function, options.datasym, options.apix, options.debug)
global_def.BATCH = False
elif len(options.gendisk)> 0:
from applications import gendisks_MPI
global_def.BATCH = True
if len(args) == 1: mask3d = None
else: mask3d = args[1]
if options.dp < 0:
print "Helical symmetry paramter rise --dp must be explictly set!"
sys.exit()
gendisks_MPI(args[0], mask3d, options.ref_nx, options.apix, options.dp, options.dphi, options.fract, rmaxp, rminp, options.CTF, options.function, options.sym, options.gendisk, options.maxerror, options.new_pixel_size, options.match_pixel_rise)
global_def.BATCH = False
if options.MPI:
from mpi import mpi_finalize
mpi_finalize()
