def generate_helimic(refvol, outdir, pixel, CTF=False, Cs=2.0,voltage = 200.0, ampcont = 10.0, nonoise = False, rand_seed=14567):
from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im
from random import random
from projection import prgs, prep_vol
from filter import filt_gaussl, filt_ctf
from EMAN2 import EMAN2Ctf
if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxhelical_demo", 1)
os.mkdir(outdir)
seed(rand_seed)
Util.set_randnum_seed(rand_seed)
angles =[]
for i in xrange(3):
angles.append( [0.0+60.0*i, 90.0-i*5, 0.0, 0.0, 0.0] )
nangle = len(angles)
volfts = get_im(refvol)
nx = volfts.get_xsize()
ny = volfts.get_ysize()
nz = volfts.get_zsize()
volfts, kbx, kby, kbz = prep_vol( volfts )
iprj = 0
width = 500
xstart = 0
ystart = 0
for idef in xrange(3,6):
mic = model_blank(2048, 2048)
#defocus = idef*0.2
defocus = idef*0.6 ##@ming
if CTF :
#ctf = EMAN2Ctf()
#ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
from utilities import generate_ctf
ctf = generate_ctf([defocus,2,200,1.84,0.0,ampcont,defocus*0.2,80]) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
i = idef - 4
for k in xrange(1):
psi = 90 + 10*i
proj = prgs(volfts, kbz, [angles[idef-3][0], angles[idef-3][1], psi, 0.0, 0.0], kbx, kby)
proj = Util.window(proj, 320, nz)
mic += pad(proj, 2048, 2048, 1, 0.0, 750*i, 20*i, 0)
if not nonoise: mic += model_gauss_noise(30.0,2048,2048)
if CTF :
#apply CTF
mic = filt_ctf(mic, ctf)
if not nonoise: mic += filt_gaussl(model_gauss_noise(17.5,2048,2048), 0.3)
mic.write_image("%s/mic%1d.hdf"%(outdir, idef-3),0)
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ Input Output [options]
Generate three micrographs, each micrograph contains one projection of a long filament.
Input: Reference Volume, output directory
Output: Three micrographs stored in output directory
sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84
Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
Generate run through example script for helicon
sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
"""
parser = OptionParser(usage, version=SPARXVERSION)
# helicise the Atom coordinates
# generate micrographs of helical filament
parser.add_option(
"--generate_micrograph",
action="store_true",
default=False,
help=
"Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="Use CTF correction")
parser.add_option("--apix",
type="float",
default=-1,
help="pixel size in Angstroms")
parser.add_option(
"--rand_seed",
type="int",
default=14567,
help=
"the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs."
)
parser.add_option("--Cs",
type="float",
default=2.0,
help="Microscope Cs (spherical aberation)")
parser.add_option("--voltage",
type="float",
default=200.0,
help="Microscope voltage in KV")
parser.add_option("--ac",
type="float",
default=10.0,
help="Amplitude contrast (percentage, default=10)")
parser.add_option("--nonoise",
action="store_true",
default=False,
help="Do not add noise to the micrograph.")
# generate initial volume
parser.add_option("--generate_noisycyl",
action="store_true",
default=False,
help="Generate initial volume of noisy cylinder.")
parser.add_option(
"--boxsize",
type="string",
default="100,100,200",
help=
"String containing x , y, z dimensions (separated by comma) in pixels")
parser.add_option("--rad",
type="int",
default=35,
help="Radius of initial volume in pixels")
# generate 2D mask
parser.add_option("--generate_mask",
action="store_true",
default=False,
help="Generate 2D rectangular mask.")
parser.add_option(
"--masksize",
type="string",
default="200,200",
help=
"String containing x and y dimensions (separated by comma) in pixels")
parser.add_option("--maskwidth",
type="int",
default=60,
help="Width of rectangular mask")
# Apply 2D alignment parameters to input stack and output new images to output stack
parser.add_option(
"--applyparams",
action="store_true",
default=False,
help=
"Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack"
)
# Generate run script
parser.add_option("--generate_script",
action="store_true",
default=False,
help="Generate script for helicon run through example")
parser.add_option("--filename",
type="string",
default="runhelicon",
help="Name of run script to generate")
parser.add_option("--seg_ny",
type="int",
default=180,
help="y-dimension of segment used for refinement")
parser.add_option(
"--ptcl_dist",
type="int",
default=15,
help=
"Distance in pixels between adjacent segments windowed from same filament"
)
parser.add_option(
"--fract",
type="float",
default=0.35,
help="Fraction of the volume used for applying helical symmetry.")
(options, args) = parser.parse_args()
if len(args) > 3:
print("usage: " + usage)
print("Please run '" + progname + " -h' for detailed options")
else:
if options.generate_script:
generate_runscript(options.filename, options.seg_ny,
options.ptcl_dist, options.fract)
if options.generate_micrograph:
if options.apix <= 0:
print("Please enter pixel size.")
sys.exit()
generate_helimic(args[0], args[1], options.apix, options.CTF,
options.Cs, options.voltage, options.ac,
options.nonoise, options.rand_seed)
if options.generate_noisycyl:
from utilities import model_cylinder, model_gauss_noise
outvol = args[0]
boxdims = options.boxsize.split(',')
if len(boxdims) < 1 or len(boxdims) > 3:
print(
"Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'"
)
sys.exit()
nx = int(boxdims[0])
if len(boxdims) == 1:
ny = nx
nz = nx
else:
ny = int(boxdims[1])
if len(boxdims) == 3:
nz = int(boxdims[2])
(model_cylinder(options.rad, nx, ny, nz) *
model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol)
if options.generate_mask:
from utilities import model_blank, pad
outvol = args[0]
maskdims = options.masksize.split(',')
if len(maskdims) < 1 or len(maskdims) > 2:
print(
"Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'"
)
sys.exit()
nx = int(maskdims[0])
if len(maskdims) == 1:
ny = nx
else:
ny = int(maskdims[1])
mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1,
0.0)
mask.write_image(outvol)
if options.applyparams:
from utilities import get_im, get_params2D, set_params2D
from fundamentals import cyclic_shift
stack = args[0]
newstack = args[1]
mask = get_im(args[2])
nima = EMUtil.get_image_count(stack)
for im in range(nima):
prj = get_im(stack, im)
alpha, sx, sy, mirror, scale = get_params2D(prj)
prj = cyclic_shift(prj, int(sx))
set_params2D(prj, [0.0, 0., 0.0, 0, 1])
stat = Util.infomask(prj, mask, False)
prj = old_div((prj - stat[0]), stat[1])
ctf_params = prj.get_attr("ctf")
prj.set_attr('ctf_applied', 0)
prj.write_image(newstack, im)
def generate_helimic(refvol,
outdir,
pixel,
CTF=False,
Cs=2.0,
voltage=200.0,
ampcont=10.0,
nonoise=False,
rand_seed=14567):
from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im
from random import random
from projection import prgs, prep_vol
from filter import filt_gaussl, filt_ctf
from EMAN2 import EMAN2Ctf
if os.path.exists(outdir):
ERROR(
'Output directory exists, please change the name and restart the program',
"sxhelical_demo", 1)
os.mkdir(outdir)
seed(rand_seed)
Util.set_randnum_seed(rand_seed)
angles = []
for i in range(3):
angles.append([0.0 + 60.0 * i, 90.0 - i * 5, 0.0, 0.0, 0.0])
nangle = len(angles)
volfts = get_im(refvol)
nx = volfts.get_xsize()
ny = volfts.get_ysize()
nz = volfts.get_zsize()
volfts, kbx, kby, kbz = prep_vol(volfts)
iprj = 0
width = 500
xstart = 0
ystart = 0
for idef in range(3, 6):
mic = model_blank(2048, 2048)
#defocus = idef*0.2
defocus = idef * 0.6 ##@ming
if CTF:
#ctf = EMAN2Ctf()
#ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
from utilities import generate_ctf
ctf = generate_ctf(
[defocus, 2, 200, 1.84, 0.0, ampcont, defocus * 0.2, 80]
) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
i = idef - 4
for k in range(1):
psi = 90 + 10 * i
proj = prgs(
volfts, kbz,
[angles[idef - 3][0], angles[idef - 3][1], psi, 0.0, 0.0], kbx,
kby)
proj = Util.window(proj, 320, nz)
mic += pad(proj, 2048, 2048, 1, 0.0, 750 * i, 20 * i, 0)
if not nonoise: mic += model_gauss_noise(30.0, 2048, 2048)
if CTF:
#apply CTF
mic = filt_ctf(mic, ctf)
if not nonoise:
mic += filt_gaussl(model_gauss_noise(17.5, 2048, 2048), 0.3)
mic.write_image("%s/mic%1d.hdf" % (outdir, idef - 3), 0)
def main():
from utilities import write_text_row, drop_image, model_gauss_noise, get_im, set_params_proj, wrap_mpi_bcast, model_circle
import user_functions
from applications import MPI_start_end
from optparse import OptionParser
from global_def import SPARXVERSION
from EMAN2 import EMData
from multi_shc import multi_shc, do_volume
from logger import Logger, BaseLogger_Files
import sys
import os
import time
import socket
progname = os.path.basename(sys.argv[0])
usage = progname + " stack [output_directory] initial_volume --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --center=center_type --fl --aa --ref_a=S --sym=c1"
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--ir", type= "int", default= 1, help="inner radius for rotational correlation > 0 (set to 1)")
parser.add_option("--ou", type= "int", default= -1, help="outer radius for rotational correlation < int(nx/2)-1 (set to the radius of the particle)")
parser.add_option("--rs", type= "int", default= 1, help="step between rings in rotational correlation >0 (set to 1)" )
parser.add_option("--xr", type="string", default= "-1", help="range for translation search in x direction, search is +/xr (default 0)")
parser.add_option("--yr", type="string", default= "-1", help="range for translation search in y direction, search is +/yr (default = same as xr)")
parser.add_option("--ts", type="string", default= "1", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional")
parser.add_option("--delta", type="string", default= "-1", help="angular step of reference projections during initialization step (default automatically selected based on radius of the structure.)")
#parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches (phi and theta)")
parser.add_option("--center", type="float", default= -1, help="-1: average shift method; 0: no centering; 1: center of gravity (default=-1)")
parser.add_option("--maxit", type="int", default= 400, help="maximum number of iterations performed for the GA part (set to 400) ")
parser.add_option("--outlier_percentile",type="float", default= 95, help="percentile above which outliers are removed every iteration")
parser.add_option("--iteration_start",type="int", default= 0, help="starting iteration for rviper, 0 means go to the most recent one (default).")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF (Default no CTF correction)")
parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data (default 1.0)")
parser.add_option("--ref_a", type="string", default= "S", help="method for generating the quasi-uniformly distributed projection directions (default S)")
parser.add_option("--sym", type="string", default= "c1", help="symmetry of the refined structure")
parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default=2)")
parser.add_option("--startangles", action="store_true", default=False, help="Use orientation parameters in the input file header to jumpstart the procedure")
#options introduced for the do_volume function
parser.add_option("--fl", type="float", default=0.12, help="cut-off frequency of hyperbolic tangent low-pass Fourier filte (default 0.12)")
parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass Fourier filter (default 0.1)")
parser.add_option("--pwreference", type="string", default="", help="text file with a reference power spectrum (default no power spectrum adjustment)")
parser.add_option("--mask3D", type="string", default=None, help="3D mask file (default a sphere WHAT RADIUS??)")
(options, args) = parser.parse_args(sys.argv[1:])
#print( " args ",args)
if( len(args) == 3):
volinit = args[2]
masterdir = args[1]
elif(len(args) == 2):
volinit = args[1]
masterdir = ""
else:
print( "usage: " + usage)
print( "Please run '" + progname + " -h' for detailed options")
return 1
orgstack = args[0]
#print( orgstack,masterdir,volinit )
# INPUT PARAMETERS
radi = options.ou
global_def.BATCH = True
ali3d_options.ir = options.ir
ali3d_options.rs = options.rs
ali3d_options.ou = options.ou
ali3d_options.xr = options.xr
ali3d_options.yr = options.yr
ali3d_options.ts = options.ts
ali3d_options.an = "-1"
ali3d_options.sym = options.sym
ali3d_options.delta = options.delta
ali3d_options.npad = options.npad
ali3d_options.center = options.center
ali3d_options.CTF = options.CTF
ali3d_options.ref_a = options.ref_a
ali3d_options.snr = options.snr
ali3d_options.mask3D = options.mask3D
ali3d_options.pwreference = options.pwreference
ali3d_options.fl = 0.4
ali3d_options.aa = 0.1
if( ali3d_options.xr == "-1" ): ali3d_options.xr = "2"
"""
print( options)
print( 'ali3d_options', ali3d_options.ir ,\
ali3d_options.rs ,\
ali3d_options.ou ,\
ali3d_options.xr ,\
ali3d_options.yr ,\
ali3d_options.ts ,\
ali3d_options.an ,\
ali3d_options.sym ,\
ali3d_options.delta ,\
ali3d_options.npad ,\
ali3d_options.center ,\
ali3d_options.CTF ,\
ali3d_options.ref_a ,\
ali3d_options.snr ,\
ali3d_options.mask3D ,\
ali3d_options.fl ,\
ali3d_options.aa \
)
#exit()
"""
mpi_init(0, [])
nproc = mpi_comm_size(MPI_COMM_WORLD)
myid = mpi_comm_rank(MPI_COMM_WORLD)
main_node = 0
#mpi_finalize()
#exit()
nxinit = -1 #int(280*0.3*2)
nsoft = 0
mempernode = 4.0e9
# PARAMETERS OF THE PROCEDURE
# threshold error
thresherr = 0
fq = 0.11 # low-freq limit to which fuse ref volumes. Should it be estimated somehow?
# Get the pixel size, if none set to 1.0, and the original image size
if(myid == main_node):
a = get_im(orgstack)
nnxo = a.get_xsize()
if ali3d_options.CTF:
i = a.get_attr('ctf')
pixel_size = i.apix
else:
pixel_size = 1.0
del a
else:
nnxo = 0
pixel_size = 1.0
pixel_size = bcast_number_to_all(pixel_size, source_node = main_node)
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
if(radi < 1): radi = nnxo//2-2
elif((2*radi+2)>nnxo): ERROR("HERE","particle radius set too large!",1)
ali3d_options.ou = radi
if(nxinit < 0): nxinit = min(32, nnxo)
nxshrink = nxinit
minshrink = 32.0/float(nnxo)
shrink = max(float(nxshrink)/float(nnxo),minshrink)
# MASTER DIRECTORY
if(myid == main_node):
print( " masterdir ",masterdir)
if( masterdir == ""):
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "master"+timestring
li = len(masterdir)
cmd = "{} {}".format("mkdir", masterdir)
cmdexecute(cmd)
keepchecking = 0
else:
li = 0
keepchecking = 1
else:
li = 0
keepchecking = 1
li = mpi_bcast(li,1,MPI_INT,main_node,MPI_COMM_WORLD)[0]
if( li > 0 ):
masterdir = mpi_bcast(masterdir,li,MPI_CHAR,main_node,MPI_COMM_WORLD)
masterdir = string.join(masterdir,"")
# create a vstack from input stack to the local stack in masterdir
# Stack name set to default
stack = "bdb:"+masterdir+"/rdata"
# Initialization of stacks
if(myid == main_node):
if keepchecking:
if(os.path.exists(os.path.join(masterdir,"EMAN2DB/rdata.bdb"))): doit = False
else: doit = True
else: doit = True
if doit:
if(orgstack[:4] == "bdb:"): cmd = "{} {} {}".format("e2bdb.py", orgstack,"--makevstack="+stack)
else: cmd = "{} {} {}".format("sxcpy.py", orgstack, stack)
cmdexecute(cmd)
cmd = "{} {}".format("sxheader.py --consecutive --params=originalid", stack)
cmdexecute(cmd)
keepchecking = False
total_stack = EMUtil.get_image_count(stack)
junk = get_im(stack)
nnxo = junk.get_xsize()
del junk
else:
total_stack = 0
nnxo = 0
total_stack = bcast_number_to_all(total_stack, source_node = main_node)
nnxo = bcast_number_to_all(nnxo, source_node = main_node)
# INITIALIZATION
# Run exhaustive projection matching to get initial orientation parameters
# Estimate initial resolution
initdir = os.path.join(masterdir,"main000")
# make sure the initial volume is not set to zero outside of a mask, as if it is it will crach the program
if( myid == main_node and (not options.startangles)):
viv = get_im(volinit)
if(options.mask3D == None): mask33d = model_circle(radi,nnxo,nnxo,nnxo)
else: mask33d = (options.mask3D).copy()
st = Util.infomask(viv, mask33d, False)
if( st[0] == 0.0 ):
viv += (model_blank(nnxo,nnxo,nnxo,1.0) - mask33d)*model_gauss_noise(st[1]/1000.0,nnxo,nnxo,nnxo)
viv.write_image(volinit)
del mask33d, viv
doit, keepchecking = checkstep(initdir, keepchecking, myid, main_node)
if doit:
partids = os.path.join(masterdir, "ids.txt")
partstack = os.path.join(masterdir, "paramszero.txt")
xr = min(8,(nnxo - (2*radi+1))//2)
if(xr > 3): ts = "2"
else: ts = "1"
delta = int(options.delta)
if(delta <= 0.0):
delta = "%f"%round(degrees(atan(1.0/float(radi))), 2)
paramsdict = { "stack":stack,"delta":"2.0", "ts":ts, "xr":"%f"%xr, "an":"-1", "center":options.center, "maxit":1, \
"currentres":0.4, "aa":0.1, "radius":radi, "nsoft":0, "delpreviousmax":True, "shrink":1.0, "saturatecrit":1.0, \
"refvol":volinit, "mask3D":options.mask3D}
if(options.startangles):
if( myid == main_node ):
cmd = "mkdir "+initdir
cmdexecute(cmd)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"INITIALIZATION")
cmd = "{} {}".format("sxheader.py --params=xform.projection --export="+os.path.join(initdir,"params-chunk0.txt"), stack)
cmdexecute(cmd)
print(line,"Executed successfully: ","Imported initial parameters from the input stack")
else:
if( myid == main_node ):
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"INITIALIZATION")
write_text_file(range(total_stack), partids)
write_text_row([[0.0,0.0,0.0,0.0,0.0] for i in xrange(total_stack) ], partstack)
metamove(paramsdict, partids, partstack, initdir, 0, myid, main_node, nproc)
if(myid == main_node):
print(line,"Executed successfully: ","initialization ali3d_base_MPI %d"%nsoft)
# store params
partids = [None]*2
for procid in xrange(2): partids[procid] = os.path.join(initdir,"chunk%01d.txt"%procid)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(initdir,"params-chunk%01d.txt"%procid)
from random import shuffle
if(myid == main_node):
# split randomly
params = read_text_row(os.path.join(initdir,"params-chunk0.txt"))
assert(len(params) == total_stack)
ll = range(total_stack)
shuffle(ll)
l1 = ll[:total_stack//2]
l2 = ll[total_stack//2:]
del ll
l1.sort()
l2.sort()
write_text_file(l1,partids[0])
write_text_file(l2,partids[1])
write_text_row([params[i] for i in l1], partstack[0])
write_text_row([params[i] for i in l2], partstack[1])
del params, l1, l2
mpi_barrier(MPI_COMM_WORLD)
# Now parallel
vol = [None]*2
for procid in xrange(2):
projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc)
if ali3d_options.CTF: vol[procid] = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol[procid] = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if( myid == main_node):
vol[procid].write_image(os.path.join(initdir,"vol%01d.hdf"%procid) )
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Generated inivol #%01d "%procid)
if(myid == main_node):
currentres = get_resolution(vol, radi, nnxo, initdir)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Initial resolution %6.4f"%currentres)
write_text_file([currentres],os.path.join(initdir,"current_resolution.txt"))
else: currentres = 0.0
currentres = bcast_number_to_all(currentres, source_node = main_node)
else:
if(myid == main_node): currentres = read_text_file(os.path.join(initdir,"current_resolution.txt"))[0]
else: currentres = 0.0
currentres = bcast_number_to_all(currentres, source_node = main_node)
# set for the first iteration
nxshrink = min(max(32, int((currentres+paramsdict["aa"]/2.)*2*nnxo + 0.5)), nnxo)
shrink = float(nxshrink)/nnxo
tracker = {"previous-resolution":currentres, "movedup":False,"eliminated-outliers":False,\
"previous-nx":nxshrink, "previous-shrink":shrink, "extension":0, "bestsolution":0}
previousoutputdir = initdir
# MAIN ITERATION
mainiteration = 0
keepgoing = 1
while(keepgoing):
mainiteration += 1
# prepare output directory
mainoutputdir = os.path.join(masterdir,"main%03d"%mainiteration)
if(myid == main_node):
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"MAIN ITERATION #",mainiteration, shrink, nxshrink)
if keepchecking:
if(os.path.exists(mainoutputdir)):
doit = 0
print("Directory ",mainoutputdir," exists!")
else:
doit = 1
keepchecking = False
else:
doit = 1
if doit:
cmd = "{} {}".format("mkdir", mainoutputdir)
cmdexecute(cmd)
# prepare names of input file names, they are in main directory,
# log subdirectories contain outputs from specific refinements
partids = [None]*2
for procid in xrange(2): partids[procid] = os.path.join(previousoutputdir,"chunk%01d.txt"%procid)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(previousoutputdir,"params-chunk%01d.txt"%procid)
mpi_barrier(MPI_COMM_WORLD)
#mpi_finalize()
#exit()
#print("RACING A ",myid)
outvol = [os.path.join(previousoutputdir,"vol%01d.hdf"%procid) for procid in xrange(2)]
for procid in xrange(2):
doit, keepchecking = checkstep(outvol[procid], keepchecking, myid, main_node)
if doit:
from multi_shc import do_volume
projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc)
if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if( myid == main_node):
vol.write_image(outvol[procid])
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Generated inivol #%01d "%procid)
del vol
if(myid == main_node):
if keepchecking:
procid = 1
if(os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid)):
doit = 0
else:
doit = 1
keepchecking = False
else:
doit = 1
if doit:
vol = [get_im(outvol[procid]) for procid in xrange(2) ]
fq = 0.11 # which part to fuse
fuselowf(vol, fq)
for procid in xrange(2): vol[procid].write_image(os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid) )
del vol
else: doit = 0
mpi_barrier(MPI_COMM_WORLD)
doit = bcast_number_to_all(doit, source_node = main_node)
# Refine two groups at a current resolution
lastring = int(shrink*radi + 0.5)
if(lastring < 2):
print( line,"ERROR!! lastring too small ", radi, shrink, lastring)
break
# REFINEMENT
# Part "a" SHC
for procid in xrange(2):
coutdir = os.path.join(mainoutputdir,"loga%01d"%procid)
doit, keepchecking = checkstep(coutdir , keepchecking, myid, main_node)
paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":1500, \
"currentres":currentres, "aa":0.1, "radius":radi, "nsoft":1, "saturatecrit":0.75, "delpreviousmax":True, "shrink":shrink, \
"refvol":os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid),"mask3D":options.mask3D }
if doit:
metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir, "loga%01d"%procid, "params-chunk%01d.txt"%procid)
for procid in xrange(2):
outvol = os.path.join(mainoutputdir,"loga%01d"%procid,"shcvol%01d.hdf"%procid)
doit, keepchecking = checkstep(outvol, keepchecking, myid, main_node)
if doit:
from multi_shc import do_volume
projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc)
if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2)
del projdata
if( myid == main_node):
vol.write_image(outvol)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print( line,"Generated shcvol #%01d "%procid)
del vol
if(myid == main_node):
if keepchecking:
procid = 1
if(os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid) ):
doit = 0
else:
doit = 1
keepchecking = False
else:
doit = 1
if doit:
vol = []
for procid in xrange(2): vol.append(get_im(os.path.join(mainoutputdir,"loga%01d"%procid,"shcvol%01d.hdf"%procid) ))
fq = 0.11 # which part to fuse
fuselowf(vol, fq)
for procid in xrange(2): vol[procid].write_image( os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid) )
del vol
else: doit = 0
mpi_barrier(MPI_COMM_WORLD)
doit = bcast_number_to_all(doit, source_node = main_node)
# Part "b" deterministic
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir,"loga%01d"%procid,"params-chunk%01d.txt"%procid)
for procid in xrange(2):
coutdir = os.path.join(mainoutputdir,"logb%01d"%procid)
doit, keepchecking = checkstep(coutdir, keepchecking, myid, main_node)
# Run exhaustive to finish up matching
paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":10, \
"currentres":currentres, "aa":0.1, "radius":radi, "nsoft":0, "saturatecrit":0.95, "delpreviousmax":True, "shrink":shrink, \
"refvol":os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid), "mask3D":options.mask3D }
if doit:
metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc)
partstack = [None]*2
for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir,"logb%01d"%procid,"params-chunk%01d.txt"%procid)
# Compute current resolution, store result in main directory
doit, keepchecking = checkstep(os.path.join(mainoutputdir,"current_resolution.txt"), keepchecking, myid, main_node)
newres = 0.0
if doit:
newres = compute_resolution(stack, mainoutputdir, partids, partstack, radi, nnxo, ali3d_options.CTF, myid, main_node, nproc)
else:
if(myid == main_node): newres = read_text_file( os.path.join(mainoutputdir,"current_resolution.txt") )[0]
newres = bcast_number_to_all(newres, source_node = main_node)
# Here I have code to generate presentable results. IDs and params have to be merged and stored and an overall volume computed.
doit, keepchecking = checkstep(os.path.join(mainoutputdir,"volf.hdf"), keepchecking, myid, main_node)
if doit:
if( myid == main_node ):
pinids = map(int, read_text_file(partids[0]) ) + map(int, read_text_file(partids[1]) )
params = read_text_row(partstack[0]) + read_text_row(partstack[1])
assert(len(pinids) == len(params))
for i in xrange(len(pinids)):
pinids[i] = [ pinids[i], params[i] ]
del params
pinids.sort()
write_text_file([pinids[i][0] for i in xrange(len(pinids))], os.path.join(mainoutputdir,"indexes.txt"))
write_text_row( [pinids[i][1] for i in xrange(len(pinids))], os.path.join(mainoutputdir,"params.txt"))
mpi_barrier(MPI_COMM_WORLD)
ali3d_options.fl = newres
ali3d_options.ou = radi
projdata = getindexdata(stack, os.path.join(mainoutputdir,"indexes.txt"), os.path.join(mainoutputdir,"params.txt"), myid, nproc)
volf = do_volume(projdata, ali3d_options, mainiteration, mpi_comm = MPI_COMM_WORLD)
if(myid == main_node): volf.write_image(os.path.join(mainoutputdir,"volf.hdf"))
mpi_barrier(MPI_COMM_WORLD)
#print("RACING X ",myid)
if(newres == currentres):
for procid in xrange(2):
coutdir = os.path.join(mainoutputdir,"logc%01d"%procid)
doit, keepchecking = checkstep(coutdir, keepchecking, myid, main_node)
if doit:
# Do cross-check of the results
paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":1, \
"currentres":newres, "aa":0.1, "radius":radi, "nsoft":0, "saturatecrit":0.95, "delpreviousmax":True, "shrink":shrink, \
"refvol":os.path.join(mainoutputdir,"vol%01d.hdf"%(1-procid)), "mask3D":options.mask3D }
# The cross-check uses parameters from step "b" to make sure shifts are correct.
# As the check is exhaustive, angles are ignored
metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc)
# identify bad apples
doit, keepchecking = checkstep(os.path.join(mainoutputdir,"badapples.txt"), keepchecking, myid, main_node)
if doit:
if(myid == main_node):
from utilities import get_symt
from pixel_error import max_3D_pixel_error
ts = get_symt(ali3d_options.sym)
badapples = []
deltaerror = 2.0
total_images_now = 0
for procid in xrange(2):
bad = []
ids = map(int,read_text_file( partids[procid] ))
total_images_now += len(ids)
oldp = read_text_row(partstack[procid])
newp = read_text_row(os.path.join(mainoutputdir,"logc%01d"%procid,"params-chunk%01d.txt"%procid))
for i in xrange(len(ids)):
t1 = Transform({"type":"spider","phi":oldp[i][0],"theta":oldp[i][1],"psi":oldp[i][2]})
t1.set_trans(Vec2f(-oldp[i][3]*shrink, -oldp[i][4]*shrink))
t2 = Transform({"type":"spider","phi":newp[i][0],"theta":newp[i][1],"psi":newp[i][2]})
t2.set_trans(Vec2f(-newp[i][3]*shrink, -newp[i][4]*shrink))
if(len(ts) > 1):
# only do it if it is not c1
pixel_error = +1.0e23
for kts in ts:
ut = t2*kts
# we do not care which position minimizes the error
pixel_error = min(max_3D_pixel_error(t1, ut, lastring), pixel_error)
else:
pixel_error = max_3D_pixel_error(t1, t2, lastring)
if(pixel_error > deltaerror):
bad.append(i)
if(len(bad)>0):
badapples += [ids[bad[i]] for i in xrange(len(bad))]
for i in xrange(len(bad)-1,-1,-1):
del oldp[bad[i]],ids[bad[i]]
if(len(ids) == 0):
ERROR("sxpetite","program diverged, all images have large angular errors, most likely the initial model is badly off",1)
else:
# This generate new parameters, hopefully to be used as starting ones in the new iteration
write_text_file(ids,os.path.join(mainoutputdir,"chunk%01d.txt"%procid))
write_text_row(oldp,os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))
if(len(badapples)>0):
badapples.sort()
write_text_file(badapples,os.path.join(mainoutputdir,"badapples.txt"))
eli = 100*float(len(badapples))/float(total_images_now)
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
print(line,"Elimination of outliers: %5.1f percent"%eli )
else: eli = 0.0
del badapples, oldp,ids,bad,newp,ts
else: eli =0.0
eli = bcast_number_to_all(eli, source_node = main_node)
# This part under MPI
if(eli > 0.0):
# Compute current resolution
depres = compute_resolution(stack, mainoutputdir, \
[os.path.join(mainoutputdir,"chunk%01d.txt"%procid) for procid in xrange(2)], \
[os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid) for procid in xrange(2)], \
radi, nnxo, ali3d_options.CTF, myid, main_node, nproc)
depres = bcast_number_to_all(depres, source_node = main_node)
if(depres < newres):
# elimination of outliers decreased resolution, ignore the effort
eliminated_outliers = False
else:
eliminated_outliers = True
newres = depres
"""
# It does not seem to be needed, as data is there, we just point to the directory
for procid in xrange(2):
# set pointers to current parameters in main, which are for the reduced set stored above
partids[procid] = os.path.join(mainoutputdir,"chunk%01d.txt"%procid
partstack[procid] = os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid)
"""
else:
eliminated_outliers = False
else:
eliminated_outliers = False
if(myid == main_node and not eliminated_outliers):
for procid in xrange(2):
# This is standard path, copy parameters to be used to the main
cmd = "{} {} {}".format("cp -p ", partids[procid] , os.path.join(mainoutputdir,"chunk%01d.txt"%procid))
cmdexecute(cmd)
cmd = "{} {} {}".format("cp -p ", partstack[procid], os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))
cmdexecute(cmd)
keepgoing = 0
if( newres > currentres or (eliminated_outliers and not tracker["eliminated-outliers"])):
if(myid == main_node): print(" Resolution improved, full steam ahead!")
if( newres > currentres ): tracker["movedup"] = True
else: tracker["movedup"] = False
shrink = max(min(2*newres + paramsdict["aa"], 1.0),minshrink)
tracker["extension"] = 4
nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo)
tracker["previous-resolution"] = newres
currentres = newres
tracker["bestsolution"] = mainiteration
bestoutputdir = mainoutputdir
tracker["eliminated-outliers"] = eliminated_outliers
keepgoing = 1
elif(newres < currentres):
if(not tracker["movedup"] and tracker["extension"] < 2 and mainiteration > 1):
keepgoing = 0
if(myid == main_node): print(" Cannot improve resolution, the best result is in the directory main%03d"%tracker["bestsolution"])
else:
if(not tracker["movedup"] and tracker["extension"] > 1 and mainiteration > 1):
if(myid == main_node): print(" Resolution decreased. Will decrease target resolution and will fall back on the best so far: main%03d"%tracker["bestsolution"])
bestoutputdir = os.path.join(masterdir,"main%03d"%tracker["bestsolution"])
elif( tracker["movedup"] and tracker["extension"] > 1 and mainiteration > 1):
if(myid == main_node): print(" Resolution decreased. Will decrease target resolution and will try starting from previous stage: main%03d"%(mainiteration - 1))
bestoutputdir = os.path.join(masterdir,"main%03d"%(mainiteration-1))
elif( mainiteration == 1):
if(myid == main_node): print(" Resolution decreased in the first iteration. It is expected, not to worry")
bestoutputdir = mainoutputdir
tracker["extension"] += 1
else: # missing something here?
if(myid == main_node): print(" Should not be here, ERROR 175!")
break
mpi_finalize()
exit()
if( bestoutputdir != mainoutputdir ):
# This is the key, we just reset the main to previous, so it will be eventually used as a starting in the next iteration
mainoutputdir = bestoutputdir
"""
# Set data from the main previous best to the current.
for procid in xrange(2):
partids[procid] = os.path.join(bestoutputdir,"chunk%01d.txt"%procid)
partstack[procid] = os.path.join(bestoutputdir,"params-chunk%01d.txt"%procid)
"""
if(myid == main_node):
currentres = read_text_file( os.path.join(bestoutputdir,"current_resolution.txt") )[0]
currentres = bcast_number_to_all(currentres, source_node = main_node)
shrink = max(min(2*currentres + paramsdict["aa"], 1.0), minshrink)
tracker["extension"] -= 1
nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo)
tracker["previous-resolution"] = newres
tracker["eliminated-outliers"] = eliminated_outliers
tracker["movedup"] = False
keepgoing = 1
elif(newres == currentres):
if( tracker["extension"] > 0 ):
if(myid == main_node): print("The resolution did not improve. This is look ahead move. Let's try to relax slightly and hope for the best")
tracker["extension"] -= 1
tracker["movedup"] = False
shrink = max(min(2*currentres + paramsdict["aa"], 1.0), minshrink)
nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo)
if( tracker["previous-nx"] == nnxo ):
keepgoing = 0
else:
tracker["previous-resolution"] = newres
currentres = newres
tracker["eliminated-outliers"] = eliminated_outliers
tracker["movedup"] = False
keepgoing = 1
else:
if(myid == main_node): print("The resolution did not improve.")
keepgoing = 0
if( keepgoing == 1 ):
if(myid == main_node):
print(" New shrink and image dimension :",shrink,nxshrink)
"""
# It does not look like it is necessary, we just have to point to the directory as the files should be there.
# Will continue, so update the params files
for procid in xrange(2):
# partids ads partstack contain parameters to be used as starting in the next iteration
if(not os.path.exists(os.path.join(mainoutputdir,"chunk%01d.txt"%procid))):
cmd = "{} {} {}".format("cp -p ", partids[procid] , os.path.join(mainoutputdir,"chunk%01d.txt"%procid))
cmdexecute(cmd)
if(not os.path.exists(os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))):
cmd = "{} {} {}".format("cp -p ", partstack[procid], os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))
cmdexecute(cmd)
"""
previousoutputdir = mainoutputdir
tracker["previous-shrink"] = shrink
tracker["previous-nx"] = nxshrink
else:
if(myid == main_node):
print(" Terminating, the best solution is in the directory main%03d"%tracker["bestsolution"])
mpi_barrier(MPI_COMM_WORLD)
mpi_finalize()
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ Input Output [options]
Generate three micrographs, each micrograph contains one projection of a long filament.
Input: Reference Volume, output directory
Output: Three micrographs stored in output directory
sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84
Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
Generate run through example script for helicon
sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
"""
parser = OptionParser(usage,version=SPARXVERSION)
# helicise the Atom coordinates
# generate micrographs of helical filament
parser.add_option("--generate_micrograph", action="store_true", default=False, help="Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory")
parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction")
parser.add_option("--apix", type="float", default= -1, help="pixel size in Angstroms")
parser.add_option("--rand_seed", type="int", default=14567, help="the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs.")
parser.add_option("--Cs", type="float", default= 2.0, help="Microscope Cs (spherical aberation)")
parser.add_option("--voltage", type="float", default=200.0, help="Microscope voltage in KV")
parser.add_option("--ac", type="float", default=10.0, help="Amplitude contrast (percentage, default=10)")
parser.add_option("--nonoise", action="store_true", default=False, help="Do not add noise to the micrograph.")
# generate initial volume
parser.add_option("--generate_noisycyl", action="store_true", default=False, help="Generate initial volume of noisy cylinder.")
parser.add_option("--boxsize", type="string", default="100,100,200", help="String containing x , y, z dimensions (separated by comma) in pixels")
parser.add_option("--rad", type="int", default=35, help="Radius of initial volume in pixels")
# generate 2D mask
parser.add_option("--generate_mask", action="store_true", default=False, help="Generate 2D rectangular mask.")
parser.add_option("--masksize", type="string", default="200,200", help="String containing x and y dimensions (separated by comma) in pixels")
parser.add_option("--maskwidth", type="int", default=60, help="Width of rectangular mask")
# Apply 2D alignment parameters to input stack and output new images to output stack
parser.add_option("--applyparams", action="store_true", default=False, help="Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack")
# Generate run script
parser.add_option("--generate_script", action="store_true", default=False, help="Generate script for helicon run through example")
parser.add_option("--filename", type="string", default="runhelicon", help="Name of run script to generate")
parser.add_option("--seg_ny", type="int", default=180, help="y-dimension of segment used for refinement")
parser.add_option("--ptcl_dist", type="int", default=15, help="Distance in pixels between adjacent segments windowed from same filament")
parser.add_option("--fract", type="float", default=0.35, help="Fraction of the volume used for applying helical symmetry.")
(options, args) = parser.parse_args()
if len(args) > 3:
print "usage: " + usage
print "Please run '" + progname + " -h' for detailed options"
else:
if options.generate_script:
generate_runscript(options.filename, options.seg_ny, options.ptcl_dist, options.fract)
if options.generate_micrograph:
if options.apix <= 0:
print "Please enter pixel size."
sys.exit()
generate_helimic(args[0], args[1], options.apix, options.CTF, options.Cs, options.voltage, options.ac, options.nonoise, options.rand_seed)
if options.generate_noisycyl:
from utilities import model_cylinder, model_gauss_noise
outvol = args[0]
boxdims = options.boxsize.split(',')
if len(boxdims) < 1 or len(boxdims) > 3:
print "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'"
sys.exit()
nx= int(boxdims[0])
if len(boxdims) == 1:
ny = nx
nz = nx
else:
ny = int(boxdims[1])
if len(boxdims) == 3:
nz = int(boxdims[2])
(model_cylinder(options.rad,nx, ny, nz)*model_gauss_noise(1.0, nx, ny, nz) ).write_image(outvol)
if options.generate_mask:
from utilities import model_blank, pad
outvol = args[0]
maskdims = options.masksize.split(',')
if len(maskdims) < 1 or len(maskdims) > 2:
print "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'"
sys.exit()
nx= int(maskdims[0])
if len(maskdims) == 1:
ny = nx
else:
ny = int(maskdims[1])
mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1, 0.0)
mask.write_image(outvol)
if options.applyparams:
from utilities import get_im, get_params2D, set_params2D
from fundamentals import cyclic_shift
stack = args[0]
newstack = args[1]
mask = get_im(args[2])
nima = EMUtil.get_image_count(stack)
for im in xrange(nima):
prj = get_im(stack,im)
alpha, sx, sy, mirror, scale = get_params2D(prj)
prj = cyclic_shift(prj, int(sx))
set_params2D(prj, [0.0,0.,0.0,0,1])
stat = Util.infomask(prj , mask, False )
prj= (prj-stat[0])/stat[1]
ctf_params = prj.get_attr("ctf")
prj.set_attr('ctf_applied', 0)
prj.write_image(newstack, im)
def main():
import sys
import os
import math
import random
import pyemtbx.options
import time
from random import random, seed, randint
from optparse import OptionParser
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Generic 2-D image processing programs.
Functionality:
1. Phase flip a stack of images and write output to new file:
sxprocess.py input_stack.hdf output_stack.hdf --phase_flip
2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.
The window size will change accordingly.
sxprocess input.hdf output.hdf --changesize --ratio=0.5
3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes or a 3-D volume.
sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024]
4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs:
sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64
5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3):
sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid
6. Retrieve original image numbers of images listed in ISAC output stack of averages:
sxprocess.py select1.hdf ohk.txt
7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file.
Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack.
sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw
sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw
8. Generate a 1D rotationally averaged power spectrum of an image.
sxprocess.py vol.hdf --rotwp=rotpw.txt
# Output will contain three columns:
(1) rotationally averaged power spectrum
(2) logarithm of the rotationally averaged power spectrum
(3) integer line number (from zero to approximately to half the image size)
9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data.
sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt
The output file is then imported and 3D transformed volume computed:
sxheader.py bdb:p --params=xform.projection --import=output.txt
mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI
The reconstructed volume is in the position of the volume computed using the input.txt parameters and then
transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz)
10. Import ctf parameters from the output of sxcter into windowed particle headers.
There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph.
In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'.
Normally this is done by e2boxer.py during windowing.
Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created.
sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0
# Output will be a vritual stack bdb:Gdata
sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0
To concatenate output files:
cd directory
e2bdb.py . --makevstack=bdb:allparticles --filt=G
IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them.
11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor
sxprocess.py orientationparams.txt scaledparams.txt scale=0.5
12. Generate 3D mask from a given 3-D volume automatically or using threshold provided by user.
13. Postprocess 3-D or 2-D images:
for 3-D volumes: calculate FSC with provided mask; weight summed volume with FSC; estimate B-factor from FSC weighted summed two volumes; apply negative B-factor to the weighted volume.
for 2-D images: calculate B-factor and apply negative B-factor to 2-D images.
14. Winow stack file -reduce size of images without changing the pixel size.
"""
parser = OptionParser(usage,version=SPARXVERSION)
parser.add_option("--order", action="store_true", help="Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False)
parser.add_option("--order_lookup", action="store_true", help="Test/Debug.", default=False)
parser.add_option("--order_metropolis", action="store_true", help="Test/Debug.", default=False)
parser.add_option("--order_pca", action="store_true", help="Test/Debug.", default=False)
parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)")
parser.add_option("--circular", action="store_true", help="Select circular ordering (fisr image has to be similar to the last", default=False)
parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering")
parser.add_option("--changesize", action="store_true", help="resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False)
parser.add_option("--ratio", type="float", default=1.0, help="The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round")
parser.add_option("--pw", action="store_true", help="compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False)
parser.add_option("--wn", type="int", default=-1, help="Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))")
parser.add_option("--phase_flip", action="store_true", help="Phase flip the input stack", default=False)
parser.add_option("--makedb", metavar="param1=value1:param2=value2", type="string",
action="append", help="One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0")
parser.add_option("--generate_projections", metavar="param1=value1:param2=value2", type="string",
action="append", help="Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64.")
parser.add_option("--isacgroup", type="int", help="Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1)
parser.add_option("--isacselect", action="store_true", help="Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False)
parser.add_option("--params", type="string", default=None, help="Name of header of parameter, which one depends on specific option")
parser.add_option("--adjpw", action="store_true", help="Adjust rotationally averaged power spectrum of an image", default=False)
parser.add_option("--rotpw", type="string", default=None, help="Name of the text file to contain rotationally averaged power spectrum of the input image.")
parser.add_option("--transformparams", type="string", default=None, help="Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters.")
# import ctf estimates done using cter
parser.add_option("--input", type="string", default= None, help="Input particles.")
parser.add_option("--importctf", type="string", default= None, help="Name of the file containing CTF parameters produced by sxcter.")
parser.add_option("--defocuserror", type="float", default=1000000.0, help="Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%")
parser.add_option("--astigmatismerror", type="float", default=360.0, help="Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees.")
# import ctf estimates done using cter
parser.add_option("--scale", type="float", default=-1.0, help="Divide shifts in the input 3D orientation parameters text file by the scale factor.")
# generate adaptive mask from an given 3-D volume
parser.add_option("--adaptive_mask", action="store_true", help="create adavptive 3-D mask from a given volume", default=False)
parser.add_option("--nsigma", type="float", default= 1., help="number of times of sigma of the input volume to obtain the the large density cluster")
parser.add_option("--ndilation", type="int", default= 3, help="number of times of dilation applied to the largest cluster of density")
parser.add_option("--kernel_size", type="int", default= 11, help="convolution kernel for smoothing the edge of the mask")
parser.add_option("--gauss_standard_dev", type="int", default= 9, help="stanadard deviation value to generate Gaussian edge")
parser.add_option("--threshold", type="float", default= 9999., help="threshold provided by user to binarize input volume")
parser.add_option("--ne", type="int", default= 0, help="number of times to erode the binarized input image")
parser.add_option("--nd", type="int", default= 0, help="number of times to dilate the binarized input image")
parser.add_option("--postprocess", action="store_true", help="postprocess unfiltered odd, even 3-D volumes",default=False)
parser.add_option("--fsc_weighted", action="store_true", help="postprocess unfiltered odd, even 3-D volumes")
parser.add_option("--low_pass_filter", action="store_true", default=False, help="postprocess unfiltered odd, even 3-D volumes")
parser.add_option("--ff", type="float", default=.25, help="low pass filter stop band frequency in absolute unit")
parser.add_option("--aa", type="float", default=.1, help="low pass filter falloff" )
parser.add_option("--mask", type="string", help="input mask file", default=None)
parser.add_option("--output", type="string", help="output file name", default="postprocessed.hdf")
parser.add_option("--pixel_size", type="float", help="pixel size of the data", default=1.0)
parser.add_option("--B_start", type="float", help="starting frequency in Angstrom for B-factor estimation", default=10.)
parser.add_option("--FSC_cutoff", type="float", help="stop frequency in Angstrom for B-factor estimation", default=0.143)
parser.add_option("--2d", action="store_true", help="postprocess isac 2-D averaged images",default=False)
parser.add_option("--window_stack", action="store_true", help="window stack images using a smaller window size", default=False)
parser.add_option("--box", type="int", default= 0, help="the new window size ")
(options, args) = parser.parse_args()
global_def.BATCH = True
if options.phase_flip:
nargs = len(args)
if nargs != 2:
print "must provide name of input and output file!"
return
from EMAN2 import Processor
instack = args[0]
outstack = args[1]
nima = EMUtil.get_image_count(instack)
from filter import filt_ctf
for i in xrange(nima):
img = EMData()
img.read_image(instack, i)
try:
ctf = img.get_attr('ctf')
except:
print "no ctf information in input stack! Exiting..."
return
dopad = True
sign = 1
binary = 1 # phase flip
assert img.get_ysize() > 1
dict = ctf.to_dict()
dz = dict["defocus"]
cs = dict["cs"]
voltage = dict["voltage"]
pixel_size = dict["apix"]
b_factor = dict["bfactor"]
ampcont = dict["ampcont"]
dza = dict["dfdiff"]
azz = dict["dfang"]
if dopad and not img.is_complex(): ip = 1
else: ip = 0
params = {"filter_type": Processor.fourier_filter_types.CTF_,
"defocus" : dz,
"Cs": cs,
"voltage": voltage,
"Pixel_size": pixel_size,
"B_factor": b_factor,
"amp_contrast": ampcont,
"dopad": ip,
"binary": binary,
"sign": sign,
"dza": dza,
"azz":azz}
tmp = Processor.EMFourierFilter(img, params)
tmp.set_attr_dict({"ctf": ctf})
tmp.write_image(outstack, i)
elif options.changesize:
nargs = len(args)
if nargs != 2:
ERROR("must provide name of input and output file!", "change size", 1)
return
from utilities import get_im
instack = args[0]
outstack = args[1]
sub_rate = float(options.ratio)
nima = EMUtil.get_image_count(instack)
from fundamentals import resample
for i in xrange(nima):
resample(get_im(instack, i), sub_rate).write_image(outstack, i)
elif options.isacgroup>-1:
nargs = len(args)
if nargs != 3:
ERROR("Three files needed on input!", "isacgroup", 1)
return
from utilities import get_im
instack = args[0]
m=get_im(args[1],int(options.isacgroup)).get_attr("members")
l = []
for k in m:
l.append(int(get_im(args[0],k).get_attr(options.params)))
from utilities import write_text_file
write_text_file(l, args[2])
elif options.isacselect:
nargs = len(args)
if nargs != 2:
ERROR("Two files needed on input!", "isacgroup", 1)
return
from utilities import get_im
nima = EMUtil.get_image_count(args[0])
m = []
for k in xrange(nima):
m += get_im(args[0],k).get_attr("members")
m.sort()
from utilities import write_text_file
write_text_file(m, args[1])
elif options.pw:
nargs = len(args)
if nargs < 2:
ERROR("must provide name of input and output file!", "pw", 1)
return
from utilities import get_im, write_text_file
from fundamentals import rops_table
d = get_im(args[0])
ndim = d.get_ndim()
if ndim ==3:
pw = rops_table(d)
write_text_file(pw, args[1])
else:
nx = d.get_xsize()
ny = d.get_ysize()
if nargs ==3: mask = get_im(args[2])
wn = int(options.wn)
if wn == -1:
wn = max(nx, ny)
else:
if( (wn<nx) or (wn<ny) ): ERROR("window size cannot be smaller than the image size","pw",1)
n = EMUtil.get_image_count(args[0])
from utilities import model_blank, model_circle, pad
from EMAN2 import periodogram
p = model_blank(wn,wn)
for i in xrange(n):
d = get_im(args[0], i)
if nargs==3:
d *=mask
st = Util.infomask(d, None, True)
d -= st[0]
p += periodogram(pad(d, wn, wn, 1, 0.))
p /= n
p.write_image(args[1])
elif options.adjpw:
if len(args) < 3:
ERROR("filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)","adjpw",1)
return
img_stack = args[0]
from math import sqrt
from fundamentals import rops_table, fft
from utilities import read_text_file, get_im
from filter import filt_tanl, filt_table
if( args[1][-3:] == 'txt'):
rops_dst = read_text_file( args[1] )
else:
rops_dst = rops_table(get_im( args[1] ))
out_stack = args[2]
if(len(args) >4):
fl = float(args[3])
aa = float(args[4])
else:
fl = -1.0
aa = 0.0
nimage = EMUtil.get_image_count( img_stack )
for i in xrange(nimage):
img = fft(get_im(img_stack, i) )
rops_src = rops_table(img)
assert len(rops_dst) == len(rops_src)
table = [0.0]*len(rops_dst)
for j in xrange( len(rops_dst) ):
table[j] = sqrt( rops_dst[j]/rops_src[j] )
if( fl > 0.0):
img = filt_tanl(img, fl, aa)
img = fft(filt_table(img, table))
img.write_image(out_stack, i)
elif options.rotpw != None:
if len(args) != 1:
ERROR("Only one input permitted","rotpw",1)
return
from utilities import write_text_file, get_im
from fundamentals import rops_table
from math import log10
t = rops_table(get_im(args[0]))
x = range(len(t))
r = [0.0]*len(x)
for i in x: r[i] = log10(t[i])
write_text_file([t,r,x],options.rotpw)
elif options.transformparams != None:
if len(args) != 2:
ERROR("Please provide names of input and output files with orientation parameters","transformparams",1)
return
from utilities import read_text_row, write_text_row
transf = [0.0]*6
spl=options.transformparams.split(',')
for i in xrange(len(spl)): transf[i] = float(spl[i])
write_text_row( rotate_shift_params(read_text_row(args[0]), transf) , args[1])
elif options.makedb != None:
nargs = len(args)
if nargs != 1:
print "must provide exactly one argument denoting database key under which the input params will be stored"
return
dbkey = args[0]
print "database key under which params will be stored: ", dbkey
gbdb = js_open_dict("e2boxercache/gauss_box_DB.json")
parmstr = 'dummy:'+options.makedb[0]
(processorname, param_dict) = parsemodopt(parmstr)
dbdict = {}
for pkey in param_dict:
if (pkey == 'invert_contrast') or (pkey == 'use_variance'):
if param_dict[pkey] == 'True':
dbdict[pkey] = True
else:
dbdict[pkey] = False
else:
dbdict[pkey] = param_dict[pkey]
gbdb[dbkey] = dbdict
elif options.generate_projections:
nargs = len(args)
if nargs != 3:
ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\
"sxprocess - generate projections",1)
return
inpstr = args[0]
outstk = args[1]
micpref = args[2]
parmstr = 'dummy:'+options.generate_projections[0]
(processorname, param_dict) = parsemodopt(parmstr)
parm_CTF = False
parm_format = 'bdb'
parm_apix = 2.5
if 'CTF' in param_dict:
if param_dict['CTF'] == 'True':
parm_CTF = True
if 'format' in param_dict:
parm_format = param_dict['format']
if 'apix' in param_dict:
parm_apix = float(param_dict['apix'])
boxsize = 64
if 'boxsize' in param_dict:
boxsize = int(param_dict['boxsize'])
print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize
scale_mult = 2500
sigma_add = 1.5
sigma_proj = 30.0
sigma2_proj = 17.5
sigma_gauss = 0.3
sigma_mic = 30.0
sigma2_mic = 17.5
sigma_gauss_mic = 0.3
if 'scale_mult' in param_dict:
scale_mult = float(param_dict['scale_mult'])
if 'sigma_add' in param_dict:
sigma_add = float(param_dict['sigma_add'])
if 'sigma_proj' in param_dict:
sigma_proj = float(param_dict['sigma_proj'])
if 'sigma2_proj' in param_dict:
sigma2_proj = float(param_dict['sigma2_proj'])
if 'sigma_gauss' in param_dict:
sigma_gauss = float(param_dict['sigma_gauss'])
if 'sigma_mic' in param_dict:
sigma_mic = float(param_dict['sigma_mic'])
if 'sigma2_mic' in param_dict:
sigma2_mic = float(param_dict['sigma2_mic'])
if 'sigma_gauss_mic' in param_dict:
sigma_gauss_mic = float(param_dict['sigma_gauss_mic'])
from filter import filt_gaussl, filt_ctf
from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank,pad,model_gauss_noise,set_params2D, set_params_proj
from projection import prep_vol,prgs
seed(14567)
delta = 29
angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S")
nangle = len(angles)
modelvol = []
nvlms = EMUtil.get_image_count(inpstr)
from utilities import get_im
for k in xrange(nvlms): modelvol.append(get_im(inpstr,k))
nx = modelvol[0].get_xsize()
if nx != boxsize:
ERROR("Requested box dimension does not match dimension of the input model.", \
"sxprocess - generate projections",1)
nvol = 10
volfts = [[] for k in xrange(nvlms)]
for k in xrange(nvlms):
for i in xrange(nvol):
sigma = sigma_add + random() # 1.5-2.5
addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40 )
scale = scale_mult * (0.5+random())
vf, kb = prep_vol(modelvol[k] + scale*addon)
volfts[k].append(vf)
del vf, modelvol
if parm_format == "bdb":
stack_data = "bdb:"+outstk
delete_bdb(stack_data)
else:
stack_data = outstk + ".hdf"
Cs = 2.0
pixel = parm_apix
voltage = 120.0
ampcont = 10.0
ibd = 4096/2-boxsize
iprj = 0
width = 240
xstart = 8 + boxsize/2
ystart = 8 + boxsize/2
rowlen = 17
from random import randint
params = []
for idef in xrange(3, 8):
irow = 0
icol = 0
mic = model_blank(4096, 4096)
defocus = idef * 0.5#0.2
if parm_CTF:
astampl=defocus*0.15
astangl=50.0
ctf = generate_ctf([defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl])
for i in xrange(nangle):
for k in xrange(12):
dphi = 8.0*(random()-0.5)
dtht = 8.0*(random()-0.5)
psi = 360.0*random()
phi = angles[i][0]+dphi
tht = angles[i][1]+dtht
s2x = 4.0*(random()-0.5)
s2y = 4.0*(random()-0.5)
params.append([phi, tht, psi, s2x, s2y])
ivol = iprj % nvol
#imgsrc = randint(0,nvlms-1)
imgsrc = iprj % nvlms
proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y])
x = xstart + irow * width
y = ystart + icol * width
mic += pad(proj, 4096, 4096, 1, 0.0, x-2048, y-2048, 0)
proj = proj + model_gauss_noise( sigma_proj, nx, nx )
if parm_CTF:
proj = filt_ctf(proj, ctf)
proj.set_attr_dict({"ctf":ctf, "ctf_applied":0})
proj = proj + filt_gaussl(model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss)
proj.set_attr("origimgsrc",imgsrc)
proj.set_attr("test_id", iprj)
# flags describing the status of the image (1 = true, 0 = false)
set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0])
set_params_proj(proj, [phi, tht, psi, s2x, s2y])
proj.write_image(stack_data, iprj)
icol += 1
if icol == rowlen:
icol = 0
irow += 1
iprj += 1
mic += model_gauss_noise(sigma_mic,4096,4096)
if parm_CTF:
#apply CTF
mic = filt_ctf(mic, ctf)
mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic)
mic.write_image(micpref + "%1d.hdf" % (idef-3), 0)
drop_spider_doc("params.txt", params)
elif options.importctf != None:
print ' IMPORTCTF '
from utilities import read_text_row,write_text_row
from random import randint
import subprocess
grpfile = 'groupid%04d'%randint(1000,9999)
ctfpfile = 'ctfpfile%04d'%randint(1000,9999)
cterr = [options.defocuserror/100.0, options.astigmatismerror]
ctfs = read_text_row(options.importctf)
for kk in xrange(len(ctfs)):
root,name = os.path.split(ctfs[kk][-1])
ctfs[kk][-1] = name[:-4]
if(options.input[:4] != 'bdb:'):
ERROR('Sorry, only bdb files implemented','importctf',1)
d = options.input[4:]
#try: str = d.index('*')
#except: str = -1
from string import split
import glob
uu = os.path.split(d)
uu = os.path.join(uu[0],'EMAN2DB',uu[1]+'.bdb')
flist = glob.glob(uu)
for i in xrange(len(flist)):
root,name = os.path.split(flist[i])
root = root[:-7]
name = name[:-4]
fil = 'bdb:'+os.path.join(root,name)
sourcemic = EMUtil.get_all_attributes(fil,'ptcl_source_image')
nn = len(sourcemic)
gctfp = []
groupid = []
for kk in xrange(nn):
junk,name2 = os.path.split(sourcemic[kk])
name2 = name2[:-4]
ctfp = [-1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]
for ll in xrange(len(ctfs)):
if(name2 == ctfs[ll][-1]):
# found correct
if(ctfs[ll][8]/ctfs[ll][0] <= cterr[0]):
# acceptable defocus error
ctfp = ctfs[ll][:8]
if(ctfs[ll][10] > cterr[1] ):
# error of astigmatism exceed the threshold, set astigmatism to zero.
ctfp[6] = 0.0
ctfp[7] = 0.0
gctfp.append(ctfp)
groupid.append(kk)
break
if(len(groupid) > 0):
write_text_row(groupid, grpfile)
write_text_row(gctfp, ctfpfile)
cmd = "{} {} {} {}".format('e2bdb.py',fil,'--makevstack=bdb:'+root+'G'+name,'--list='+grpfile)
#print cmd
subprocess.call(cmd, shell=True)
cmd = "{} {} {} {}".format('sxheader.py','bdb:'+root+'G'+name,'--params=ctf','--import='+ctfpfile)
#print cmd
subprocess.call(cmd, shell=True)
else:
print ' >>> Group ',name,' skipped.'
cmd = "{} {} {}".format("rm -f",grpfile,ctfpfile)
subprocess.call(cmd, shell=True)
elif options.scale > 0.0:
from utilities import read_text_row,write_text_row
scale = options.scale
nargs = len(args)
if nargs != 2:
print "Please provide names of input and output file!"
return
p = read_text_row(args[0])
for i in xrange(len(p)):
p[i][3] /= scale
p[i][4] /= scale
write_text_row(p, args[1])
elif options.adaptive_mask:
from utilities import get_im
from morphology import adaptive_mask, binarize, erosion, dilation
nsigma = options.nsigma
ndilation = options.ndilation
kernel_size = options.kernel_size
gauss_standard_dev = options.gauss_standard_dev
nargs = len(args)
if nargs ==0:
print " Create 3D mask from a given volume, either automatically or from the user provided threshold."
elif nargs > 2:
print "Too many inputs are given, try again!"
return
else:
inputvol = get_im(args[0])
input_path, input_file_name = os.path.split(args[0])
input_file_name_root,ext=os.path.splitext(input_file_name)
if nargs == 2: mask_file_name = args[1]
else: mask_file_name = "adaptive_mask_for_"+input_file_name_root+".hdf" # Only hdf file is output.
if options.threshold !=9999.:
mask3d = binarize(inputvol, options.threshold)
for i in xrange(options.ne): mask3d = erosion(mask3d)
for i in xrange(options.nd): mask3d = dilation(mask3d)
else:
mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev)
mask3d.write_image(mask_file_name)
elif options.postprocess:
from utilities import get_im
from fundamentals import rot_avg_table
from morphology import compute_bfactor,power
from statistics import fsc
from filter import filt_table, filt_gaussinv
from EMAN2 import periodogram
e1 = get_im(args[0],0)
if e1.get_zsize()==1:
nimage = EMUtil.get_image_count(args[0])
if options.mask !=None: m = get_im(options.mask)
else: m = None
for i in xrange(nimage):
e1 = get_im(args[0],i)
if m: e1 *=m
guinerline = rot_avg_table(power(periodogram(e1),.5))
freq_max = 1/(2.*pixel_size)
freq_min = 1./options.B_start
b,junk=compute_bfactor(guinerline, freq_min, freq_max, pixel_size)
tmp = b/pixel_size**2
sigma_of_inverse=sqrt(2./tmp)
e1 = filt_gaussinv(e1,sigma_of_inverse)
if options.low_pass_filter:
from filter import filt_tanl
e1 =filt_tanl(e1,options.ff, options.aa)
e1.write_image(options.output)
else:
nargs = len(args)
e1 = get_im(args[0])
if nargs >1: e2 = get_im(args[1])
if options.mask !=None: m = get_im(options.mask)
else: m =None
pixel_size = options.pixel_size
from math import sqrt
if m !=None:
e1 *=m
if nargs >1 :e2 *=m
if options.fsc_weighted:
frc = fsc(e1,e2,1)
## FSC is done on masked two images
#### FSC weighting sqrt((2.*fsc)/(1+fsc));
fil = len(frc[1])*[None]
for i in xrange(len(fil)):
if frc[1][i]>=options.FSC_cutoff: tmp = frc[1][i]
else: tmp = 0.0
fil[i] = sqrt(2.*tmp/(1.+tmp))
if nargs>1: e1 +=e2
if options.fsc_weighted: e1=filt_table(e1,fil)
guinerline = rot_avg_table(power(periodogram(e1),.5))
freq_max = 1/(2.*pixel_size)
freq_min = 1./options.B_start
b,junk = compute_bfactor(guinerline, freq_min, freq_max, pixel_size)
tmp = b/pixel_size**2
sigma_of_inverse=sqrt(2./tmp)
e1 = filt_gaussinv(e1,sigma_of_inverse)
if options.low_pass_filter:
from filter import filt_tanl
e1 =filt_tanl(e1,options.ff, options.aa)
e1.write_image(options.output)
elif options.window_stack:
nargs = len(args)
if nargs ==0:
print " Reduce image size of a stack"
return
else:
output_stack_name = None
inputstack = args[0]
if nargs ==2:output_stack_name = args[1]
input_path,input_file_name=os.path.split(inputstack)
input_file_name_root,ext=os.path.splitext(input_file_name)
if input_file_name_root[0:3]=="bdb":stack_is_bdb= True
else: stack_is_bdb= False
if output_stack_name is None:
if stack_is_bdb: output_stack_name ="bdb:reduced_"+input_file_name_root[4:]
else:output_stack_name = "reduced_"+input_file_name_root+".hdf" # Only hdf file is output.
nimage = EMUtil.get_image_count(inputstack)
from fundamentals import window2d
for i in xrange(nimage):
image = EMData()
image.read_image(inputstack,i)
w = window2d(image,options.box,options.box)
w.write_image(output_stack_name,i)
else: ERROR("Please provide option name","sxprocess.py",1)
def main():
import sys
import os
import math
import random
import pyemtbx.options
import time
from random import random, seed, randint
from optparse import OptionParser
progname = os.path.basename(sys.argv[0])
usage = progname + """ [options] <inputfile> <outputfile>
Generic 2-D image processing programs.
Functionality:
1. Phase flip a stack of images and write output to new file:
sxprocess.py input_stack.hdf output_stack.hdf --phase_flip
2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.
The window size will change accordingly.
sxprocess input.hdf output.hdf --changesize --ratio=0.5
3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes.
sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024]
4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs:
sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64
5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3):
sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid
6. Retrieve original image numbers of images listed in ISAC output stack of averages:
sxprocess.py select1.hdf ohk.txt
7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file.
Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack.
sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw
sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw
8. Generate a 1D rotationally averaged power spectrum of an image.
sxprocess.py vol.hdf --rotwp=rotpw.txt
# Output will contain three columns:
(1) rotationally averaged power spectrum
(2) logarithm of the rotationally averaged power spectrum
(3) integer line number (from zero to approximately to half the image size)
9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data.
sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt
The output file is then imported and 3D transformed volume computed:
sxheader.py bdb:p --params=xform.projection --import=output.txt
mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI
The reconstructed volume is in the position of the volume computed using the input.txt parameters and then
transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz)
10. Import ctf parameters from the output of sxcter into windowed particle headers.
There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph.
In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'.
Normally this is done by e2boxer.py during windowing.
Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created.
sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0
# Output will be a vritual stack bdb:Gdata
sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0
To concatenate output files:
cd directory
e2bdb.py . --makevstack=bdb:allparticles --filt=G
IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them.
11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor
sxprocess.py orientationparams.txt scaledparams.txt scale=0.5
12. Generate adaptive mask from a given 3-D volume.
"""
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--order",
action="store_true",
help=
"Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.",
default=False)
parser.add_option("--order_lookup",
action="store_true",
help="Test/Debug.",
default=False)
parser.add_option("--order_metropolis",
action="store_true",
help="Test/Debug.",
default=False)
parser.add_option("--order_pca",
action="store_true",
help="Test/Debug.",
default=False)
parser.add_option(
"--initial",
type="int",
default=-1,
help=
"Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)"
)
parser.add_option(
"--circular",
action="store_true",
help=
"Select circular ordering (fisr image has to be similar to the last",
default=False)
parser.add_option(
"--radius",
type="int",
default=-1,
help="Radius of a circular mask for similarity based ordering")
parser.add_option(
"--changesize",
action="store_true",
help=
"resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.",
default=False)
parser.add_option(
"--ratio",
type="float",
default=1.0,
help=
"The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round"
)
parser.add_option(
"--pw",
action="store_true",
help=
"compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes",
default=False)
parser.add_option(
"--wn",
type="int",
default=-1,
help=
"Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))"
)
parser.add_option("--phase_flip",
action="store_true",
help="Phase flip the input stack",
default=False)
parser.add_option(
"--makedb",
metavar="param1=value1:param2=value2",
type="string",
action="append",
help=
"One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0"
)
parser.add_option(
"--generate_projections",
metavar="param1=value1:param2=value2",
type="string",
action="append",
help=
"Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64."
)
parser.add_option(
"--isacgroup",
type="int",
help=
"Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.",
default=-1)
parser.add_option(
"--isacselect",
action="store_true",
help=
"Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.",
default=False)
parser.add_option(
"--params",
type="string",
default=None,
help="Name of header of parameter, which one depends on specific option"
)
parser.add_option(
"--adjpw",
action="store_true",
help="Adjust rotationally averaged power spectrum of an image",
default=False)
parser.add_option(
"--rotpw",
type="string",
default=None,
help=
"Name of the text file to contain rotationally averaged power spectrum of the input image."
)
parser.add_option(
"--transformparams",
type="string",
default=None,
help=
"Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters."
)
# import ctf estimates done using cter
parser.add_option("--input",
type="string",
default=None,
help="Input particles.")
parser.add_option(
"--importctf",
type="string",
default=None,
help="Name of the file containing CTF parameters produced by sxcter.")
parser.add_option(
"--defocuserror",
type="float",
default=1000000.0,
help=
"Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%"
)
parser.add_option(
"--astigmatismerror",
type="float",
default=360.0,
help=
"Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees."
)
# import ctf estimates done using cter
parser.add_option(
"--scale",
type="float",
default=-1.0,
help=
"Divide shifts in the input 3D orientation parameters text file by the scale factor."
)
# generate adaptive mask from an given 3-Db volue
parser.add_option("--adaptive_mask",
action="store_true",
help="create adavptive 3-D mask from a given volume",
default=False)
parser.add_option(
"--nsigma",
type="float",
default=1.,
help=
"number of times of sigma of the input volume to obtain the the large density cluster"
)
parser.add_option(
"--ndilation",
type="int",
default=3,
help=
"number of times of dilation applied to the largest cluster of density"
)
parser.add_option(
"--kernel_size",
type="int",
default=11,
help="convolution kernel for smoothing the edge of the mask")
parser.add_option(
"--gauss_standard_dev",
type="int",
default=9,
help="stanadard deviation value to generate Gaussian edge")
(options, args) = parser.parse_args()
global_def.BATCH = True
if options.phase_flip:
nargs = len(args)
if nargs != 2:
print "must provide name of input and output file!"
return
from EMAN2 import Processor
instack = args[0]
outstack = args[1]
nima = EMUtil.get_image_count(instack)
from filter import filt_ctf
for i in xrange(nima):
img = EMData()
img.read_image(instack, i)
try:
ctf = img.get_attr('ctf')
except:
print "no ctf information in input stack! Exiting..."
return
dopad = True
sign = 1
binary = 1 # phase flip
assert img.get_ysize() > 1
dict = ctf.to_dict()
dz = dict["defocus"]
cs = dict["cs"]
voltage = dict["voltage"]
pixel_size = dict["apix"]
b_factor = dict["bfactor"]
ampcont = dict["ampcont"]
dza = dict["dfdiff"]
azz = dict["dfang"]
if dopad and not img.is_complex(): ip = 1
else: ip = 0
params = {
"filter_type": Processor.fourier_filter_types.CTF_,
"defocus": dz,
"Cs": cs,
"voltage": voltage,
"Pixel_size": pixel_size,
"B_factor": b_factor,
"amp_contrast": ampcont,
"dopad": ip,
"binary": binary,
"sign": sign,
"dza": dza,
"azz": azz
}
tmp = Processor.EMFourierFilter(img, params)
tmp.set_attr_dict({"ctf": ctf})
tmp.write_image(outstack, i)
elif options.changesize:
nargs = len(args)
if nargs != 2:
ERROR("must provide name of input and output file!", "change size",
1)
return
from utilities import get_im
instack = args[0]
outstack = args[1]
sub_rate = float(options.ratio)
nima = EMUtil.get_image_count(instack)
from fundamentals import resample
for i in xrange(nima):
resample(get_im(instack, i), sub_rate).write_image(outstack, i)
elif options.isacgroup > -1:
nargs = len(args)
if nargs != 3:
ERROR("Three files needed on input!", "isacgroup", 1)
return
from utilities import get_im
instack = args[0]
m = get_im(args[1], int(options.isacgroup)).get_attr("members")
l = []
for k in m:
l.append(int(get_im(args[0], k).get_attr(options.params)))
from utilities import write_text_file
write_text_file(l, args[2])
elif options.isacselect:
nargs = len(args)
if nargs != 2:
ERROR("Two files needed on input!", "isacgroup", 1)
return
from utilities import get_im
nima = EMUtil.get_image_count(args[0])
m = []
for k in xrange(nima):
m += get_im(args[0], k).get_attr("members")
m.sort()
from utilities import write_text_file
write_text_file(m, args[1])
elif options.pw:
nargs = len(args)
if nargs < 2:
ERROR("must provide name of input and output file!", "pw", 1)
return
from utilities import get_im
d = get_im(args[0])
nx = d.get_xsize()
ny = d.get_ysize()
if nargs == 3: mask = get_im(args[2])
wn = int(options.wn)
if wn == -1:
wn = max(nx, ny)
else:
if ((wn < nx) or (wn < ny)):
ERROR("window size cannot be smaller than the image size",
"pw", 1)
n = EMUtil.get_image_count(args[0])
from utilities import model_blank, model_circle, pad
from EMAN2 import periodogram
p = model_blank(wn, wn)
for i in xrange(n):
d = get_im(args[0], i)
if nargs == 3:
d *= mask
st = Util.infomask(d, None, True)
d -= st[0]
p += periodogram(pad(d, wn, wn, 1, 0.))
p /= n
p.write_image(args[1])
elif options.adjpw:
if len(args) < 3:
ERROR(
"filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)",
"adjpw", 1)
return
img_stack = args[0]
from math import sqrt
from fundamentals import rops_table, fft
from utilities import read_text_file, get_im
from filter import filt_tanl, filt_table
if (args[1][-3:] == 'txt'):
rops_dst = read_text_file(args[1])
else:
rops_dst = rops_table(get_im(args[1]))
out_stack = args[2]
if (len(args) > 4):
fl = float(args[3])
aa = float(args[4])
else:
fl = -1.0
aa = 0.0
nimage = EMUtil.get_image_count(img_stack)
for i in xrange(nimage):
img = fft(get_im(img_stack, i))
rops_src = rops_table(img)
assert len(rops_dst) == len(rops_src)
table = [0.0] * len(rops_dst)
for j in xrange(len(rops_dst)):
table[j] = sqrt(rops_dst[j] / rops_src[j])
if (fl > 0.0):
img = filt_tanl(img, fl, aa)
img = fft(filt_table(img, table))
img.write_image(out_stack, i)
elif options.rotpw != None:
if len(args) != 1:
ERROR("Only one input permitted", "rotpw", 1)
return
from utilities import write_text_file, get_im
from fundamentals import rops_table
from math import log10
t = rops_table(get_im(args[0]))
x = range(len(t))
r = [0.0] * len(x)
for i in x:
r[i] = log10(t[i])
write_text_file([t, r, x], options.rotpw)
elif options.transformparams != None:
if len(args) != 2:
ERROR(
"Please provide names of input and output files with orientation parameters",
"transformparams", 1)
return
from utilities import read_text_row, write_text_row
transf = [0.0] * 6
spl = options.transformparams.split(',')
for i in xrange(len(spl)):
transf[i] = float(spl[i])
write_text_row(rotate_shift_params(read_text_row(args[0]), transf),
args[1])
elif options.makedb != None:
nargs = len(args)
if nargs != 1:
print "must provide exactly one argument denoting database key under which the input params will be stored"
return
dbkey = args[0]
print "database key under which params will be stored: ", dbkey
gbdb = js_open_dict("e2boxercache/gauss_box_DB.json")
parmstr = 'dummy:' + options.makedb[0]
(processorname, param_dict) = parsemodopt(parmstr)
dbdict = {}
for pkey in param_dict:
if (pkey == 'invert_contrast') or (pkey == 'use_variance'):
if param_dict[pkey] == 'True':
dbdict[pkey] = True
else:
dbdict[pkey] = False
else:
dbdict[pkey] = param_dict[pkey]
gbdb[dbkey] = dbdict
elif options.generate_projections:
nargs = len(args)
if nargs != 3:
ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\
"sxprocess - generate projections",1)
return
inpstr = args[0]
outstk = args[1]
micpref = args[2]
parmstr = 'dummy:' + options.generate_projections[0]
(processorname, param_dict) = parsemodopt(parmstr)
parm_CTF = False
parm_format = 'bdb'
parm_apix = 2.5
if 'CTF' in param_dict:
if param_dict['CTF'] == 'True':
parm_CTF = True
if 'format' in param_dict:
parm_format = param_dict['format']
if 'apix' in param_dict:
parm_apix = float(param_dict['apix'])
boxsize = 64
if 'boxsize' in param_dict:
boxsize = int(param_dict['boxsize'])
print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize
scale_mult = 2500
sigma_add = 1.5
sigma_proj = 30.0
sigma2_proj = 17.5
sigma_gauss = 0.3
sigma_mic = 30.0
sigma2_mic = 17.5
sigma_gauss_mic = 0.3
if 'scale_mult' in param_dict:
scale_mult = float(param_dict['scale_mult'])
if 'sigma_add' in param_dict:
sigma_add = float(param_dict['sigma_add'])
if 'sigma_proj' in param_dict:
sigma_proj = float(param_dict['sigma_proj'])
if 'sigma2_proj' in param_dict:
sigma2_proj = float(param_dict['sigma2_proj'])
if 'sigma_gauss' in param_dict:
sigma_gauss = float(param_dict['sigma_gauss'])
if 'sigma_mic' in param_dict:
sigma_mic = float(param_dict['sigma_mic'])
if 'sigma2_mic' in param_dict:
sigma2_mic = float(param_dict['sigma2_mic'])
if 'sigma_gauss_mic' in param_dict:
sigma_gauss_mic = float(param_dict['sigma_gauss_mic'])
from filter import filt_gaussl, filt_ctf
from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank, pad, model_gauss_noise, set_params2D, set_params_proj
from projection import prep_vol, prgs
seed(14567)
delta = 29
angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S")
nangle = len(angles)
modelvol = []
nvlms = EMUtil.get_image_count(inpstr)
from utilities import get_im
for k in xrange(nvlms):
modelvol.append(get_im(inpstr, k))
nx = modelvol[0].get_xsize()
if nx != boxsize:
ERROR("Requested box dimension does not match dimension of the input model.", \
"sxprocess - generate projections",1)
nvol = 10
volfts = [[] for k in xrange(nvlms)]
for k in xrange(nvlms):
for i in xrange(nvol):
sigma = sigma_add + random() # 1.5-2.5
addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma,
sigma, 38, 38, 40)
scale = scale_mult * (0.5 + random())
vf, kb = prep_vol(modelvol[k] + scale * addon)
volfts[k].append(vf)
del vf, modelvol
if parm_format == "bdb":
stack_data = "bdb:" + outstk
delete_bdb(stack_data)
else:
stack_data = outstk + ".hdf"
Cs = 2.0
pixel = parm_apix
voltage = 120.0
ampcont = 10.0
ibd = 4096 / 2 - boxsize
iprj = 0
width = 240
xstart = 8 + boxsize / 2
ystart = 8 + boxsize / 2
rowlen = 17
from random import randint
params = []
for idef in xrange(3, 8):
irow = 0
icol = 0
mic = model_blank(4096, 4096)
defocus = idef * 0.5 #0.2
if parm_CTF:
astampl = defocus * 0.15
astangl = 50.0
ctf = generate_ctf([
defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl
])
for i in xrange(nangle):
for k in xrange(12):
dphi = 8.0 * (random() - 0.5)
dtht = 8.0 * (random() - 0.5)
psi = 360.0 * random()
phi = angles[i][0] + dphi
tht = angles[i][1] + dtht
s2x = 4.0 * (random() - 0.5)
s2y = 4.0 * (random() - 0.5)
params.append([phi, tht, psi, s2x, s2y])
ivol = iprj % nvol
#imgsrc = randint(0,nvlms-1)
imgsrc = iprj % nvlms
proj = prgs(volfts[imgsrc][ivol], kb,
[phi, tht, psi, -s2x, -s2y])
x = xstart + irow * width
y = ystart + icol * width
mic += pad(proj, 4096, 4096, 1, 0.0, x - 2048, y - 2048, 0)
proj = proj + model_gauss_noise(sigma_proj, nx, nx)
if parm_CTF:
proj = filt_ctf(proj, ctf)
proj.set_attr_dict({"ctf": ctf, "ctf_applied": 0})
proj = proj + filt_gaussl(
model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss)
proj.set_attr("origimgsrc", imgsrc)
proj.set_attr("test_id", iprj)
# flags describing the status of the image (1 = true, 0 = false)
set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0])
set_params_proj(proj, [phi, tht, psi, s2x, s2y])
proj.write_image(stack_data, iprj)
icol += 1
if icol == rowlen:
icol = 0
irow += 1
iprj += 1
mic += model_gauss_noise(sigma_mic, 4096, 4096)
if parm_CTF:
#apply CTF
mic = filt_ctf(mic, ctf)
mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096),
sigma_gauss_mic)
mic.write_image(micpref + "%1d.hdf" % (idef - 3), 0)
drop_spider_doc("params.txt", params)
elif options.importctf != None:
print ' IMPORTCTF '
from utilities import read_text_row, write_text_row
from random import randint
import subprocess
grpfile = 'groupid%04d' % randint(1000, 9999)
ctfpfile = 'ctfpfile%04d' % randint(1000, 9999)
cterr = [options.defocuserror / 100.0, options.astigmatismerror]
ctfs = read_text_row(options.importctf)
for kk in xrange(len(ctfs)):
root, name = os.path.split(ctfs[kk][-1])
ctfs[kk][-1] = name[:-4]
if (options.input[:4] != 'bdb:'):
ERROR('Sorry, only bdb files implemented', 'importctf', 1)
d = options.input[4:]
#try: str = d.index('*')
#except: str = -1
from string import split
import glob
uu = os.path.split(d)
uu = os.path.join(uu[0], 'EMAN2DB', uu[1] + '.bdb')
flist = glob.glob(uu)
for i in xrange(len(flist)):
root, name = os.path.split(flist[i])
root = root[:-7]
name = name[:-4]
fil = 'bdb:' + os.path.join(root, name)
sourcemic = EMUtil.get_all_attributes(fil, 'ptcl_source_image')
nn = len(sourcemic)
gctfp = []
groupid = []
for kk in xrange(nn):
junk, name2 = os.path.split(sourcemic[kk])
name2 = name2[:-4]
ctfp = [-1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for ll in xrange(len(ctfs)):
if (name2 == ctfs[ll][-1]):
# found correct
if (ctfs[ll][8] / ctfs[ll][0] <= cterr[0]):
# acceptable defocus error
ctfp = ctfs[ll][:8]
if (ctfs[ll][10] > cterr[1]):
# error of astigmatism exceed the threshold, set astigmatism to zero.
ctfp[6] = 0.0
ctfp[7] = 0.0
gctfp.append(ctfp)
groupid.append(kk)
break
if (len(groupid) > 0):
write_text_row(groupid, grpfile)
write_text_row(gctfp, ctfpfile)
cmd = "{} {} {} {}".format(
'e2bdb.py', fil, '--makevstack=bdb:' + root + 'G' + name,
'--list=' + grpfile)
#print cmd
subprocess.call(cmd, shell=True)
cmd = "{} {} {} {}".format('sxheader.py',
'bdb:' + root + 'G' + name,
'--params=ctf',
'--import=' + ctfpfile)
#print cmd
subprocess.call(cmd, shell=True)
else:
print ' >>> Group ', name, ' skipped.'
cmd = "{} {} {}".format("rm -f", grpfile, ctfpfile)
subprocess.call(cmd, shell=True)
elif options.scale > 0.0:
from utilities import read_text_row, write_text_row
scale = options.scale
nargs = len(args)
if nargs != 2:
print "Please provide names of input and output file!"
return
p = read_text_row(args[0])
for i in xrange(len(p)):
p[i][3] /= scale
p[i][4] /= scale
write_text_row(p, args[1])
elif options.adaptive_mask:
from utilities import get_im
from morphology import adaptive_mask
nsigma = options.nsigma
ndilation = options.ndilation
kernel_size = options.kernel_size
gauss_standard_dev = options.gauss_standard_dev
nargs = len(args)
if nargs > 2:
print "Too many inputs are given, try again!"
return
else:
inputvol = get_im(args[0])
input_path, input_file_name = os.path.split(args[0])
input_file_name_root, ext = os.path.splitext(input_file_name)
if nargs == 2: mask_file_name = args[1]
else:
mask_file_name = "adaptive_mask_for" + input_file_name_root + ".hdf" # Only hdf file is output.
mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size,
gauss_standard_dev)
mask3d.write_image(mask_file_name)
else:
ERROR("Please provide option name", "sxprocess.py", 1)
