def generate_helimic(refvol, outdir, pixel, CTF=False, Cs=2.0,voltage = 200.0, ampcont = 10.0, nonoise = False, rand_seed=14567): from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im from random import random from projection import prgs, prep_vol from filter import filt_gaussl, filt_ctf from EMAN2 import EMAN2Ctf if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "sxhelical_demo", 1) os.mkdir(outdir) seed(rand_seed) Util.set_randnum_seed(rand_seed) angles =[] for i in xrange(3): angles.append( [0.0+60.0*i, 90.0-i*5, 0.0, 0.0, 0.0] ) nangle = len(angles) volfts = get_im(refvol) nx = volfts.get_xsize() ny = volfts.get_ysize() nz = volfts.get_zsize() volfts, kbx, kby, kbz = prep_vol( volfts ) iprj = 0 width = 500 xstart = 0 ystart = 0 for idef in xrange(3,6): mic = model_blank(2048, 2048) #defocus = idef*0.2 defocus = idef*0.6 ##@ming if CTF : #ctf = EMAN2Ctf() #ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} ) from utilities import generate_ctf ctf = generate_ctf([defocus,2,200,1.84,0.0,ampcont,defocus*0.2,80]) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice. i = idef - 4 for k in xrange(1): psi = 90 + 10*i proj = prgs(volfts, kbz, [angles[idef-3][0], angles[idef-3][1], psi, 0.0, 0.0], kbx, kby) proj = Util.window(proj, 320, nz) mic += pad(proj, 2048, 2048, 1, 0.0, 750*i, 20*i, 0) if not nonoise: mic += model_gauss_noise(30.0,2048,2048) if CTF : #apply CTF mic = filt_ctf(mic, ctf) if not nonoise: mic += filt_gaussl(model_gauss_noise(17.5,2048,2048), 0.3) mic.write_image("%s/mic%1d.hdf"%(outdir, idef-3),0)
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + """ Input Output [options] Generate three micrographs, each micrograph contains one projection of a long filament. Input: Reference Volume, output directory Output: Three micrographs stored in output directory sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84 Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200 sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35 Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60 Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams Generate run through example script for helicon sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35 """ parser = OptionParser(usage, version=SPARXVERSION) # helicise the Atom coordinates # generate micrographs of helical filament parser.add_option( "--generate_micrograph", action="store_true", default=False, help= "Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory" ) parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction") parser.add_option("--apix", type="float", default=-1, help="pixel size in Angstroms") parser.add_option( "--rand_seed", type="int", default=14567, help= "the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs." ) parser.add_option("--Cs", type="float", default=2.0, help="Microscope Cs (spherical aberation)") parser.add_option("--voltage", type="float", default=200.0, help="Microscope voltage in KV") parser.add_option("--ac", type="float", default=10.0, help="Amplitude contrast (percentage, default=10)") parser.add_option("--nonoise", action="store_true", default=False, help="Do not add noise to the micrograph.") # generate initial volume parser.add_option("--generate_noisycyl", action="store_true", default=False, help="Generate initial volume of noisy cylinder.") parser.add_option( "--boxsize", type="string", default="100,100,200", help= "String containing x , y, z dimensions (separated by comma) in pixels") parser.add_option("--rad", type="int", default=35, help="Radius of initial volume in pixels") # generate 2D mask parser.add_option("--generate_mask", action="store_true", default=False, help="Generate 2D rectangular mask.") parser.add_option( "--masksize", type="string", default="200,200", help= "String containing x and y dimensions (separated by comma) in pixels") parser.add_option("--maskwidth", type="int", default=60, help="Width of rectangular mask") # Apply 2D alignment parameters to input stack and output new images to output stack parser.add_option( "--applyparams", action="store_true", default=False, help= "Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack" ) # Generate run script parser.add_option("--generate_script", action="store_true", default=False, help="Generate script for helicon run through example") parser.add_option("--filename", type="string", default="runhelicon", help="Name of run script to generate") parser.add_option("--seg_ny", type="int", default=180, help="y-dimension of segment used for refinement") parser.add_option( "--ptcl_dist", type="int", default=15, help= "Distance in pixels between adjacent segments windowed from same filament" ) parser.add_option( "--fract", type="float", default=0.35, help="Fraction of the volume used for applying helical symmetry.") (options, args) = parser.parse_args() if len(args) > 3: print("usage: " + usage) print("Please run '" + progname + " -h' for detailed options") else: if options.generate_script: generate_runscript(options.filename, options.seg_ny, options.ptcl_dist, options.fract) if options.generate_micrograph: if options.apix <= 0: print("Please enter pixel size.") sys.exit() generate_helimic(args[0], args[1], options.apix, options.CTF, options.Cs, options.voltage, options.ac, options.nonoise, options.rand_seed) if options.generate_noisycyl: from utilities import model_cylinder, model_gauss_noise outvol = args[0] boxdims = options.boxsize.split(',') if len(boxdims) < 1 or len(boxdims) > 3: print( "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'" ) sys.exit() nx = int(boxdims[0]) if len(boxdims) == 1: ny = nx nz = nx else: ny = int(boxdims[1]) if len(boxdims) == 3: nz = int(boxdims[2]) (model_cylinder(options.rad, nx, ny, nz) * model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol) if options.generate_mask: from utilities import model_blank, pad outvol = args[0] maskdims = options.masksize.split(',') if len(maskdims) < 1 or len(maskdims) > 2: print( "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'" ) sys.exit() nx = int(maskdims[0]) if len(maskdims) == 1: ny = nx else: ny = int(maskdims[1]) mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1, 0.0) mask.write_image(outvol) if options.applyparams: from utilities import get_im, get_params2D, set_params2D from fundamentals import cyclic_shift stack = args[0] newstack = args[1] mask = get_im(args[2]) nima = EMUtil.get_image_count(stack) for im in range(nima): prj = get_im(stack, im) alpha, sx, sy, mirror, scale = get_params2D(prj) prj = cyclic_shift(prj, int(sx)) set_params2D(prj, [0.0, 0., 0.0, 0, 1]) stat = Util.infomask(prj, mask, False) prj = old_div((prj - stat[0]), stat[1]) ctf_params = prj.get_attr("ctf") prj.set_attr('ctf_applied', 0) prj.write_image(newstack, im)
def generate_helimic(refvol, outdir, pixel, CTF=False, Cs=2.0, voltage=200.0, ampcont=10.0, nonoise=False, rand_seed=14567): from utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im from random import random from projection import prgs, prep_vol from filter import filt_gaussl, filt_ctf from EMAN2 import EMAN2Ctf if os.path.exists(outdir): ERROR( 'Output directory exists, please change the name and restart the program', "sxhelical_demo", 1) os.mkdir(outdir) seed(rand_seed) Util.set_randnum_seed(rand_seed) angles = [] for i in range(3): angles.append([0.0 + 60.0 * i, 90.0 - i * 5, 0.0, 0.0, 0.0]) nangle = len(angles) volfts = get_im(refvol) nx = volfts.get_xsize() ny = volfts.get_ysize() nz = volfts.get_zsize() volfts, kbx, kby, kbz = prep_vol(volfts) iprj = 0 width = 500 xstart = 0 ystart = 0 for idef in range(3, 6): mic = model_blank(2048, 2048) #defocus = idef*0.2 defocus = idef * 0.6 ##@ming if CTF: #ctf = EMAN2Ctf() #ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} ) from utilities import generate_ctf ctf = generate_ctf( [defocus, 2, 200, 1.84, 0.0, ampcont, defocus * 0.2, 80] ) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice. i = idef - 4 for k in range(1): psi = 90 + 10 * i proj = prgs( volfts, kbz, [angles[idef - 3][0], angles[idef - 3][1], psi, 0.0, 0.0], kbx, kby) proj = Util.window(proj, 320, nz) mic += pad(proj, 2048, 2048, 1, 0.0, 750 * i, 20 * i, 0) if not nonoise: mic += model_gauss_noise(30.0, 2048, 2048) if CTF: #apply CTF mic = filt_ctf(mic, ctf) if not nonoise: mic += filt_gaussl(model_gauss_noise(17.5, 2048, 2048), 0.3) mic.write_image("%s/mic%1d.hdf" % (outdir, idef - 3), 0)
def main(): from utilities import write_text_row, drop_image, model_gauss_noise, get_im, set_params_proj, wrap_mpi_bcast, model_circle import user_functions from applications import MPI_start_end from optparse import OptionParser from global_def import SPARXVERSION from EMAN2 import EMData from multi_shc import multi_shc, do_volume from logger import Logger, BaseLogger_Files import sys import os import time import socket progname = os.path.basename(sys.argv[0]) usage = progname + " stack [output_directory] initial_volume --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --yr=y_range --ts=translational_search_step --delta=angular_step --an=angular_neighborhood --center=center_type --fl --aa --ref_a=S --sym=c1" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--ir", type= "int", default= 1, help="inner radius for rotational correlation > 0 (set to 1)") parser.add_option("--ou", type= "int", default= -1, help="outer radius for rotational correlation < int(nx/2)-1 (set to the radius of the particle)") parser.add_option("--rs", type= "int", default= 1, help="step between rings in rotational correlation >0 (set to 1)" ) parser.add_option("--xr", type="string", default= "-1", help="range for translation search in x direction, search is +/xr (default 0)") parser.add_option("--yr", type="string", default= "-1", help="range for translation search in y direction, search is +/yr (default = same as xr)") parser.add_option("--ts", type="string", default= "1", help="step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional") parser.add_option("--delta", type="string", default= "-1", help="angular step of reference projections during initialization step (default automatically selected based on radius of the structure.)") #parser.add_option("--an", type="string", default= "-1", help="angular neighborhood for local searches (phi and theta)") parser.add_option("--center", type="float", default= -1, help="-1: average shift method; 0: no centering; 1: center of gravity (default=-1)") parser.add_option("--maxit", type="int", default= 400, help="maximum number of iterations performed for the GA part (set to 400) ") parser.add_option("--outlier_percentile",type="float", default= 95, help="percentile above which outliers are removed every iteration") parser.add_option("--iteration_start",type="int", default= 0, help="starting iteration for rviper, 0 means go to the most recent one (default).") parser.add_option("--CTF", action="store_true", default=False, help="Use CTF (Default no CTF correction)") parser.add_option("--snr", type="float", default= 1.0, help="Signal-to-Noise Ratio of the data (default 1.0)") parser.add_option("--ref_a", type="string", default= "S", help="method for generating the quasi-uniformly distributed projection directions (default S)") parser.add_option("--sym", type="string", default= "c1", help="symmetry of the refined structure") parser.add_option("--npad", type="int", default= 2, help="padding size for 3D reconstruction (default=2)") parser.add_option("--startangles", action="store_true", default=False, help="Use orientation parameters in the input file header to jumpstart the procedure") #options introduced for the do_volume function parser.add_option("--fl", type="float", default=0.12, help="cut-off frequency of hyperbolic tangent low-pass Fourier filte (default 0.12)") parser.add_option("--aa", type="float", default=0.1, help="fall-off of hyperbolic tangent low-pass Fourier filter (default 0.1)") parser.add_option("--pwreference", type="string", default="", help="text file with a reference power spectrum (default no power spectrum adjustment)") parser.add_option("--mask3D", type="string", default=None, help="3D mask file (default a sphere WHAT RADIUS??)") (options, args) = parser.parse_args(sys.argv[1:]) #print( " args ",args) if( len(args) == 3): volinit = args[2] masterdir = args[1] elif(len(args) == 2): volinit = args[1] masterdir = "" else: print( "usage: " + usage) print( "Please run '" + progname + " -h' for detailed options") return 1 orgstack = args[0] #print( orgstack,masterdir,volinit ) # INPUT PARAMETERS radi = options.ou global_def.BATCH = True ali3d_options.ir = options.ir ali3d_options.rs = options.rs ali3d_options.ou = options.ou ali3d_options.xr = options.xr ali3d_options.yr = options.yr ali3d_options.ts = options.ts ali3d_options.an = "-1" ali3d_options.sym = options.sym ali3d_options.delta = options.delta ali3d_options.npad = options.npad ali3d_options.center = options.center ali3d_options.CTF = options.CTF ali3d_options.ref_a = options.ref_a ali3d_options.snr = options.snr ali3d_options.mask3D = options.mask3D ali3d_options.pwreference = options.pwreference ali3d_options.fl = 0.4 ali3d_options.aa = 0.1 if( ali3d_options.xr == "-1" ): ali3d_options.xr = "2" """ print( options) print( 'ali3d_options', ali3d_options.ir ,\ ali3d_options.rs ,\ ali3d_options.ou ,\ ali3d_options.xr ,\ ali3d_options.yr ,\ ali3d_options.ts ,\ ali3d_options.an ,\ ali3d_options.sym ,\ ali3d_options.delta ,\ ali3d_options.npad ,\ ali3d_options.center ,\ ali3d_options.CTF ,\ ali3d_options.ref_a ,\ ali3d_options.snr ,\ ali3d_options.mask3D ,\ ali3d_options.fl ,\ ali3d_options.aa \ ) #exit() """ mpi_init(0, []) nproc = mpi_comm_size(MPI_COMM_WORLD) myid = mpi_comm_rank(MPI_COMM_WORLD) main_node = 0 #mpi_finalize() #exit() nxinit = -1 #int(280*0.3*2) nsoft = 0 mempernode = 4.0e9 # PARAMETERS OF THE PROCEDURE # threshold error thresherr = 0 fq = 0.11 # low-freq limit to which fuse ref volumes. Should it be estimated somehow? # Get the pixel size, if none set to 1.0, and the original image size if(myid == main_node): a = get_im(orgstack) nnxo = a.get_xsize() if ali3d_options.CTF: i = a.get_attr('ctf') pixel_size = i.apix else: pixel_size = 1.0 del a else: nnxo = 0 pixel_size = 1.0 pixel_size = bcast_number_to_all(pixel_size, source_node = main_node) nnxo = bcast_number_to_all(nnxo, source_node = main_node) if(radi < 1): radi = nnxo//2-2 elif((2*radi+2)>nnxo): ERROR("HERE","particle radius set too large!",1) ali3d_options.ou = radi if(nxinit < 0): nxinit = min(32, nnxo) nxshrink = nxinit minshrink = 32.0/float(nnxo) shrink = max(float(nxshrink)/float(nnxo),minshrink) # MASTER DIRECTORY if(myid == main_node): print( " masterdir ",masterdir) if( masterdir == ""): timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime()) masterdir = "master"+timestring li = len(masterdir) cmd = "{} {}".format("mkdir", masterdir) cmdexecute(cmd) keepchecking = 0 else: li = 0 keepchecking = 1 else: li = 0 keepchecking = 1 li = mpi_bcast(li,1,MPI_INT,main_node,MPI_COMM_WORLD)[0] if( li > 0 ): masterdir = mpi_bcast(masterdir,li,MPI_CHAR,main_node,MPI_COMM_WORLD) masterdir = string.join(masterdir,"") # create a vstack from input stack to the local stack in masterdir # Stack name set to default stack = "bdb:"+masterdir+"/rdata" # Initialization of stacks if(myid == main_node): if keepchecking: if(os.path.exists(os.path.join(masterdir,"EMAN2DB/rdata.bdb"))): doit = False else: doit = True else: doit = True if doit: if(orgstack[:4] == "bdb:"): cmd = "{} {} {}".format("e2bdb.py", orgstack,"--makevstack="+stack) else: cmd = "{} {} {}".format("sxcpy.py", orgstack, stack) cmdexecute(cmd) cmd = "{} {}".format("sxheader.py --consecutive --params=originalid", stack) cmdexecute(cmd) keepchecking = False total_stack = EMUtil.get_image_count(stack) junk = get_im(stack) nnxo = junk.get_xsize() del junk else: total_stack = 0 nnxo = 0 total_stack = bcast_number_to_all(total_stack, source_node = main_node) nnxo = bcast_number_to_all(nnxo, source_node = main_node) # INITIALIZATION # Run exhaustive projection matching to get initial orientation parameters # Estimate initial resolution initdir = os.path.join(masterdir,"main000") # make sure the initial volume is not set to zero outside of a mask, as if it is it will crach the program if( myid == main_node and (not options.startangles)): viv = get_im(volinit) if(options.mask3D == None): mask33d = model_circle(radi,nnxo,nnxo,nnxo) else: mask33d = (options.mask3D).copy() st = Util.infomask(viv, mask33d, False) if( st[0] == 0.0 ): viv += (model_blank(nnxo,nnxo,nnxo,1.0) - mask33d)*model_gauss_noise(st[1]/1000.0,nnxo,nnxo,nnxo) viv.write_image(volinit) del mask33d, viv doit, keepchecking = checkstep(initdir, keepchecking, myid, main_node) if doit: partids = os.path.join(masterdir, "ids.txt") partstack = os.path.join(masterdir, "paramszero.txt") xr = min(8,(nnxo - (2*radi+1))//2) if(xr > 3): ts = "2" else: ts = "1" delta = int(options.delta) if(delta <= 0.0): delta = "%f"%round(degrees(atan(1.0/float(radi))), 2) paramsdict = { "stack":stack,"delta":"2.0", "ts":ts, "xr":"%f"%xr, "an":"-1", "center":options.center, "maxit":1, \ "currentres":0.4, "aa":0.1, "radius":radi, "nsoft":0, "delpreviousmax":True, "shrink":1.0, "saturatecrit":1.0, \ "refvol":volinit, "mask3D":options.mask3D} if(options.startangles): if( myid == main_node ): cmd = "mkdir "+initdir cmdexecute(cmd) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print(line,"INITIALIZATION") cmd = "{} {}".format("sxheader.py --params=xform.projection --export="+os.path.join(initdir,"params-chunk0.txt"), stack) cmdexecute(cmd) print(line,"Executed successfully: ","Imported initial parameters from the input stack") else: if( myid == main_node ): line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print(line,"INITIALIZATION") write_text_file(range(total_stack), partids) write_text_row([[0.0,0.0,0.0,0.0,0.0] for i in xrange(total_stack) ], partstack) metamove(paramsdict, partids, partstack, initdir, 0, myid, main_node, nproc) if(myid == main_node): print(line,"Executed successfully: ","initialization ali3d_base_MPI %d"%nsoft) # store params partids = [None]*2 for procid in xrange(2): partids[procid] = os.path.join(initdir,"chunk%01d.txt"%procid) partstack = [None]*2 for procid in xrange(2): partstack[procid] = os.path.join(initdir,"params-chunk%01d.txt"%procid) from random import shuffle if(myid == main_node): # split randomly params = read_text_row(os.path.join(initdir,"params-chunk0.txt")) assert(len(params) == total_stack) ll = range(total_stack) shuffle(ll) l1 = ll[:total_stack//2] l2 = ll[total_stack//2:] del ll l1.sort() l2.sort() write_text_file(l1,partids[0]) write_text_file(l2,partids[1]) write_text_row([params[i] for i in l1], partstack[0]) write_text_row([params[i] for i in l2], partstack[1]) del params, l1, l2 mpi_barrier(MPI_COMM_WORLD) # Now parallel vol = [None]*2 for procid in xrange(2): projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc) if ali3d_options.CTF: vol[procid] = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) else: vol[procid] = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) del projdata if( myid == main_node): vol[procid].write_image(os.path.join(initdir,"vol%01d.hdf"%procid) ) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print( line,"Generated inivol #%01d "%procid) if(myid == main_node): currentres = get_resolution(vol, radi, nnxo, initdir) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print( line,"Initial resolution %6.4f"%currentres) write_text_file([currentres],os.path.join(initdir,"current_resolution.txt")) else: currentres = 0.0 currentres = bcast_number_to_all(currentres, source_node = main_node) else: if(myid == main_node): currentres = read_text_file(os.path.join(initdir,"current_resolution.txt"))[0] else: currentres = 0.0 currentres = bcast_number_to_all(currentres, source_node = main_node) # set for the first iteration nxshrink = min(max(32, int((currentres+paramsdict["aa"]/2.)*2*nnxo + 0.5)), nnxo) shrink = float(nxshrink)/nnxo tracker = {"previous-resolution":currentres, "movedup":False,"eliminated-outliers":False,\ "previous-nx":nxshrink, "previous-shrink":shrink, "extension":0, "bestsolution":0} previousoutputdir = initdir # MAIN ITERATION mainiteration = 0 keepgoing = 1 while(keepgoing): mainiteration += 1 # prepare output directory mainoutputdir = os.path.join(masterdir,"main%03d"%mainiteration) if(myid == main_node): line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print(line,"MAIN ITERATION #",mainiteration, shrink, nxshrink) if keepchecking: if(os.path.exists(mainoutputdir)): doit = 0 print("Directory ",mainoutputdir," exists!") else: doit = 1 keepchecking = False else: doit = 1 if doit: cmd = "{} {}".format("mkdir", mainoutputdir) cmdexecute(cmd) # prepare names of input file names, they are in main directory, # log subdirectories contain outputs from specific refinements partids = [None]*2 for procid in xrange(2): partids[procid] = os.path.join(previousoutputdir,"chunk%01d.txt"%procid) partstack = [None]*2 for procid in xrange(2): partstack[procid] = os.path.join(previousoutputdir,"params-chunk%01d.txt"%procid) mpi_barrier(MPI_COMM_WORLD) #mpi_finalize() #exit() #print("RACING A ",myid) outvol = [os.path.join(previousoutputdir,"vol%01d.hdf"%procid) for procid in xrange(2)] for procid in xrange(2): doit, keepchecking = checkstep(outvol[procid], keepchecking, myid, main_node) if doit: from multi_shc import do_volume projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc) if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) del projdata if( myid == main_node): vol.write_image(outvol[procid]) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print( line,"Generated inivol #%01d "%procid) del vol if(myid == main_node): if keepchecking: procid = 1 if(os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid)): doit = 0 else: doit = 1 keepchecking = False else: doit = 1 if doit: vol = [get_im(outvol[procid]) for procid in xrange(2) ] fq = 0.11 # which part to fuse fuselowf(vol, fq) for procid in xrange(2): vol[procid].write_image(os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid) ) del vol else: doit = 0 mpi_barrier(MPI_COMM_WORLD) doit = bcast_number_to_all(doit, source_node = main_node) # Refine two groups at a current resolution lastring = int(shrink*radi + 0.5) if(lastring < 2): print( line,"ERROR!! lastring too small ", radi, shrink, lastring) break # REFINEMENT # Part "a" SHC for procid in xrange(2): coutdir = os.path.join(mainoutputdir,"loga%01d"%procid) doit, keepchecking = checkstep(coutdir , keepchecking, myid, main_node) paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":1500, \ "currentres":currentres, "aa":0.1, "radius":radi, "nsoft":1, "saturatecrit":0.75, "delpreviousmax":True, "shrink":shrink, \ "refvol":os.path.join(mainoutputdir,"fusevol%01d.hdf"%procid),"mask3D":options.mask3D } if doit: metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc) partstack = [None]*2 for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir, "loga%01d"%procid, "params-chunk%01d.txt"%procid) for procid in xrange(2): outvol = os.path.join(mainoutputdir,"loga%01d"%procid,"shcvol%01d.hdf"%procid) doit, keepchecking = checkstep(outvol, keepchecking, myid, main_node) if doit: from multi_shc import do_volume projdata = getindexdata(stack, partids[procid], partstack[procid], myid, nproc) if ali3d_options.CTF: vol = recons3d_4nn_ctf_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) else: vol = recons3d_4nn_MPI(myid, projdata, symmetry=ali3d_options.sym, npad = 2) del projdata if( myid == main_node): vol.write_image(outvol) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print( line,"Generated shcvol #%01d "%procid) del vol if(myid == main_node): if keepchecking: procid = 1 if(os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid) ): doit = 0 else: doit = 1 keepchecking = False else: doit = 1 if doit: vol = [] for procid in xrange(2): vol.append(get_im(os.path.join(mainoutputdir,"loga%01d"%procid,"shcvol%01d.hdf"%procid) )) fq = 0.11 # which part to fuse fuselowf(vol, fq) for procid in xrange(2): vol[procid].write_image( os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid) ) del vol else: doit = 0 mpi_barrier(MPI_COMM_WORLD) doit = bcast_number_to_all(doit, source_node = main_node) # Part "b" deterministic partstack = [None]*2 for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir,"loga%01d"%procid,"params-chunk%01d.txt"%procid) for procid in xrange(2): coutdir = os.path.join(mainoutputdir,"logb%01d"%procid) doit, keepchecking = checkstep(coutdir, keepchecking, myid, main_node) # Run exhaustive to finish up matching paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":10, \ "currentres":currentres, "aa":0.1, "radius":radi, "nsoft":0, "saturatecrit":0.95, "delpreviousmax":True, "shrink":shrink, \ "refvol":os.path.join(mainoutputdir,"loga%01d"%procid,"fusevol%01d.hdf"%procid), "mask3D":options.mask3D } if doit: metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc) partstack = [None]*2 for procid in xrange(2): partstack[procid] = os.path.join(mainoutputdir,"logb%01d"%procid,"params-chunk%01d.txt"%procid) # Compute current resolution, store result in main directory doit, keepchecking = checkstep(os.path.join(mainoutputdir,"current_resolution.txt"), keepchecking, myid, main_node) newres = 0.0 if doit: newres = compute_resolution(stack, mainoutputdir, partids, partstack, radi, nnxo, ali3d_options.CTF, myid, main_node, nproc) else: if(myid == main_node): newres = read_text_file( os.path.join(mainoutputdir,"current_resolution.txt") )[0] newres = bcast_number_to_all(newres, source_node = main_node) # Here I have code to generate presentable results. IDs and params have to be merged and stored and an overall volume computed. doit, keepchecking = checkstep(os.path.join(mainoutputdir,"volf.hdf"), keepchecking, myid, main_node) if doit: if( myid == main_node ): pinids = map(int, read_text_file(partids[0]) ) + map(int, read_text_file(partids[1]) ) params = read_text_row(partstack[0]) + read_text_row(partstack[1]) assert(len(pinids) == len(params)) for i in xrange(len(pinids)): pinids[i] = [ pinids[i], params[i] ] del params pinids.sort() write_text_file([pinids[i][0] for i in xrange(len(pinids))], os.path.join(mainoutputdir,"indexes.txt")) write_text_row( [pinids[i][1] for i in xrange(len(pinids))], os.path.join(mainoutputdir,"params.txt")) mpi_barrier(MPI_COMM_WORLD) ali3d_options.fl = newres ali3d_options.ou = radi projdata = getindexdata(stack, os.path.join(mainoutputdir,"indexes.txt"), os.path.join(mainoutputdir,"params.txt"), myid, nproc) volf = do_volume(projdata, ali3d_options, mainiteration, mpi_comm = MPI_COMM_WORLD) if(myid == main_node): volf.write_image(os.path.join(mainoutputdir,"volf.hdf")) mpi_barrier(MPI_COMM_WORLD) #print("RACING X ",myid) if(newres == currentres): for procid in xrange(2): coutdir = os.path.join(mainoutputdir,"logc%01d"%procid) doit, keepchecking = checkstep(coutdir, keepchecking, myid, main_node) if doit: # Do cross-check of the results paramsdict = { "stack":stack,"delta":"%f"%round(degrees(atan(1.0/lastring)), 2) , "ts":"1", "xr":"2", "an":"-1", "center":options.center, "maxit":1, \ "currentres":newres, "aa":0.1, "radius":radi, "nsoft":0, "saturatecrit":0.95, "delpreviousmax":True, "shrink":shrink, \ "refvol":os.path.join(mainoutputdir,"vol%01d.hdf"%(1-procid)), "mask3D":options.mask3D } # The cross-check uses parameters from step "b" to make sure shifts are correct. # As the check is exhaustive, angles are ignored metamove(paramsdict, partids[procid], partstack[procid], coutdir, procid, myid, main_node, nproc) # identify bad apples doit, keepchecking = checkstep(os.path.join(mainoutputdir,"badapples.txt"), keepchecking, myid, main_node) if doit: if(myid == main_node): from utilities import get_symt from pixel_error import max_3D_pixel_error ts = get_symt(ali3d_options.sym) badapples = [] deltaerror = 2.0 total_images_now = 0 for procid in xrange(2): bad = [] ids = map(int,read_text_file( partids[procid] )) total_images_now += len(ids) oldp = read_text_row(partstack[procid]) newp = read_text_row(os.path.join(mainoutputdir,"logc%01d"%procid,"params-chunk%01d.txt"%procid)) for i in xrange(len(ids)): t1 = Transform({"type":"spider","phi":oldp[i][0],"theta":oldp[i][1],"psi":oldp[i][2]}) t1.set_trans(Vec2f(-oldp[i][3]*shrink, -oldp[i][4]*shrink)) t2 = Transform({"type":"spider","phi":newp[i][0],"theta":newp[i][1],"psi":newp[i][2]}) t2.set_trans(Vec2f(-newp[i][3]*shrink, -newp[i][4]*shrink)) if(len(ts) > 1): # only do it if it is not c1 pixel_error = +1.0e23 for kts in ts: ut = t2*kts # we do not care which position minimizes the error pixel_error = min(max_3D_pixel_error(t1, ut, lastring), pixel_error) else: pixel_error = max_3D_pixel_error(t1, t2, lastring) if(pixel_error > deltaerror): bad.append(i) if(len(bad)>0): badapples += [ids[bad[i]] for i in xrange(len(bad))] for i in xrange(len(bad)-1,-1,-1): del oldp[bad[i]],ids[bad[i]] if(len(ids) == 0): ERROR("sxpetite","program diverged, all images have large angular errors, most likely the initial model is badly off",1) else: # This generate new parameters, hopefully to be used as starting ones in the new iteration write_text_file(ids,os.path.join(mainoutputdir,"chunk%01d.txt"%procid)) write_text_row(oldp,os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid)) if(len(badapples)>0): badapples.sort() write_text_file(badapples,os.path.join(mainoutputdir,"badapples.txt")) eli = 100*float(len(badapples))/float(total_images_now) line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>" print(line,"Elimination of outliers: %5.1f percent"%eli ) else: eli = 0.0 del badapples, oldp,ids,bad,newp,ts else: eli =0.0 eli = bcast_number_to_all(eli, source_node = main_node) # This part under MPI if(eli > 0.0): # Compute current resolution depres = compute_resolution(stack, mainoutputdir, \ [os.path.join(mainoutputdir,"chunk%01d.txt"%procid) for procid in xrange(2)], \ [os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid) for procid in xrange(2)], \ radi, nnxo, ali3d_options.CTF, myid, main_node, nproc) depres = bcast_number_to_all(depres, source_node = main_node) if(depres < newres): # elimination of outliers decreased resolution, ignore the effort eliminated_outliers = False else: eliminated_outliers = True newres = depres """ # It does not seem to be needed, as data is there, we just point to the directory for procid in xrange(2): # set pointers to current parameters in main, which are for the reduced set stored above partids[procid] = os.path.join(mainoutputdir,"chunk%01d.txt"%procid partstack[procid] = os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid) """ else: eliminated_outliers = False else: eliminated_outliers = False if(myid == main_node and not eliminated_outliers): for procid in xrange(2): # This is standard path, copy parameters to be used to the main cmd = "{} {} {}".format("cp -p ", partids[procid] , os.path.join(mainoutputdir,"chunk%01d.txt"%procid)) cmdexecute(cmd) cmd = "{} {} {}".format("cp -p ", partstack[procid], os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid)) cmdexecute(cmd) keepgoing = 0 if( newres > currentres or (eliminated_outliers and not tracker["eliminated-outliers"])): if(myid == main_node): print(" Resolution improved, full steam ahead!") if( newres > currentres ): tracker["movedup"] = True else: tracker["movedup"] = False shrink = max(min(2*newres + paramsdict["aa"], 1.0),minshrink) tracker["extension"] = 4 nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo) tracker["previous-resolution"] = newres currentres = newres tracker["bestsolution"] = mainiteration bestoutputdir = mainoutputdir tracker["eliminated-outliers"] = eliminated_outliers keepgoing = 1 elif(newres < currentres): if(not tracker["movedup"] and tracker["extension"] < 2 and mainiteration > 1): keepgoing = 0 if(myid == main_node): print(" Cannot improve resolution, the best result is in the directory main%03d"%tracker["bestsolution"]) else: if(not tracker["movedup"] and tracker["extension"] > 1 and mainiteration > 1): if(myid == main_node): print(" Resolution decreased. Will decrease target resolution and will fall back on the best so far: main%03d"%tracker["bestsolution"]) bestoutputdir = os.path.join(masterdir,"main%03d"%tracker["bestsolution"]) elif( tracker["movedup"] and tracker["extension"] > 1 and mainiteration > 1): if(myid == main_node): print(" Resolution decreased. Will decrease target resolution and will try starting from previous stage: main%03d"%(mainiteration - 1)) bestoutputdir = os.path.join(masterdir,"main%03d"%(mainiteration-1)) elif( mainiteration == 1): if(myid == main_node): print(" Resolution decreased in the first iteration. It is expected, not to worry") bestoutputdir = mainoutputdir tracker["extension"] += 1 else: # missing something here? if(myid == main_node): print(" Should not be here, ERROR 175!") break mpi_finalize() exit() if( bestoutputdir != mainoutputdir ): # This is the key, we just reset the main to previous, so it will be eventually used as a starting in the next iteration mainoutputdir = bestoutputdir """ # Set data from the main previous best to the current. for procid in xrange(2): partids[procid] = os.path.join(bestoutputdir,"chunk%01d.txt"%procid) partstack[procid] = os.path.join(bestoutputdir,"params-chunk%01d.txt"%procid) """ if(myid == main_node): currentres = read_text_file( os.path.join(bestoutputdir,"current_resolution.txt") )[0] currentres = bcast_number_to_all(currentres, source_node = main_node) shrink = max(min(2*currentres + paramsdict["aa"], 1.0), minshrink) tracker["extension"] -= 1 nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo) tracker["previous-resolution"] = newres tracker["eliminated-outliers"] = eliminated_outliers tracker["movedup"] = False keepgoing = 1 elif(newres == currentres): if( tracker["extension"] > 0 ): if(myid == main_node): print("The resolution did not improve. This is look ahead move. Let's try to relax slightly and hope for the best") tracker["extension"] -= 1 tracker["movedup"] = False shrink = max(min(2*currentres + paramsdict["aa"], 1.0), minshrink) nxshrink = min(int(nnxo*shrink + 0.5) + tracker["extension"],nnxo) if( tracker["previous-nx"] == nnxo ): keepgoing = 0 else: tracker["previous-resolution"] = newres currentres = newres tracker["eliminated-outliers"] = eliminated_outliers tracker["movedup"] = False keepgoing = 1 else: if(myid == main_node): print("The resolution did not improve.") keepgoing = 0 if( keepgoing == 1 ): if(myid == main_node): print(" New shrink and image dimension :",shrink,nxshrink) """ # It does not look like it is necessary, we just have to point to the directory as the files should be there. # Will continue, so update the params files for procid in xrange(2): # partids ads partstack contain parameters to be used as starting in the next iteration if(not os.path.exists(os.path.join(mainoutputdir,"chunk%01d.txt"%procid))): cmd = "{} {} {}".format("cp -p ", partids[procid] , os.path.join(mainoutputdir,"chunk%01d.txt"%procid)) cmdexecute(cmd) if(not os.path.exists(os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid))): cmd = "{} {} {}".format("cp -p ", partstack[procid], os.path.join(mainoutputdir,"params-chunk%01d.txt"%procid)) cmdexecute(cmd) """ previousoutputdir = mainoutputdir tracker["previous-shrink"] = shrink tracker["previous-nx"] = nxshrink else: if(myid == main_node): print(" Terminating, the best solution is in the directory main%03d"%tracker["bestsolution"]) mpi_barrier(MPI_COMM_WORLD) mpi_finalize()
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + """ Input Output [options] Generate three micrographs, each micrograph contains one projection of a long filament. Input: Reference Volume, output directory Output: Three micrographs stored in output directory sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84 Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200 sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35 Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60 Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams Generate run through example script for helicon sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35 """ parser = OptionParser(usage,version=SPARXVERSION) # helicise the Atom coordinates # generate micrographs of helical filament parser.add_option("--generate_micrograph", action="store_true", default=False, help="Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory") parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction") parser.add_option("--apix", type="float", default= -1, help="pixel size in Angstroms") parser.add_option("--rand_seed", type="int", default=14567, help="the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs.") parser.add_option("--Cs", type="float", default= 2.0, help="Microscope Cs (spherical aberation)") parser.add_option("--voltage", type="float", default=200.0, help="Microscope voltage in KV") parser.add_option("--ac", type="float", default=10.0, help="Amplitude contrast (percentage, default=10)") parser.add_option("--nonoise", action="store_true", default=False, help="Do not add noise to the micrograph.") # generate initial volume parser.add_option("--generate_noisycyl", action="store_true", default=False, help="Generate initial volume of noisy cylinder.") parser.add_option("--boxsize", type="string", default="100,100,200", help="String containing x , y, z dimensions (separated by comma) in pixels") parser.add_option("--rad", type="int", default=35, help="Radius of initial volume in pixels") # generate 2D mask parser.add_option("--generate_mask", action="store_true", default=False, help="Generate 2D rectangular mask.") parser.add_option("--masksize", type="string", default="200,200", help="String containing x and y dimensions (separated by comma) in pixels") parser.add_option("--maskwidth", type="int", default=60, help="Width of rectangular mask") # Apply 2D alignment parameters to input stack and output new images to output stack parser.add_option("--applyparams", action="store_true", default=False, help="Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack") # Generate run script parser.add_option("--generate_script", action="store_true", default=False, help="Generate script for helicon run through example") parser.add_option("--filename", type="string", default="runhelicon", help="Name of run script to generate") parser.add_option("--seg_ny", type="int", default=180, help="y-dimension of segment used for refinement") parser.add_option("--ptcl_dist", type="int", default=15, help="Distance in pixels between adjacent segments windowed from same filament") parser.add_option("--fract", type="float", default=0.35, help="Fraction of the volume used for applying helical symmetry.") (options, args) = parser.parse_args() if len(args) > 3: print "usage: " + usage print "Please run '" + progname + " -h' for detailed options" else: if options.generate_script: generate_runscript(options.filename, options.seg_ny, options.ptcl_dist, options.fract) if options.generate_micrograph: if options.apix <= 0: print "Please enter pixel size." sys.exit() generate_helimic(args[0], args[1], options.apix, options.CTF, options.Cs, options.voltage, options.ac, options.nonoise, options.rand_seed) if options.generate_noisycyl: from utilities import model_cylinder, model_gauss_noise outvol = args[0] boxdims = options.boxsize.split(',') if len(boxdims) < 1 or len(boxdims) > 3: print "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize='100,100,200'" sys.exit() nx= int(boxdims[0]) if len(boxdims) == 1: ny = nx nz = nx else: ny = int(boxdims[1]) if len(boxdims) == 3: nz = int(boxdims[2]) (model_cylinder(options.rad,nx, ny, nz)*model_gauss_noise(1.0, nx, ny, nz) ).write_image(outvol) if options.generate_mask: from utilities import model_blank, pad outvol = args[0] maskdims = options.masksize.split(',') if len(maskdims) < 1 or len(maskdims) > 2: print "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize='200,200'" sys.exit() nx= int(maskdims[0]) if len(maskdims) == 1: ny = nx else: ny = int(maskdims[1]) mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1, 0.0) mask.write_image(outvol) if options.applyparams: from utilities import get_im, get_params2D, set_params2D from fundamentals import cyclic_shift stack = args[0] newstack = args[1] mask = get_im(args[2]) nima = EMUtil.get_image_count(stack) for im in xrange(nima): prj = get_im(stack,im) alpha, sx, sy, mirror, scale = get_params2D(prj) prj = cyclic_shift(prj, int(sx)) set_params2D(prj, [0.0,0.,0.0,0,1]) stat = Util.infomask(prj , mask, False ) prj= (prj-stat[0])/stat[1] ctf_params = prj.get_attr("ctf") prj.set_attr('ctf_applied', 0) prj.write_image(newstack, im)
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Generic 2-D image processing programs. Functionality: 1. Phase flip a stack of images and write output to new file: sxprocess.py input_stack.hdf output_stack.hdf --phase_flip 2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size. The window size will change accordingly. sxprocess input.hdf output.hdf --changesize --ratio=0.5 3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes or a 3-D volume. sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024] 4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs: sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64 5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3): sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid 6. Retrieve original image numbers of images listed in ISAC output stack of averages: sxprocess.py select1.hdf ohk.txt 7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file. Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack. sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw 8. Generate a 1D rotationally averaged power spectrum of an image. sxprocess.py vol.hdf --rotwp=rotpw.txt # Output will contain three columns: (1) rotationally averaged power spectrum (2) logarithm of the rotationally averaged power spectrum (3) integer line number (from zero to approximately to half the image size) 9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data. sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt The output file is then imported and 3D transformed volume computed: sxheader.py bdb:p --params=xform.projection --import=output.txt mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI The reconstructed volume is in the position of the volume computed using the input.txt parameters and then transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz) 10. Import ctf parameters from the output of sxcter into windowed particle headers. There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph. In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'. Normally this is done by e2boxer.py during windowing. Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created. sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 # Output will be a vritual stack bdb:Gdata sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 To concatenate output files: cd directory e2bdb.py . --makevstack=bdb:allparticles --filt=G IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them. 11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor sxprocess.py orientationparams.txt scaledparams.txt scale=0.5 12. Generate 3D mask from a given 3-D volume automatically or using threshold provided by user. 13. Postprocess 3-D or 2-D images: for 3-D volumes: calculate FSC with provided mask; weight summed volume with FSC; estimate B-factor from FSC weighted summed two volumes; apply negative B-factor to the weighted volume. for 2-D images: calculate B-factor and apply negative B-factor to 2-D images. 14. Winow stack file -reduce size of images without changing the pixel size. """ parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--order", action="store_true", help="Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False) parser.add_option("--order_lookup", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_metropolis", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_pca", action="store_true", help="Test/Debug.", default=False) parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)") parser.add_option("--circular", action="store_true", help="Select circular ordering (fisr image has to be similar to the last", default=False) parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") parser.add_option("--changesize", action="store_true", help="resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False) parser.add_option("--ratio", type="float", default=1.0, help="The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round") parser.add_option("--pw", action="store_true", help="compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False) parser.add_option("--wn", type="int", default=-1, help="Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))") parser.add_option("--phase_flip", action="store_true", help="Phase flip the input stack", default=False) parser.add_option("--makedb", metavar="param1=value1:param2=value2", type="string", action="append", help="One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0") parser.add_option("--generate_projections", metavar="param1=value1:param2=value2", type="string", action="append", help="Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64.") parser.add_option("--isacgroup", type="int", help="Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1) parser.add_option("--isacselect", action="store_true", help="Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False) parser.add_option("--params", type="string", default=None, help="Name of header of parameter, which one depends on specific option") parser.add_option("--adjpw", action="store_true", help="Adjust rotationally averaged power spectrum of an image", default=False) parser.add_option("--rotpw", type="string", default=None, help="Name of the text file to contain rotationally averaged power spectrum of the input image.") parser.add_option("--transformparams", type="string", default=None, help="Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters.") # import ctf estimates done using cter parser.add_option("--input", type="string", default= None, help="Input particles.") parser.add_option("--importctf", type="string", default= None, help="Name of the file containing CTF parameters produced by sxcter.") parser.add_option("--defocuserror", type="float", default=1000000.0, help="Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%") parser.add_option("--astigmatismerror", type="float", default=360.0, help="Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees.") # import ctf estimates done using cter parser.add_option("--scale", type="float", default=-1.0, help="Divide shifts in the input 3D orientation parameters text file by the scale factor.") # generate adaptive mask from an given 3-D volume parser.add_option("--adaptive_mask", action="store_true", help="create adavptive 3-D mask from a given volume", default=False) parser.add_option("--nsigma", type="float", default= 1., help="number of times of sigma of the input volume to obtain the the large density cluster") parser.add_option("--ndilation", type="int", default= 3, help="number of times of dilation applied to the largest cluster of density") parser.add_option("--kernel_size", type="int", default= 11, help="convolution kernel for smoothing the edge of the mask") parser.add_option("--gauss_standard_dev", type="int", default= 9, help="stanadard deviation value to generate Gaussian edge") parser.add_option("--threshold", type="float", default= 9999., help="threshold provided by user to binarize input volume") parser.add_option("--ne", type="int", default= 0, help="number of times to erode the binarized input image") parser.add_option("--nd", type="int", default= 0, help="number of times to dilate the binarized input image") parser.add_option("--postprocess", action="store_true", help="postprocess unfiltered odd, even 3-D volumes",default=False) parser.add_option("--fsc_weighted", action="store_true", help="postprocess unfiltered odd, even 3-D volumes") parser.add_option("--low_pass_filter", action="store_true", default=False, help="postprocess unfiltered odd, even 3-D volumes") parser.add_option("--ff", type="float", default=.25, help="low pass filter stop band frequency in absolute unit") parser.add_option("--aa", type="float", default=.1, help="low pass filter falloff" ) parser.add_option("--mask", type="string", help="input mask file", default=None) parser.add_option("--output", type="string", help="output file name", default="postprocessed.hdf") parser.add_option("--pixel_size", type="float", help="pixel size of the data", default=1.0) parser.add_option("--B_start", type="float", help="starting frequency in Angstrom for B-factor estimation", default=10.) parser.add_option("--FSC_cutoff", type="float", help="stop frequency in Angstrom for B-factor estimation", default=0.143) parser.add_option("--2d", action="store_true", help="postprocess isac 2-D averaged images",default=False) parser.add_option("--window_stack", action="store_true", help="window stack images using a smaller window size", default=False) parser.add_option("--box", type="int", default= 0, help="the new window size ") (options, args) = parser.parse_args() global_def.BATCH = True if options.phase_flip: nargs = len(args) if nargs != 2: print "must provide name of input and output file!" return from EMAN2 import Processor instack = args[0] outstack = args[1] nima = EMUtil.get_image_count(instack) from filter import filt_ctf for i in xrange(nima): img = EMData() img.read_image(instack, i) try: ctf = img.get_attr('ctf') except: print "no ctf information in input stack! Exiting..." return dopad = True sign = 1 binary = 1 # phase flip assert img.get_ysize() > 1 dict = ctf.to_dict() dz = dict["defocus"] cs = dict["cs"] voltage = dict["voltage"] pixel_size = dict["apix"] b_factor = dict["bfactor"] ampcont = dict["ampcont"] dza = dict["dfdiff"] azz = dict["dfang"] if dopad and not img.is_complex(): ip = 1 else: ip = 0 params = {"filter_type": Processor.fourier_filter_types.CTF_, "defocus" : dz, "Cs": cs, "voltage": voltage, "Pixel_size": pixel_size, "B_factor": b_factor, "amp_contrast": ampcont, "dopad": ip, "binary": binary, "sign": sign, "dza": dza, "azz":azz} tmp = Processor.EMFourierFilter(img, params) tmp.set_attr_dict({"ctf": ctf}) tmp.write_image(outstack, i) elif options.changesize: nargs = len(args) if nargs != 2: ERROR("must provide name of input and output file!", "change size", 1) return from utilities import get_im instack = args[0] outstack = args[1] sub_rate = float(options.ratio) nima = EMUtil.get_image_count(instack) from fundamentals import resample for i in xrange(nima): resample(get_im(instack, i), sub_rate).write_image(outstack, i) elif options.isacgroup>-1: nargs = len(args) if nargs != 3: ERROR("Three files needed on input!", "isacgroup", 1) return from utilities import get_im instack = args[0] m=get_im(args[1],int(options.isacgroup)).get_attr("members") l = [] for k in m: l.append(int(get_im(args[0],k).get_attr(options.params))) from utilities import write_text_file write_text_file(l, args[2]) elif options.isacselect: nargs = len(args) if nargs != 2: ERROR("Two files needed on input!", "isacgroup", 1) return from utilities import get_im nima = EMUtil.get_image_count(args[0]) m = [] for k in xrange(nima): m += get_im(args[0],k).get_attr("members") m.sort() from utilities import write_text_file write_text_file(m, args[1]) elif options.pw: nargs = len(args) if nargs < 2: ERROR("must provide name of input and output file!", "pw", 1) return from utilities import get_im, write_text_file from fundamentals import rops_table d = get_im(args[0]) ndim = d.get_ndim() if ndim ==3: pw = rops_table(d) write_text_file(pw, args[1]) else: nx = d.get_xsize() ny = d.get_ysize() if nargs ==3: mask = get_im(args[2]) wn = int(options.wn) if wn == -1: wn = max(nx, ny) else: if( (wn<nx) or (wn<ny) ): ERROR("window size cannot be smaller than the image size","pw",1) n = EMUtil.get_image_count(args[0]) from utilities import model_blank, model_circle, pad from EMAN2 import periodogram p = model_blank(wn,wn) for i in xrange(n): d = get_im(args[0], i) if nargs==3: d *=mask st = Util.infomask(d, None, True) d -= st[0] p += periodogram(pad(d, wn, wn, 1, 0.)) p /= n p.write_image(args[1]) elif options.adjpw: if len(args) < 3: ERROR("filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)","adjpw",1) return img_stack = args[0] from math import sqrt from fundamentals import rops_table, fft from utilities import read_text_file, get_im from filter import filt_tanl, filt_table if( args[1][-3:] == 'txt'): rops_dst = read_text_file( args[1] ) else: rops_dst = rops_table(get_im( args[1] )) out_stack = args[2] if(len(args) >4): fl = float(args[3]) aa = float(args[4]) else: fl = -1.0 aa = 0.0 nimage = EMUtil.get_image_count( img_stack ) for i in xrange(nimage): img = fft(get_im(img_stack, i) ) rops_src = rops_table(img) assert len(rops_dst) == len(rops_src) table = [0.0]*len(rops_dst) for j in xrange( len(rops_dst) ): table[j] = sqrt( rops_dst[j]/rops_src[j] ) if( fl > 0.0): img = filt_tanl(img, fl, aa) img = fft(filt_table(img, table)) img.write_image(out_stack, i) elif options.rotpw != None: if len(args) != 1: ERROR("Only one input permitted","rotpw",1) return from utilities import write_text_file, get_im from fundamentals import rops_table from math import log10 t = rops_table(get_im(args[0])) x = range(len(t)) r = [0.0]*len(x) for i in x: r[i] = log10(t[i]) write_text_file([t,r,x],options.rotpw) elif options.transformparams != None: if len(args) != 2: ERROR("Please provide names of input and output files with orientation parameters","transformparams",1) return from utilities import read_text_row, write_text_row transf = [0.0]*6 spl=options.transformparams.split(',') for i in xrange(len(spl)): transf[i] = float(spl[i]) write_text_row( rotate_shift_params(read_text_row(args[0]), transf) , args[1]) elif options.makedb != None: nargs = len(args) if nargs != 1: print "must provide exactly one argument denoting database key under which the input params will be stored" return dbkey = args[0] print "database key under which params will be stored: ", dbkey gbdb = js_open_dict("e2boxercache/gauss_box_DB.json") parmstr = 'dummy:'+options.makedb[0] (processorname, param_dict) = parsemodopt(parmstr) dbdict = {} for pkey in param_dict: if (pkey == 'invert_contrast') or (pkey == 'use_variance'): if param_dict[pkey] == 'True': dbdict[pkey] = True else: dbdict[pkey] = False else: dbdict[pkey] = param_dict[pkey] gbdb[dbkey] = dbdict elif options.generate_projections: nargs = len(args) if nargs != 3: ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\ "sxprocess - generate projections",1) return inpstr = args[0] outstk = args[1] micpref = args[2] parmstr = 'dummy:'+options.generate_projections[0] (processorname, param_dict) = parsemodopt(parmstr) parm_CTF = False parm_format = 'bdb' parm_apix = 2.5 if 'CTF' in param_dict: if param_dict['CTF'] == 'True': parm_CTF = True if 'format' in param_dict: parm_format = param_dict['format'] if 'apix' in param_dict: parm_apix = float(param_dict['apix']) boxsize = 64 if 'boxsize' in param_dict: boxsize = int(param_dict['boxsize']) print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize scale_mult = 2500 sigma_add = 1.5 sigma_proj = 30.0 sigma2_proj = 17.5 sigma_gauss = 0.3 sigma_mic = 30.0 sigma2_mic = 17.5 sigma_gauss_mic = 0.3 if 'scale_mult' in param_dict: scale_mult = float(param_dict['scale_mult']) if 'sigma_add' in param_dict: sigma_add = float(param_dict['sigma_add']) if 'sigma_proj' in param_dict: sigma_proj = float(param_dict['sigma_proj']) if 'sigma2_proj' in param_dict: sigma2_proj = float(param_dict['sigma2_proj']) if 'sigma_gauss' in param_dict: sigma_gauss = float(param_dict['sigma_gauss']) if 'sigma_mic' in param_dict: sigma_mic = float(param_dict['sigma_mic']) if 'sigma2_mic' in param_dict: sigma2_mic = float(param_dict['sigma2_mic']) if 'sigma_gauss_mic' in param_dict: sigma_gauss_mic = float(param_dict['sigma_gauss_mic']) from filter import filt_gaussl, filt_ctf from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank,pad,model_gauss_noise,set_params2D, set_params_proj from projection import prep_vol,prgs seed(14567) delta = 29 angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S") nangle = len(angles) modelvol = [] nvlms = EMUtil.get_image_count(inpstr) from utilities import get_im for k in xrange(nvlms): modelvol.append(get_im(inpstr,k)) nx = modelvol[0].get_xsize() if nx != boxsize: ERROR("Requested box dimension does not match dimension of the input model.", \ "sxprocess - generate projections",1) nvol = 10 volfts = [[] for k in xrange(nvlms)] for k in xrange(nvlms): for i in xrange(nvol): sigma = sigma_add + random() # 1.5-2.5 addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40 ) scale = scale_mult * (0.5+random()) vf, kb = prep_vol(modelvol[k] + scale*addon) volfts[k].append(vf) del vf, modelvol if parm_format == "bdb": stack_data = "bdb:"+outstk delete_bdb(stack_data) else: stack_data = outstk + ".hdf" Cs = 2.0 pixel = parm_apix voltage = 120.0 ampcont = 10.0 ibd = 4096/2-boxsize iprj = 0 width = 240 xstart = 8 + boxsize/2 ystart = 8 + boxsize/2 rowlen = 17 from random import randint params = [] for idef in xrange(3, 8): irow = 0 icol = 0 mic = model_blank(4096, 4096) defocus = idef * 0.5#0.2 if parm_CTF: astampl=defocus*0.15 astangl=50.0 ctf = generate_ctf([defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl]) for i in xrange(nangle): for k in xrange(12): dphi = 8.0*(random()-0.5) dtht = 8.0*(random()-0.5) psi = 360.0*random() phi = angles[i][0]+dphi tht = angles[i][1]+dtht s2x = 4.0*(random()-0.5) s2y = 4.0*(random()-0.5) params.append([phi, tht, psi, s2x, s2y]) ivol = iprj % nvol #imgsrc = randint(0,nvlms-1) imgsrc = iprj % nvlms proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y]) x = xstart + irow * width y = ystart + icol * width mic += pad(proj, 4096, 4096, 1, 0.0, x-2048, y-2048, 0) proj = proj + model_gauss_noise( sigma_proj, nx, nx ) if parm_CTF: proj = filt_ctf(proj, ctf) proj.set_attr_dict({"ctf":ctf, "ctf_applied":0}) proj = proj + filt_gaussl(model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss) proj.set_attr("origimgsrc",imgsrc) proj.set_attr("test_id", iprj) # flags describing the status of the image (1 = true, 0 = false) set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0]) set_params_proj(proj, [phi, tht, psi, s2x, s2y]) proj.write_image(stack_data, iprj) icol += 1 if icol == rowlen: icol = 0 irow += 1 iprj += 1 mic += model_gauss_noise(sigma_mic,4096,4096) if parm_CTF: #apply CTF mic = filt_ctf(mic, ctf) mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic) mic.write_image(micpref + "%1d.hdf" % (idef-3), 0) drop_spider_doc("params.txt", params) elif options.importctf != None: print ' IMPORTCTF ' from utilities import read_text_row,write_text_row from random import randint import subprocess grpfile = 'groupid%04d'%randint(1000,9999) ctfpfile = 'ctfpfile%04d'%randint(1000,9999) cterr = [options.defocuserror/100.0, options.astigmatismerror] ctfs = read_text_row(options.importctf) for kk in xrange(len(ctfs)): root,name = os.path.split(ctfs[kk][-1]) ctfs[kk][-1] = name[:-4] if(options.input[:4] != 'bdb:'): ERROR('Sorry, only bdb files implemented','importctf',1) d = options.input[4:] #try: str = d.index('*') #except: str = -1 from string import split import glob uu = os.path.split(d) uu = os.path.join(uu[0],'EMAN2DB',uu[1]+'.bdb') flist = glob.glob(uu) for i in xrange(len(flist)): root,name = os.path.split(flist[i]) root = root[:-7] name = name[:-4] fil = 'bdb:'+os.path.join(root,name) sourcemic = EMUtil.get_all_attributes(fil,'ptcl_source_image') nn = len(sourcemic) gctfp = [] groupid = [] for kk in xrange(nn): junk,name2 = os.path.split(sourcemic[kk]) name2 = name2[:-4] ctfp = [-1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] for ll in xrange(len(ctfs)): if(name2 == ctfs[ll][-1]): # found correct if(ctfs[ll][8]/ctfs[ll][0] <= cterr[0]): # acceptable defocus error ctfp = ctfs[ll][:8] if(ctfs[ll][10] > cterr[1] ): # error of astigmatism exceed the threshold, set astigmatism to zero. ctfp[6] = 0.0 ctfp[7] = 0.0 gctfp.append(ctfp) groupid.append(kk) break if(len(groupid) > 0): write_text_row(groupid, grpfile) write_text_row(gctfp, ctfpfile) cmd = "{} {} {} {}".format('e2bdb.py',fil,'--makevstack=bdb:'+root+'G'+name,'--list='+grpfile) #print cmd subprocess.call(cmd, shell=True) cmd = "{} {} {} {}".format('sxheader.py','bdb:'+root+'G'+name,'--params=ctf','--import='+ctfpfile) #print cmd subprocess.call(cmd, shell=True) else: print ' >>> Group ',name,' skipped.' cmd = "{} {} {}".format("rm -f",grpfile,ctfpfile) subprocess.call(cmd, shell=True) elif options.scale > 0.0: from utilities import read_text_row,write_text_row scale = options.scale nargs = len(args) if nargs != 2: print "Please provide names of input and output file!" return p = read_text_row(args[0]) for i in xrange(len(p)): p[i][3] /= scale p[i][4] /= scale write_text_row(p, args[1]) elif options.adaptive_mask: from utilities import get_im from morphology import adaptive_mask, binarize, erosion, dilation nsigma = options.nsigma ndilation = options.ndilation kernel_size = options.kernel_size gauss_standard_dev = options.gauss_standard_dev nargs = len(args) if nargs ==0: print " Create 3D mask from a given volume, either automatically or from the user provided threshold." elif nargs > 2: print "Too many inputs are given, try again!" return else: inputvol = get_im(args[0]) input_path, input_file_name = os.path.split(args[0]) input_file_name_root,ext=os.path.splitext(input_file_name) if nargs == 2: mask_file_name = args[1] else: mask_file_name = "adaptive_mask_for_"+input_file_name_root+".hdf" # Only hdf file is output. if options.threshold !=9999.: mask3d = binarize(inputvol, options.threshold) for i in xrange(options.ne): mask3d = erosion(mask3d) for i in xrange(options.nd): mask3d = dilation(mask3d) else: mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev) mask3d.write_image(mask_file_name) elif options.postprocess: from utilities import get_im from fundamentals import rot_avg_table from morphology import compute_bfactor,power from statistics import fsc from filter import filt_table, filt_gaussinv from EMAN2 import periodogram e1 = get_im(args[0],0) if e1.get_zsize()==1: nimage = EMUtil.get_image_count(args[0]) if options.mask !=None: m = get_im(options.mask) else: m = None for i in xrange(nimage): e1 = get_im(args[0],i) if m: e1 *=m guinerline = rot_avg_table(power(periodogram(e1),.5)) freq_max = 1/(2.*pixel_size) freq_min = 1./options.B_start b,junk=compute_bfactor(guinerline, freq_min, freq_max, pixel_size) tmp = b/pixel_size**2 sigma_of_inverse=sqrt(2./tmp) e1 = filt_gaussinv(e1,sigma_of_inverse) if options.low_pass_filter: from filter import filt_tanl e1 =filt_tanl(e1,options.ff, options.aa) e1.write_image(options.output) else: nargs = len(args) e1 = get_im(args[0]) if nargs >1: e2 = get_im(args[1]) if options.mask !=None: m = get_im(options.mask) else: m =None pixel_size = options.pixel_size from math import sqrt if m !=None: e1 *=m if nargs >1 :e2 *=m if options.fsc_weighted: frc = fsc(e1,e2,1) ## FSC is done on masked two images #### FSC weighting sqrt((2.*fsc)/(1+fsc)); fil = len(frc[1])*[None] for i in xrange(len(fil)): if frc[1][i]>=options.FSC_cutoff: tmp = frc[1][i] else: tmp = 0.0 fil[i] = sqrt(2.*tmp/(1.+tmp)) if nargs>1: e1 +=e2 if options.fsc_weighted: e1=filt_table(e1,fil) guinerline = rot_avg_table(power(periodogram(e1),.5)) freq_max = 1/(2.*pixel_size) freq_min = 1./options.B_start b,junk = compute_bfactor(guinerline, freq_min, freq_max, pixel_size) tmp = b/pixel_size**2 sigma_of_inverse=sqrt(2./tmp) e1 = filt_gaussinv(e1,sigma_of_inverse) if options.low_pass_filter: from filter import filt_tanl e1 =filt_tanl(e1,options.ff, options.aa) e1.write_image(options.output) elif options.window_stack: nargs = len(args) if nargs ==0: print " Reduce image size of a stack" return else: output_stack_name = None inputstack = args[0] if nargs ==2:output_stack_name = args[1] input_path,input_file_name=os.path.split(inputstack) input_file_name_root,ext=os.path.splitext(input_file_name) if input_file_name_root[0:3]=="bdb":stack_is_bdb= True else: stack_is_bdb= False if output_stack_name is None: if stack_is_bdb: output_stack_name ="bdb:reduced_"+input_file_name_root[4:] else:output_stack_name = "reduced_"+input_file_name_root+".hdf" # Only hdf file is output. nimage = EMUtil.get_image_count(inputstack) from fundamentals import window2d for i in xrange(nimage): image = EMData() image.read_image(inputstack,i) w = window2d(image,options.box,options.box) w.write_image(output_stack_name,i) else: ERROR("Please provide option name","sxprocess.py",1)
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Generic 2-D image processing programs. Functionality: 1. Phase flip a stack of images and write output to new file: sxprocess.py input_stack.hdf output_stack.hdf --phase_flip 2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size. The window size will change accordingly. sxprocess input.hdf output.hdf --changesize --ratio=0.5 3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes. sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024] 4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs: sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64 5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3): sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid 6. Retrieve original image numbers of images listed in ISAC output stack of averages: sxprocess.py select1.hdf ohk.txt 7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file. Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack. sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw 8. Generate a 1D rotationally averaged power spectrum of an image. sxprocess.py vol.hdf --rotwp=rotpw.txt # Output will contain three columns: (1) rotationally averaged power spectrum (2) logarithm of the rotationally averaged power spectrum (3) integer line number (from zero to approximately to half the image size) 9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data. sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt The output file is then imported and 3D transformed volume computed: sxheader.py bdb:p --params=xform.projection --import=output.txt mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI The reconstructed volume is in the position of the volume computed using the input.txt parameters and then transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz) 10. Import ctf parameters from the output of sxcter into windowed particle headers. There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph. In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'. Normally this is done by e2boxer.py during windowing. Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created. sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 # Output will be a vritual stack bdb:Gdata sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 To concatenate output files: cd directory e2bdb.py . --makevstack=bdb:allparticles --filt=G IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them. 11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor sxprocess.py orientationparams.txt scaledparams.txt scale=0.5 12. Generate adaptive mask from a given 3-D volume. """ parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--order", action="store_true", help= "Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False) parser.add_option("--order_lookup", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_metropolis", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_pca", action="store_true", help="Test/Debug.", default=False) parser.add_option( "--initial", type="int", default=-1, help= "Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)" ) parser.add_option( "--circular", action="store_true", help= "Select circular ordering (fisr image has to be similar to the last", default=False) parser.add_option( "--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") parser.add_option( "--changesize", action="store_true", help= "resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False) parser.add_option( "--ratio", type="float", default=1.0, help= "The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round" ) parser.add_option( "--pw", action="store_true", help= "compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False) parser.add_option( "--wn", type="int", default=-1, help= "Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))" ) parser.add_option("--phase_flip", action="store_true", help="Phase flip the input stack", default=False) parser.add_option( "--makedb", metavar="param1=value1:param2=value2", type="string", action="append", help= "One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0" ) parser.add_option( "--generate_projections", metavar="param1=value1:param2=value2", type="string", action="append", help= "Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64." ) parser.add_option( "--isacgroup", type="int", help= "Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1) parser.add_option( "--isacselect", action="store_true", help= "Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False) parser.add_option( "--params", type="string", default=None, help="Name of header of parameter, which one depends on specific option" ) parser.add_option( "--adjpw", action="store_true", help="Adjust rotationally averaged power spectrum of an image", default=False) parser.add_option( "--rotpw", type="string", default=None, help= "Name of the text file to contain rotationally averaged power spectrum of the input image." ) parser.add_option( "--transformparams", type="string", default=None, help= "Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters." ) # import ctf estimates done using cter parser.add_option("--input", type="string", default=None, help="Input particles.") parser.add_option( "--importctf", type="string", default=None, help="Name of the file containing CTF parameters produced by sxcter.") parser.add_option( "--defocuserror", type="float", default=1000000.0, help= "Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%" ) parser.add_option( "--astigmatismerror", type="float", default=360.0, help= "Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees." ) # import ctf estimates done using cter parser.add_option( "--scale", type="float", default=-1.0, help= "Divide shifts in the input 3D orientation parameters text file by the scale factor." ) # generate adaptive mask from an given 3-Db volue parser.add_option("--adaptive_mask", action="store_true", help="create adavptive 3-D mask from a given volume", default=False) parser.add_option( "--nsigma", type="float", default=1., help= "number of times of sigma of the input volume to obtain the the large density cluster" ) parser.add_option( "--ndilation", type="int", default=3, help= "number of times of dilation applied to the largest cluster of density" ) parser.add_option( "--kernel_size", type="int", default=11, help="convolution kernel for smoothing the edge of the mask") parser.add_option( "--gauss_standard_dev", type="int", default=9, help="stanadard deviation value to generate Gaussian edge") (options, args) = parser.parse_args() global_def.BATCH = True if options.phase_flip: nargs = len(args) if nargs != 2: print "must provide name of input and output file!" return from EMAN2 import Processor instack = args[0] outstack = args[1] nima = EMUtil.get_image_count(instack) from filter import filt_ctf for i in xrange(nima): img = EMData() img.read_image(instack, i) try: ctf = img.get_attr('ctf') except: print "no ctf information in input stack! Exiting..." return dopad = True sign = 1 binary = 1 # phase flip assert img.get_ysize() > 1 dict = ctf.to_dict() dz = dict["defocus"] cs = dict["cs"] voltage = dict["voltage"] pixel_size = dict["apix"] b_factor = dict["bfactor"] ampcont = dict["ampcont"] dza = dict["dfdiff"] azz = dict["dfang"] if dopad and not img.is_complex(): ip = 1 else: ip = 0 params = { "filter_type": Processor.fourier_filter_types.CTF_, "defocus": dz, "Cs": cs, "voltage": voltage, "Pixel_size": pixel_size, "B_factor": b_factor, "amp_contrast": ampcont, "dopad": ip, "binary": binary, "sign": sign, "dza": dza, "azz": azz } tmp = Processor.EMFourierFilter(img, params) tmp.set_attr_dict({"ctf": ctf}) tmp.write_image(outstack, i) elif options.changesize: nargs = len(args) if nargs != 2: ERROR("must provide name of input and output file!", "change size", 1) return from utilities import get_im instack = args[0] outstack = args[1] sub_rate = float(options.ratio) nima = EMUtil.get_image_count(instack) from fundamentals import resample for i in xrange(nima): resample(get_im(instack, i), sub_rate).write_image(outstack, i) elif options.isacgroup > -1: nargs = len(args) if nargs != 3: ERROR("Three files needed on input!", "isacgroup", 1) return from utilities import get_im instack = args[0] m = get_im(args[1], int(options.isacgroup)).get_attr("members") l = [] for k in m: l.append(int(get_im(args[0], k).get_attr(options.params))) from utilities import write_text_file write_text_file(l, args[2]) elif options.isacselect: nargs = len(args) if nargs != 2: ERROR("Two files needed on input!", "isacgroup", 1) return from utilities import get_im nima = EMUtil.get_image_count(args[0]) m = [] for k in xrange(nima): m += get_im(args[0], k).get_attr("members") m.sort() from utilities import write_text_file write_text_file(m, args[1]) elif options.pw: nargs = len(args) if nargs < 2: ERROR("must provide name of input and output file!", "pw", 1) return from utilities import get_im d = get_im(args[0]) nx = d.get_xsize() ny = d.get_ysize() if nargs == 3: mask = get_im(args[2]) wn = int(options.wn) if wn == -1: wn = max(nx, ny) else: if ((wn < nx) or (wn < ny)): ERROR("window size cannot be smaller than the image size", "pw", 1) n = EMUtil.get_image_count(args[0]) from utilities import model_blank, model_circle, pad from EMAN2 import periodogram p = model_blank(wn, wn) for i in xrange(n): d = get_im(args[0], i) if nargs == 3: d *= mask st = Util.infomask(d, None, True) d -= st[0] p += periodogram(pad(d, wn, wn, 1, 0.)) p /= n p.write_image(args[1]) elif options.adjpw: if len(args) < 3: ERROR( "filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)", "adjpw", 1) return img_stack = args[0] from math import sqrt from fundamentals import rops_table, fft from utilities import read_text_file, get_im from filter import filt_tanl, filt_table if (args[1][-3:] == 'txt'): rops_dst = read_text_file(args[1]) else: rops_dst = rops_table(get_im(args[1])) out_stack = args[2] if (len(args) > 4): fl = float(args[3]) aa = float(args[4]) else: fl = -1.0 aa = 0.0 nimage = EMUtil.get_image_count(img_stack) for i in xrange(nimage): img = fft(get_im(img_stack, i)) rops_src = rops_table(img) assert len(rops_dst) == len(rops_src) table = [0.0] * len(rops_dst) for j in xrange(len(rops_dst)): table[j] = sqrt(rops_dst[j] / rops_src[j]) if (fl > 0.0): img = filt_tanl(img, fl, aa) img = fft(filt_table(img, table)) img.write_image(out_stack, i) elif options.rotpw != None: if len(args) != 1: ERROR("Only one input permitted", "rotpw", 1) return from utilities import write_text_file, get_im from fundamentals import rops_table from math import log10 t = rops_table(get_im(args[0])) x = range(len(t)) r = [0.0] * len(x) for i in x: r[i] = log10(t[i]) write_text_file([t, r, x], options.rotpw) elif options.transformparams != None: if len(args) != 2: ERROR( "Please provide names of input and output files with orientation parameters", "transformparams", 1) return from utilities import read_text_row, write_text_row transf = [0.0] * 6 spl = options.transformparams.split(',') for i in xrange(len(spl)): transf[i] = float(spl[i]) write_text_row(rotate_shift_params(read_text_row(args[0]), transf), args[1]) elif options.makedb != None: nargs = len(args) if nargs != 1: print "must provide exactly one argument denoting database key under which the input params will be stored" return dbkey = args[0] print "database key under which params will be stored: ", dbkey gbdb = js_open_dict("e2boxercache/gauss_box_DB.json") parmstr = 'dummy:' + options.makedb[0] (processorname, param_dict) = parsemodopt(parmstr) dbdict = {} for pkey in param_dict: if (pkey == 'invert_contrast') or (pkey == 'use_variance'): if param_dict[pkey] == 'True': dbdict[pkey] = True else: dbdict[pkey] = False else: dbdict[pkey] = param_dict[pkey] gbdb[dbkey] = dbdict elif options.generate_projections: nargs = len(args) if nargs != 3: ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\ "sxprocess - generate projections",1) return inpstr = args[0] outstk = args[1] micpref = args[2] parmstr = 'dummy:' + options.generate_projections[0] (processorname, param_dict) = parsemodopt(parmstr) parm_CTF = False parm_format = 'bdb' parm_apix = 2.5 if 'CTF' in param_dict: if param_dict['CTF'] == 'True': parm_CTF = True if 'format' in param_dict: parm_format = param_dict['format'] if 'apix' in param_dict: parm_apix = float(param_dict['apix']) boxsize = 64 if 'boxsize' in param_dict: boxsize = int(param_dict['boxsize']) print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize scale_mult = 2500 sigma_add = 1.5 sigma_proj = 30.0 sigma2_proj = 17.5 sigma_gauss = 0.3 sigma_mic = 30.0 sigma2_mic = 17.5 sigma_gauss_mic = 0.3 if 'scale_mult' in param_dict: scale_mult = float(param_dict['scale_mult']) if 'sigma_add' in param_dict: sigma_add = float(param_dict['sigma_add']) if 'sigma_proj' in param_dict: sigma_proj = float(param_dict['sigma_proj']) if 'sigma2_proj' in param_dict: sigma2_proj = float(param_dict['sigma2_proj']) if 'sigma_gauss' in param_dict: sigma_gauss = float(param_dict['sigma_gauss']) if 'sigma_mic' in param_dict: sigma_mic = float(param_dict['sigma_mic']) if 'sigma2_mic' in param_dict: sigma2_mic = float(param_dict['sigma2_mic']) if 'sigma_gauss_mic' in param_dict: sigma_gauss_mic = float(param_dict['sigma_gauss_mic']) from filter import filt_gaussl, filt_ctf from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank, pad, model_gauss_noise, set_params2D, set_params_proj from projection import prep_vol, prgs seed(14567) delta = 29 angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S") nangle = len(angles) modelvol = [] nvlms = EMUtil.get_image_count(inpstr) from utilities import get_im for k in xrange(nvlms): modelvol.append(get_im(inpstr, k)) nx = modelvol[0].get_xsize() if nx != boxsize: ERROR("Requested box dimension does not match dimension of the input model.", \ "sxprocess - generate projections",1) nvol = 10 volfts = [[] for k in xrange(nvlms)] for k in xrange(nvlms): for i in xrange(nvol): sigma = sigma_add + random() # 1.5-2.5 addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40) scale = scale_mult * (0.5 + random()) vf, kb = prep_vol(modelvol[k] + scale * addon) volfts[k].append(vf) del vf, modelvol if parm_format == "bdb": stack_data = "bdb:" + outstk delete_bdb(stack_data) else: stack_data = outstk + ".hdf" Cs = 2.0 pixel = parm_apix voltage = 120.0 ampcont = 10.0 ibd = 4096 / 2 - boxsize iprj = 0 width = 240 xstart = 8 + boxsize / 2 ystart = 8 + boxsize / 2 rowlen = 17 from random import randint params = [] for idef in xrange(3, 8): irow = 0 icol = 0 mic = model_blank(4096, 4096) defocus = idef * 0.5 #0.2 if parm_CTF: astampl = defocus * 0.15 astangl = 50.0 ctf = generate_ctf([ defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl ]) for i in xrange(nangle): for k in xrange(12): dphi = 8.0 * (random() - 0.5) dtht = 8.0 * (random() - 0.5) psi = 360.0 * random() phi = angles[i][0] + dphi tht = angles[i][1] + dtht s2x = 4.0 * (random() - 0.5) s2y = 4.0 * (random() - 0.5) params.append([phi, tht, psi, s2x, s2y]) ivol = iprj % nvol #imgsrc = randint(0,nvlms-1) imgsrc = iprj % nvlms proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y]) x = xstart + irow * width y = ystart + icol * width mic += pad(proj, 4096, 4096, 1, 0.0, x - 2048, y - 2048, 0) proj = proj + model_gauss_noise(sigma_proj, nx, nx) if parm_CTF: proj = filt_ctf(proj, ctf) proj.set_attr_dict({"ctf": ctf, "ctf_applied": 0}) proj = proj + filt_gaussl( model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss) proj.set_attr("origimgsrc", imgsrc) proj.set_attr("test_id", iprj) # flags describing the status of the image (1 = true, 0 = false) set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0]) set_params_proj(proj, [phi, tht, psi, s2x, s2y]) proj.write_image(stack_data, iprj) icol += 1 if icol == rowlen: icol = 0 irow += 1 iprj += 1 mic += model_gauss_noise(sigma_mic, 4096, 4096) if parm_CTF: #apply CTF mic = filt_ctf(mic, ctf) mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic) mic.write_image(micpref + "%1d.hdf" % (idef - 3), 0) drop_spider_doc("params.txt", params) elif options.importctf != None: print ' IMPORTCTF ' from utilities import read_text_row, write_text_row from random import randint import subprocess grpfile = 'groupid%04d' % randint(1000, 9999) ctfpfile = 'ctfpfile%04d' % randint(1000, 9999) cterr = [options.defocuserror / 100.0, options.astigmatismerror] ctfs = read_text_row(options.importctf) for kk in xrange(len(ctfs)): root, name = os.path.split(ctfs[kk][-1]) ctfs[kk][-1] = name[:-4] if (options.input[:4] != 'bdb:'): ERROR('Sorry, only bdb files implemented', 'importctf', 1) d = options.input[4:] #try: str = d.index('*') #except: str = -1 from string import split import glob uu = os.path.split(d) uu = os.path.join(uu[0], 'EMAN2DB', uu[1] + '.bdb') flist = glob.glob(uu) for i in xrange(len(flist)): root, name = os.path.split(flist[i]) root = root[:-7] name = name[:-4] fil = 'bdb:' + os.path.join(root, name) sourcemic = EMUtil.get_all_attributes(fil, 'ptcl_source_image') nn = len(sourcemic) gctfp = [] groupid = [] for kk in xrange(nn): junk, name2 = os.path.split(sourcemic[kk]) name2 = name2[:-4] ctfp = [-1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] for ll in xrange(len(ctfs)): if (name2 == ctfs[ll][-1]): # found correct if (ctfs[ll][8] / ctfs[ll][0] <= cterr[0]): # acceptable defocus error ctfp = ctfs[ll][:8] if (ctfs[ll][10] > cterr[1]): # error of astigmatism exceed the threshold, set astigmatism to zero. ctfp[6] = 0.0 ctfp[7] = 0.0 gctfp.append(ctfp) groupid.append(kk) break if (len(groupid) > 0): write_text_row(groupid, grpfile) write_text_row(gctfp, ctfpfile) cmd = "{} {} {} {}".format( 'e2bdb.py', fil, '--makevstack=bdb:' + root + 'G' + name, '--list=' + grpfile) #print cmd subprocess.call(cmd, shell=True) cmd = "{} {} {} {}".format('sxheader.py', 'bdb:' + root + 'G' + name, '--params=ctf', '--import=' + ctfpfile) #print cmd subprocess.call(cmd, shell=True) else: print ' >>> Group ', name, ' skipped.' cmd = "{} {} {}".format("rm -f", grpfile, ctfpfile) subprocess.call(cmd, shell=True) elif options.scale > 0.0: from utilities import read_text_row, write_text_row scale = options.scale nargs = len(args) if nargs != 2: print "Please provide names of input and output file!" return p = read_text_row(args[0]) for i in xrange(len(p)): p[i][3] /= scale p[i][4] /= scale write_text_row(p, args[1]) elif options.adaptive_mask: from utilities import get_im from morphology import adaptive_mask nsigma = options.nsigma ndilation = options.ndilation kernel_size = options.kernel_size gauss_standard_dev = options.gauss_standard_dev nargs = len(args) if nargs > 2: print "Too many inputs are given, try again!" return else: inputvol = get_im(args[0]) input_path, input_file_name = os.path.split(args[0]) input_file_name_root, ext = os.path.splitext(input_file_name) if nargs == 2: mask_file_name = args[1] else: mask_file_name = "adaptive_mask_for" + input_file_name_root + ".hdf" # Only hdf file is output. mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev) mask3d.write_image(mask_file_name) else: ERROR("Please provide option name", "sxprocess.py", 1)