def test_tight_mask(): ''' ''' try: from utilities import gauss_edge from morphology import binarize, dilation except: return rad, width = 13, 78 ndilate = 1 kernel_size = 3 gauss_standard_dev = 3 obj = ndimage_utility.model_disk(rad, (width, width)).astype(numpy.float) mask = obj + numpy.random.rand(width, width) * 0.2 emmask = eman2_utility.numpy2em(mask) threshold = unary_classification.otsu(mask.ravel()) if 1 == 1: m1 = eman2_utility.EMAN2.Util.get_biggest_cluster( binarize(emmask, threshold)) for i in xrange(ndilate): m1 = dilation(m1) if kernel_size > 0: m1 = gauss_edge(m1, kernel_size, gauss_standard_dev) m2 = ndimage_utility.tight_mask(mask, threshold, ndilate, kernel_size, gauss_standard_dev)[0] m3 = eman2_utility.em2numpy(m1) #print numpy.sum(m3), numpy.sum(m2), numpy.max(m3), numpy.max(m2), numpy.sqrt(numpy.sum((m3-m2)**2)), numpy.sqrt(numpy.max((m3-m2)**2)) numpy.testing.assert_allclose(m3, m2)
def test_tight_mask(): ''' ''' try: from utilities import gauss_edge from morphology import binarize, dilation except: return rad, width = 13, 78 ndilate=1 kernel_size=3 gauss_standard_dev=3 obj = ndimage_utility.model_disk(rad, (width, width)).astype(numpy.float) mask = obj + numpy.random.rand(width,width)*0.2 emmask = eman2_utility.numpy2em(mask) threshold = unary_classification.otsu(mask.ravel()) if 1 == 1: m1 = eman2_utility.EMAN2.Util.get_biggest_cluster(binarize(emmask, threshold)) for i in xrange(ndilate): m1 = dilation(m1) if kernel_size > 0: m1 = gauss_edge(m1, kernel_size, gauss_standard_dev) m2 = ndimage_utility.tight_mask(mask, threshold, ndilate, kernel_size, gauss_standard_dev)[0] m3 = eman2_utility.em2numpy(m1) #print numpy.sum(m3), numpy.sum(m2), numpy.max(m3), numpy.max(m2), numpy.sqrt(numpy.sum((m3-m2)**2)), numpy.sqrt(numpy.max((m3-m2)**2)) numpy.testing.assert_allclose(m3, m2)
def test_biggest_object(): ''' ''' try: from morphology import binarize except: return rad, width = 13, 78 obj = ndimage_utility.model_disk(rad, (width, width)).astype(numpy.float) mask = obj + numpy.random.rand(width,width)*0.2 emmask = eman2_utility.numpy2em(mask) threshold = unary_classification.otsu(mask.ravel()) embin = binarize(emmask, threshold) m1 = eman2_utility.EMAN2.Util.get_biggest_cluster(embin) m2 = ndimage_utility.biggest_object(mask>threshold) numpy.testing.assert_allclose(eman2_utility.em2numpy(m1), m2)
def test_biggest_object(): ''' ''' try: from morphology import binarize except: return rad, width = 13, 78 obj = ndimage_utility.model_disk(rad, (width, width)).astype(numpy.float) mask = obj + numpy.random.rand(width, width) * 0.2 emmask = eman2_utility.numpy2em(mask) threshold = unary_classification.otsu(mask.ravel()) embin = binarize(emmask, threshold) m1 = eman2_utility.EMAN2.Util.get_biggest_cluster(embin) m2 = ndimage_utility.biggest_object(mask > threshold) numpy.testing.assert_allclose(eman2_utility.em2numpy(m1), m2)
def main(): import os, sys arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " volume binarymask smoothmask --variance --repair" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--variance", type="string", default=None, help="variance map") parser.add_option("--repair", type="string", default="repair.hdf", help="repair map") (options, args) = parser.parse_args(arglist[1:]) if (len(args) != 3): print("usage: " + usage) return None if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() from utilities import get_im from morphology import adaptive_mask, binarize v = get_im(args[0]) m = adaptive_mask(v, 2.0, 2) bm = binarize(m, 0.5) bm.write_image(args[1]) adaptive_mask(v, 2.0, 2, 9, 3).write_image(args[2]) if (options.variance != None): from fundamentals import rot_avg_image from morphology import square_root trovc = rot_avg_image(get_im(options.variance)) nc = trovc.get_xsize() // 2 trovc /= trovc[nc, nc, nc] square_root(trovc).write_image(options.repair)
def adaptive_mask2D(img, nsigma = 1.0, ndilation = 3, kernel_size = 11, gauss_standard_dev =9): """ Name adaptive_mask - create a mask from a given image. Input img: input image nsigma: value for initial thresholding of the image. Output mask: The mask will have values one, zero, with Gaussian smooth transition between two regions. """ from utilities import gauss_edge, model_circle from morphology import binarize, dilation nx = img.get_xsize() ny = img.get_ysize() mc = model_circle(nx//2, nx, ny) - model_circle(nx//3, nx, ny) s1 = Util.infomask(img, mc, True) mask = Util.get_biggest_cluster(binarize(img, s1[0]+s1[1]*nsigma)) for i in xrange(ndilation): mask = dilation(mask) #mask = gauss_edge(mask, kernel_size, gauss_standard_dev) return mask
def main(): import os,sys arglist = [] for arg in sys.argv: arglist.append( arg ) progname = os.path.basename(arglist[0]) usage = progname + " volume binarymask smoothmask --variance --repair" parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--variance", type="string", default=None, help="variance map") parser.add_option("--repair", type="string", default="repair.hdf", help="repair map") (options, args) = parser.parse_args( arglist[1:] ) if( len(args) != 3): print "usage: " + usage return None if global_def.CACHE_DISABLE: from utilities import disable_bdb_cache disable_bdb_cache() from utilities import get_im from morphology import adaptive_mask, binarize v = get_im( args[0] ) m = adaptive_mask( v , 2.0, 2) bm = binarize( m, 0.5 ) bm.write_image( args[1] ) adaptive_mask( v , 2.0, 2, 9, 3).write_image( args[2] ) if(options.variance != None): from fundamentals import rot_avg_image from morphology import square_root trovc = rot_avg_image(get_im(options.variance)) nc = trovc.get_xsize()//2 trovc /= trovc[nc,nc,nc] square_root(trovc).write_image( options.repair )
def do_volume_mrk02(ref_data): """ data - projections (scattered between cpus) or the volume. If volume, just do the volume processing options - the same for all cpus return - volume the same for all cpus """ from EMAN2 import Util from mpi import mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD from filter import filt_table from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI from utilities import bcast_EMData_to_all, bcast_number_to_all, model_blank from fundamentals import rops_table, fftip, fft import types # Retrieve the function specific input arguments from ref_data data = ref_data[0] Tracker = ref_data[1] iter = ref_data[2] mpi_comm = ref_data[3] # # For DEBUG # print "Type of data %s" % (type(data)) # print "Type of Tracker %s" % (type(Tracker)) # print "Type of iter %s" % (type(iter)) # print "Type of mpi_comm %s" % (type(mpi_comm)) if(mpi_comm == None): mpi_comm = MPI_COMM_WORLD myid = mpi_comm_rank(mpi_comm) nproc = mpi_comm_size(mpi_comm) try: local_filter = Tracker["local_filter"] except: local_filter = False #========================================================================= # volume reconstruction if( type(data) == types.ListType ): if Tracker["constants"]["CTF"]: vol = recons3d_4nn_ctf_MPI(myid, data, Tracker["constants"]["snr"], \ symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm, smearstep = Tracker["smearstep"]) else: vol = recons3d_4nn_MPI (myid, data,\ symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm) else: vol = data if myid == 0: from morphology import threshold from filter import filt_tanl, filt_btwl from utilities import model_circle, get_im import types nx = vol.get_xsize() if(Tracker["constants"]["mask3D"] == None): mask3D = model_circle(int(Tracker["constants"]["radius"]*float(nx)/float(Tracker["constants"]["nnxo"])+0.5), nx, nx, nx) elif(Tracker["constants"]["mask3D"] == "auto"): from utilities import adaptive_mask mask3D = adaptive_mask(vol) else: if( type(Tracker["constants"]["mask3D"]) == types.StringType ): mask3D = get_im(Tracker["constants"]["mask3D"]) else: mask3D = (Tracker["constants"]["mask3D"]).copy() nxm = mask3D.get_xsize() if( nx != nxm): from fundamentals import rot_shift3D mask3D = Util.window(rot_shift3D(mask3D,scale=float(nx)/float(nxm)),nx,nx,nx) nxm = mask3D.get_xsize() assert(nx == nxm) stat = Util.infomask(vol, mask3D, False) vol -= stat[0] Util.mul_scalar(vol, 1.0/stat[1]) vol = threshold(vol) Util.mul_img(vol, mask3D) if( Tracker["PWadjustment"] ): from utilities import read_text_file, write_text_file rt = read_text_file( Tracker["PWadjustment"] ) fftip(vol) ro = rops_table(vol) # Here unless I am mistaken it is enough to take the beginning of the reference pw. for i in xrange(1,len(ro)): ro[i] = (rt[i]/ro[i])**Tracker["upscale"] #write_text_file(rops_table(filt_table( vol, ro),1),"foo.txt") if Tracker["constants"]["sausage"]: ny = vol.get_ysize() y = float(ny) from math import exp for i in xrange(len(ro)): ro[i] *= \ (1.0+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.10)/0.025)**2)+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.215)/0.025)**2)) if local_filter: # skip low-pass filtration vol = fft( filt_table( vol, ro) ) else: if( type(Tracker["lowpass"]) == types.ListType ): vol = fft( filt_table( filt_table(vol, Tracker["lowpass"]), ro) ) else: vol = fft( filt_table( filt_tanl(vol, Tracker["lowpass"], Tracker["falloff"]), ro) ) del ro else: if Tracker["constants"]["sausage"]: ny = vol.get_ysize() y = float(ny) ro = [0.0]*(ny//2+2) from math import exp for i in xrange(len(ro)): ro[i] = \ (1.0+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.10)/0.025)**2)+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.215)/0.025)**2)) fftip(vol) filt_table(vol, ro) del ro if not local_filter: if( type(Tracker["lowpass"]) == types.ListType ): vol = filt_table(vol, Tracker["lowpass"]) else: vol = filt_tanl(vol, Tracker["lowpass"], Tracker["falloff"]) if Tracker["constants"]["sausage"]: vol = fft(vol) if local_filter: from morphology import binarize if(myid == 0): nx = mask3D.get_xsize() else: nx = 0 nx = bcast_number_to_all(nx, source_node = 0) # only main processor needs the two input volumes if(myid == 0): mask = binarize(mask3D, 0.5) locres = get_im(Tracker["local_filter"]) lx = locres.get_xsize() if(lx != nx): if(lx < nx): from fundamentals import fdecimate, rot_shift3D mask = Util.window(rot_shift3D(mask,scale=float(lx)/float(nx)),lx,lx,lx) vol = fdecimate(vol, lx,lx,lx) else: ERROR("local filter cannot be larger than input volume","user function",1) stat = Util.infomask(vol, mask, False) vol -= stat[0] Util.mul_scalar(vol, 1.0/stat[1]) else: lx = 0 locres = model_blank(1,1,1) vol = model_blank(1,1,1) lx = bcast_number_to_all(lx, source_node = 0) if( myid != 0 ): mask = model_blank(lx,lx,lx) bcast_EMData_to_all(mask, myid, 0, comm=mpi_comm) from filter import filterlocal vol = filterlocal( locres, vol, mask, Tracker["falloff"], myid, 0, nproc) if myid == 0: if(lx < nx): from fundamentals import fpol vol = fpol(vol, nx,nx,nx) vol = threshold(vol) vol = filt_btwl(vol, 0.38, 0.5)# This will have to be corrected. Util.mul_img(vol, mask3D) del mask3D # vol.write_image('toto%03d.hdf'%iter) else: vol = model_blank(nx,nx,nx) else: if myid == 0: #from utilities import write_text_file #write_text_file(rops_table(vol,1),"goo.txt") stat = Util.infomask(vol, mask3D, False) vol -= stat[0] Util.mul_scalar(vol, 1.0/stat[1]) vol = threshold(vol) vol = filt_btwl(vol, 0.38, 0.5)# This will have to be corrected. Util.mul_img(vol, mask3D) del mask3D # vol.write_image('toto%03d.hdf'%iter) # broadcast volume bcast_EMData_to_all(vol, myid, 0, comm=mpi_comm) #========================================================================= return vol
def do_volume_mrk03(ref_data): """ data - projections (scattered between cpus) or the volume. If volume, just do the volume processing options - the same for all cpus return - volume the same for all cpus """ from EMAN2 import Util from mpi import mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD from filter import filt_table from reconstruction import recons3d_4nn_MPI, recons3d_4nnw_MPI # recons3d_4nn_ctf_MPI from utilities import bcast_EMData_to_all, bcast_number_to_all, model_blank from fundamentals import rops_table, fftip, fft import types # Retrieve the function specific input arguments from ref_data data = ref_data[0] Tracker = ref_data[1] iter = ref_data[2] mpi_comm = ref_data[3] # # For DEBUG # print "Type of data %s" % (type(data)) # print "Type of Tracker %s" % (type(Tracker)) # print "Type of iter %s" % (type(iter)) # print "Type of mpi_comm %s" % (type(mpi_comm)) if(mpi_comm == None): mpi_comm = MPI_COMM_WORLD myid = mpi_comm_rank(mpi_comm) nproc = mpi_comm_size(mpi_comm) try: local_filter = Tracker["local_filter"] except: local_filter = False #========================================================================= # volume reconstruction if( type(data) == types.ListType ): if Tracker["constants"]["CTF"]: #vol = recons3d_4nn_ctf_MPI(myid, data, Tracker["constants"]["snr"], \ # symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm, smearstep = Tracker["smearstep"]) vol = recons3d_4nnw_MPI(myid, data, Tracker["bckgnoise"], Tracker["constants"]["snr"], \ symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm, smearstep = Tracker["smearstep"]) else: vol = recons3d_4nn_MPI (myid, data,\ symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm) else: vol = data if myid == 0: from morphology import threshold from filter import filt_tanl, filt_btwl from utilities import model_circle, get_im import types nx = vol.get_xsize() if(Tracker["constants"]["mask3D"] == None): mask3D = model_circle(int(Tracker["constants"]["radius"]*float(nx)/float(Tracker["constants"]["nnxo"])+0.5), nx, nx, nx) elif(Tracker["constants"]["mask3D"] == "auto"): from utilities import adaptive_mask mask3D = adaptive_mask(vol) else: if( type(Tracker["constants"]["mask3D"]) == types.StringType ): mask3D = get_im(Tracker["constants"]["mask3D"]) else: mask3D = (Tracker["constants"]["mask3D"]).copy() nxm = mask3D.get_xsize() if( nx != nxm): from fundamentals import rot_shift3D mask3D = Util.window(rot_shift3D(mask3D,scale=float(nx)/float(nxm)),nx,nx,nx) nxm = mask3D.get_xsize() assert(nx == nxm) stat = Util.infomask(vol, mask3D, False) vol -= stat[0] Util.mul_scalar(vol, 1.0/stat[1]) vol = threshold(vol) Util.mul_img(vol, mask3D) if not local_filter: if( type(Tracker["lowpass"]) == types.ListType ): vol = filt_table(vol, Tracker["lowpass"]) else: vol = filt_tanl(vol, Tracker["lowpass"], Tracker["falloff"]) if local_filter: from morphology import binarize if(myid == 0): nx = mask3D.get_xsize() else: nx = 0 nx = bcast_number_to_all(nx, source_node = 0) # only main processor needs the two input volumes if(myid == 0): mask = binarize(mask3D, 0.5) locres = get_im(Tracker["local_filter"]) lx = locres.get_xsize() if(lx != nx): if(lx < nx): from fundamentals import fdecimate, rot_shift3D mask = Util.window(rot_shift3D(mask,scale=float(lx)/float(nx)),lx,lx,lx) vol = fdecimate(vol, lx,lx,lx) else: ERROR("local filter cannot be larger than input volume","user function",1) stat = Util.infomask(vol, mask, False) vol -= stat[0] Util.mul_scalar(vol, 1.0/stat[1]) else: lx = 0 locres = model_blank(1,1,1) vol = model_blank(1,1,1) lx = bcast_number_to_all(lx, source_node = 0) if( myid != 0 ): mask = model_blank(lx,lx,lx) bcast_EMData_to_all(mask, myid, 0, comm=mpi_comm) from filter import filterlocal vol = filterlocal( locres, vol, mask, Tracker["falloff"], myid, 0, nproc) if myid == 0: if(lx < nx): from fundamentals import fpol vol = fpol(vol, nx,nx,nx) vol = threshold(vol) Util.mul_img(vol, mask3D) del mask3D # vol.write_image('toto%03d.hdf'%iter) else: vol = model_blank(nx,nx,nx) """ else: if myid == 0: #from utilities import write_text_file #write_text_file(rops_table(vol,1),"goo.txt") stat = Util.infomask(vol, mask3D, False) vol -= stat[0] Util.mul_scalar(vol, 1.0/stat[1]) vol = threshold(vol) Util.mul_img(vol, mask3D) del mask3D # vol.write_image('toto%03d.hdf'%iter) """ # broadcast volume bcast_EMData_to_all(vol, myid, 0, comm=mpi_comm) #========================================================================= return vol
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + """ firstvolume secondvolume maskfile outputfile --wn --step --cutoff --radius --fsc --res_overall --out_ang_res --apix --MPI Compute local resolution in real space within area outlined by the maskfile and within regions wn x wn x wn """ parser = optparse.OptionParser(usage, version=global_def.SPARXVERSION) parser.add_option( "--wn", type="int", default=7, help= "Size of window within which local real-space FSC is computed. (default 7)" ) parser.add_option( "--step", type="float", default=1.0, help="Shell step in Fourier size in pixels. (default 1.0)") parser.add_option("--cutoff", type="float", default=0.5, help="Resolution cut-off for FSC. (default 0.5)") parser.add_option( "--radius", type="int", default=-1, help= "If there is no maskfile, sphere with r=radius will be used. By default, the radius is nx/2-wn (default -1)" ) parser.add_option( "--fsc", type="string", default=None, help= "Save overall FSC curve (might be truncated). By default, the program does not save the FSC curve. (default none)" ) parser.add_option( "--res_overall", type="float", default=-1.0, help= "Overall resolution at the cutoff level estimated by the user [abs units]. (default None)" ) parser.add_option( "--out_ang_res", action="store_true", default=False, help= "Additionally creates a local resolution file in Angstroms. (default False)" ) parser.add_option( "--apix", type="float", default=1.0, help= "Pixel size in Angstrom. Effective only with --out_ang_res options. (default 1.0)" ) parser.add_option("--MPI", action="store_true", default=False, help="Use MPI version.") (options, args) = parser.parse_args(arglist[1:]) if len(args) < 3 or len(args) > 4: print("See usage " + usage) sys.exit() if global_def.CACHE_DISABLE: utilities.disable_bdb_cache() res_overall = options.res_overall if options.MPI: sys.argv = mpi.mpi_init(len(sys.argv), sys.argv) number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD) myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD) main_node = 0 global_def.MPI = True cutoff = options.cutoff nk = int(options.wn) if (myid == main_node): #print sys.argv vi = utilities.get_im(sys.argv[1]) ui = utilities.get_im(sys.argv[2]) nx = vi.get_xsize() ny = vi.get_ysize() nz = vi.get_zsize() dis = [nx, ny, nz] else: dis = [0, 0, 0, 0] global_def.BATCH = True dis = utilities.bcast_list_to_all(dis, myid, source_node=main_node) if (myid != main_node): nx = int(dis[0]) ny = int(dis[1]) nz = int(dis[2]) vi = utilities.model_blank(nx, ny, nz) ui = utilities.model_blank(nx, ny, nz) if len(args) == 3: m = utilities.model_circle((min(nx, ny, nz) - nk) // 2, nx, ny, nz) outvol = args[2] elif len(args) == 4: if (myid == main_node): m = morphology.binarize(utilities.get_im(args[2]), 0.5) else: m = utilities.model_blank(nx, ny, nz) outvol = args[3] utilities.bcast_EMData_to_all(m, myid, main_node) """Multiline Comment0""" freqvol, resolut = statistics.locres(vi, ui, m, nk, cutoff, options.step, myid, main_node, number_of_proc) if (myid == 0): # Remove outliers based on the Interquartile range output_volume(freqvol, resolut, options.apix, outvol, options.fsc, options.out_ang_res, nx, ny, nz, res_overall) mpi.mpi_finalize() else: cutoff = options.cutoff vi = utilities.get_im(args[0]) ui = utilities.get_im(args[1]) nn = vi.get_xsize() nk = int(options.wn) if len(args) == 3: m = utilities.model_circle((nn - nk) // 2, nn, nn, nn) outvol = args[2] elif len(args) == 4: m = morphology.binarize(utilities.get_im(args[2]), 0.5) outvol = args[3] mc = utilities.model_blank(nn, nn, nn, 1.0) - m vf = fundamentals.fft(vi) uf = fundamentals.fft(ui) """Multiline Comment1""" lp = int(nn / 2 / options.step + 0.5) step = 0.5 / lp freqvol = utilities.model_blank(nn, nn, nn) resolut = [] for i in range(1, lp): fl = step * i fh = fl + step #print(lp,i,step,fl,fh) v = fundamentals.fft(filter.filt_tophatb(vf, fl, fh)) u = fundamentals.fft(filter.filt_tophatb(uf, fl, fh)) tmp1 = EMAN2_cppwrap.Util.muln_img(v, v) tmp2 = EMAN2_cppwrap.Util.muln_img(u, u) do = EMAN2_cppwrap.Util.infomask( morphology.square_root( morphology.threshold( EMAN2_cppwrap.Util.muln_img(tmp1, tmp2))), m, True)[0] tmp3 = EMAN2_cppwrap.Util.muln_img(u, v) dp = EMAN2_cppwrap.Util.infomask(tmp3, m, True)[0] resolut.append([i, (fl + fh) / 2.0, dp / do]) tmp1 = EMAN2_cppwrap.Util.box_convolution(tmp1, nk) tmp2 = EMAN2_cppwrap.Util.box_convolution(tmp2, nk) tmp3 = EMAN2_cppwrap.Util.box_convolution(tmp3, nk) EMAN2_cppwrap.Util.mul_img(tmp1, tmp2) tmp1 = morphology.square_root(morphology.threshold(tmp1)) EMAN2_cppwrap.Util.mul_img(tmp1, m) EMAN2_cppwrap.Util.add_img(tmp1, mc) EMAN2_cppwrap.Util.mul_img(tmp3, m) EMAN2_cppwrap.Util.add_img(tmp3, mc) EMAN2_cppwrap.Util.div_img(tmp3, tmp1) EMAN2_cppwrap.Util.mul_img(tmp3, m) freq = (fl + fh) / 2.0 bailout = True for x in range(nn): for y in range(nn): for z in range(nn): if (m.get_value_at(x, y, z) > 0.5): if (freqvol.get_value_at(x, y, z) == 0.0): if (tmp3.get_value_at(x, y, z) < cutoff): freqvol.set_value_at(x, y, z, freq) bailout = False else: bailout = False if (bailout): break #print(len(resolut)) # remove outliers output_volume(freqvol, resolut, options.apix, outvol, options.fsc, options.out_ang_res, nx, ny, nz, res_overall)
def get_adaptive_slab_mask(volume, membrane_height): ''' Creates an adaptive mask which detects the membrane density and cuts regions accordingly. ============= z-height .... ------------- 1st cut: user defined membrane height ..... ------------- 2nd cut: density detected cut ;;;;;;; ---/\./\./\-- 3rd cut: thresholding to detect noise ;;;;;;; ---oooooooo-- Image z-center SAME CUTS ON Lower HALF ''' print 'Adaptively masking the membrane..' nx = volume.get_xsize() ny = volume.get_ysize() nz = volume.get_zsize() z_center = int(nz / 2) - 1 # Calculate first cut first_cut = int((nz - membrane_height) / 2) - 1 # Calculate second cut second_cut = first_cut # Niko Grigorieff says that for alignments highest resolution should not be more than 15 A vol_lp = volume.low_pass(1.0 / 15.0) vol_lp_thrs = EMVol(vol_lp * binarize(vol_lp, vol_lp.get_mean() + vol_lp.get_std())) dens_profile = vol_lp_thrs.z_density_profile() dens_profile_change = [d1 - d2 for d1, d2 in zip(dens_profile[1:], dens_profile)] change_max_indices = get_local_maxima(dens_profile_change) if(change_max_indices[0] < z_center): second_cut = change_max_indices[0] - 20 # Breathing space of 20 voxels ''' change_max_possible = [] for i in change_max_indices: if i < z_center: change_max_possible.append(i) second_cut = dens_profile_change.index(max([dens_profile_change[c] for c in change_max_possible])) ''' if (second_cut < first_cut): second_cut = first_cut # Calculate third cut third_cut = z_center # third_cut = second_cut + int(nz*0.1) # if(third_cut >= z_center): # third_cut = z_center print 'Cuts applied: {} {} {}' .format(first_cut, second_cut, third_cut) # Generate the mask threshold_mask = binarize(volume, volume.get_mean() + 0.5 * volume.get_std()) mask = model_blank(nx, ny, nz) for ix in range(0, nx): for iy in range(0, ny): for iz in range(third_cut, nz - third_cut): mask[ix, iy, iz] = 1 for ix in range(0, nx): for iy in range(0, ny): for iz in range(second_cut, third_cut) + range(nz - third_cut, nz - second_cut): mask[ix, iy, iz] = threshold_mask[ix, iy, iz] return mask
def do_volume_mrk02(ref_data): """ data - projections (scattered between cpus) or the volume. If volume, just do the volume processing options - the same for all cpus return - volume the same for all cpus """ from EMAN2 import Util from mpi import mpi_comm_rank, mpi_comm_size, MPI_COMM_WORLD from filter import filt_table from reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI from utilities import bcast_EMData_to_all, bcast_number_to_all, model_blank from fundamentals import rops_table, fftip, fft import types # Retrieve the function specific input arguments from ref_data data = ref_data[0] Tracker = ref_data[1] iter = ref_data[2] mpi_comm = ref_data[3] # # For DEBUG # print "Type of data %s" % (type(data)) # print "Type of Tracker %s" % (type(Tracker)) # print "Type of iter %s" % (type(iter)) # print "Type of mpi_comm %s" % (type(mpi_comm)) if (mpi_comm == None): mpi_comm = MPI_COMM_WORLD myid = mpi_comm_rank(mpi_comm) nproc = mpi_comm_size(mpi_comm) try: local_filter = Tracker["local_filter"] except: local_filter = False #========================================================================= # volume reconstruction if (type(data) == types.ListType): if Tracker["constants"]["CTF"]: vol = recons3d_4nn_ctf_MPI(myid, data, Tracker["constants"]["snr"], \ symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm, smearstep = Tracker["smearstep"]) else: vol = recons3d_4nn_MPI (myid, data,\ symmetry=Tracker["constants"]["sym"], npad=Tracker["constants"]["npad"], mpi_comm=mpi_comm) else: vol = data if myid == 0: from morphology import threshold from filter import filt_tanl, filt_btwl from utilities import model_circle, get_im import types nx = vol.get_xsize() if (Tracker["constants"]["mask3D"] == None): mask3D = model_circle( int(Tracker["constants"]["radius"] * float(nx) / float(Tracker["constants"]["nnxo"]) + 0.5), nx, nx, nx) elif (Tracker["constants"]["mask3D"] == "auto"): from utilities import adaptive_mask mask3D = adaptive_mask(vol) else: if (type(Tracker["constants"]["mask3D"]) == types.StringType): mask3D = get_im(Tracker["constants"]["mask3D"]) else: mask3D = (Tracker["constants"]["mask3D"]).copy() nxm = mask3D.get_xsize() if (nx != nxm): from fundamentals import rot_shift3D mask3D = Util.window( rot_shift3D(mask3D, scale=float(nx) / float(nxm)), nx, nx, nx) nxm = mask3D.get_xsize() assert (nx == nxm) stat = Util.infomask(vol, mask3D, False) vol -= stat[0] Util.mul_scalar(vol, 1.0 / stat[1]) vol = threshold(vol) Util.mul_img(vol, mask3D) if (Tracker["PWadjustment"]): from utilities import read_text_file, write_text_file rt = read_text_file(Tracker["PWadjustment"]) fftip(vol) ro = rops_table(vol) # Here unless I am mistaken it is enough to take the beginning of the reference pw. for i in xrange(1, len(ro)): ro[i] = (rt[i] / ro[i])**Tracker["upscale"] #write_text_file(rops_table(filt_table( vol, ro),1),"foo.txt") if Tracker["constants"]["sausage"]: ny = vol.get_ysize() y = float(ny) from math import exp for i in xrange(len(ro)): ro[i] *= \ (1.0+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.10)/0.025)**2)+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.215)/0.025)**2)) if local_filter: # skip low-pass filtration vol = fft(filt_table(vol, ro)) else: if (type(Tracker["lowpass"]) == types.ListType): vol = fft( filt_table(filt_table(vol, Tracker["lowpass"]), ro)) else: vol = fft( filt_table( filt_tanl(vol, Tracker["lowpass"], Tracker["falloff"]), ro)) del ro else: if Tracker["constants"]["sausage"]: ny = vol.get_ysize() y = float(ny) ro = [0.0] * (ny // 2 + 2) from math import exp for i in xrange(len(ro)): ro[i] = \ (1.0+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.10)/0.025)**2)+1.0*exp(-(((i/y/Tracker["constants"]["pixel_size"])-0.215)/0.025)**2)) fftip(vol) filt_table(vol, ro) del ro if not local_filter: if (type(Tracker["lowpass"]) == types.ListType): vol = filt_table(vol, Tracker["lowpass"]) else: vol = filt_tanl(vol, Tracker["lowpass"], Tracker["falloff"]) if Tracker["constants"]["sausage"]: vol = fft(vol) if local_filter: from morphology import binarize if (myid == 0): nx = mask3D.get_xsize() else: nx = 0 nx = bcast_number_to_all(nx, source_node=0) # only main processor needs the two input volumes if (myid == 0): mask = binarize(mask3D, 0.5) locres = get_im(Tracker["local_filter"]) lx = locres.get_xsize() if (lx != nx): if (lx < nx): from fundamentals import fdecimate, rot_shift3D mask = Util.window( rot_shift3D(mask, scale=float(lx) / float(nx)), lx, lx, lx) vol = fdecimate(vol, lx, lx, lx) else: ERROR("local filter cannot be larger than input volume", "user function", 1) stat = Util.infomask(vol, mask, False) vol -= stat[0] Util.mul_scalar(vol, 1.0 / stat[1]) else: lx = 0 locres = model_blank(1, 1, 1) vol = model_blank(1, 1, 1) lx = bcast_number_to_all(lx, source_node=0) if (myid != 0): mask = model_blank(lx, lx, lx) bcast_EMData_to_all(mask, myid, 0, comm=mpi_comm) from filter import filterlocal vol = filterlocal(locres, vol, mask, Tracker["falloff"], myid, 0, nproc) if myid == 0: if (lx < nx): from fundamentals import fpol vol = fpol(vol, nx, nx, nx) vol = threshold(vol) vol = filt_btwl(vol, 0.38, 0.5) # This will have to be corrected. Util.mul_img(vol, mask3D) del mask3D # vol.write_image('toto%03d.hdf'%iter) else: vol = model_blank(nx, nx, nx) else: if myid == 0: #from utilities import write_text_file #write_text_file(rops_table(vol,1),"goo.txt") stat = Util.infomask(vol, mask3D, False) vol -= stat[0] Util.mul_scalar(vol, 1.0 / stat[1]) vol = threshold(vol) vol = filt_btwl(vol, 0.38, 0.5) # This will have to be corrected. Util.mul_img(vol, mask3D) del mask3D # vol.write_image('toto%03d.hdf'%iter) # broadcast volume bcast_EMData_to_all(vol, myid, 0, comm=mpi_comm) #========================================================================= return vol
def main(): import sys import os import math import random import pyemtbx.options import time from random import random, seed, randint from optparse import OptionParser progname = os.path.basename(sys.argv[0]) usage = progname + """ [options] <inputfile> <outputfile> Generic 2-D image processing programs. Functionality: 1. Phase flip a stack of images and write output to new file: sxprocess.py input_stack.hdf output_stack.hdf --phase_flip 2. Resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size. The window size will change accordingly. sxprocess input.hdf output.hdf --changesize --ratio=0.5 3. Compute average power spectrum of a stack of 2D images with optional padding (option wn) with zeroes or a 3-D volume. sxprocess.py input_stack.hdf powerspectrum.hdf --pw [--wn=1024] 4. Generate a stack of projections bdb:data and micrographs with prefix mic (i.e., mic0.hdf, mic1.hdf etc) from structure input_structure.hdf, with CTF applied to both projections and micrographs: sxprocess.py input_structure.hdf data mic --generate_projections format="bdb":apix=5.2:CTF=True:boxsize=64 5. Retrieve original image numbers in the selected ISAC group (here group 12 from generation 3): sxprocess.py bdb:test3 class_averages_generation_3.hdf list3_12.txt --isacgroup=12 --params=originalid 6. Retrieve original image numbers of images listed in ISAC output stack of averages: sxprocess.py select1.hdf ohk.txt 7. Adjust rotationally averaged power spectrum of an image to that of a reference image or a reference 1D power spectrum stored in an ASCII file. Optionally use a tangent low-pass filter. Also works for a stack of images, in which case the output is also a stack. sxprocess.py vol.hdf ref.hdf avol.hdf < 0.25 0.2> --adjpw sxprocess.py vol.hdf pw.txt avol.hdf < 0.25 0.2> --adjpw 8. Generate a 1D rotationally averaged power spectrum of an image. sxprocess.py vol.hdf --rotwp=rotpw.txt # Output will contain three columns: (1) rotationally averaged power spectrum (2) logarithm of the rotationally averaged power spectrum (3) integer line number (from zero to approximately to half the image size) 9. Apply 3D transformation (rotation and/or shift) to a set of orientation parameters associated with projection data. sxprocess.py --transfromparams=phi,theta,psi,tx,ty,tz input.txt output.txt The output file is then imported and 3D transformed volume computed: sxheader.py bdb:p --params=xform.projection --import=output.txt mpirun -np 2 sxrecons3d_n.py bdb:p tvol.hdf --MPI The reconstructed volume is in the position of the volume computed using the input.txt parameters and then transformed with rot_shift3D(vol, phi,theta,psi,tx,ty,tz) 10. Import ctf parameters from the output of sxcter into windowed particle headers. There are three possible input files formats: (1) all particles are in one stack, (2 aor 3) particles are in stacks, each stack corresponds to a single micrograph. In each case the particles should contain a name of the micrograph of origin stores using attribute name 'ptcl_source_image'. Normally this is done by e2boxer.py during windowing. Particles whose defocus or astigmatism error exceed set thresholds will be skipped, otherwise, virtual stacks with the original way preceded by G will be created. sxprocess.py --input=bdb:data --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 # Output will be a vritual stack bdb:Gdata sxprocess.py --input="bdb:directory/stacks*" --importctf=outdir/partres --defocuserror=10.0 --astigmatismerror=5.0 To concatenate output files: cd directory e2bdb.py . --makevstack=bdb:allparticles --filt=G IMPORTANT: Please do not move (or remove!) any input/intermediate EMAN2DB files as the information is linked between them. 11. Scale 3D shifts. The shifts in the input five columns text file with 3D orientation parameters will be DIVIDED by the scale factor sxprocess.py orientationparams.txt scaledparams.txt scale=0.5 12. Generate 3D mask from a given 3-D volume automatically or using threshold provided by user. 13. Postprocess 3-D or 2-D images: for 3-D volumes: calculate FSC with provided mask; weight summed volume with FSC; estimate B-factor from FSC weighted summed two volumes; apply negative B-factor to the weighted volume. for 2-D images: calculate B-factor and apply negative B-factor to 2-D images. 14. Winow stack file -reduce size of images without changing the pixel size. """ parser = OptionParser(usage,version=SPARXVERSION) parser.add_option("--order", action="store_true", help="Two arguments are required: name of input stack and desired name of output stack. The output stack is the input stack sorted by similarity in terms of cross-correlation coefficent.", default=False) parser.add_option("--order_lookup", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_metropolis", action="store_true", help="Test/Debug.", default=False) parser.add_option("--order_pca", action="store_true", help="Test/Debug.", default=False) parser.add_option("--initial", type="int", default=-1, help="Specifies which image will be used as an initial seed to form the chain. (default = 0, means the first image)") parser.add_option("--circular", action="store_true", help="Select circular ordering (fisr image has to be similar to the last", default=False) parser.add_option("--radius", type="int", default=-1, help="Radius of a circular mask for similarity based ordering") parser.add_option("--changesize", action="store_true", help="resample (decimate or interpolate up) images (2D or 3D) in a stack to change the pixel size.", default=False) parser.add_option("--ratio", type="float", default=1.0, help="The ratio of new to old image size (if <1 the pixel size will increase and image size decrease, if>1, the other way round") parser.add_option("--pw", action="store_true", help="compute average power spectrum of a stack of 2-D images with optional padding (option wn) with zeroes", default=False) parser.add_option("--wn", type="int", default=-1, help="Size of window to use (should be larger/equal than particle box size, default padding to max(nx,ny))") parser.add_option("--phase_flip", action="store_true", help="Phase flip the input stack", default=False) parser.add_option("--makedb", metavar="param1=value1:param2=value2", type="string", action="append", help="One argument is required: name of key with which the database will be created. Fill in database with parameters specified as follows: --makedb param1=value1:param2=value2, e.g. 'gauss_width'=1.0:'pixel_input'=5.2:'pixel_output'=5.2:'thr_low'=1.0") parser.add_option("--generate_projections", metavar="param1=value1:param2=value2", type="string", action="append", help="Three arguments are required: name of input structure from which to generate projections, desired name of output projection stack, and desired prefix for micrographs (e.g. if prefix is 'mic', then micrographs mic0.hdf, mic1.hdf etc will be generated). Optional arguments specifying format, apix, box size and whether to add CTF effects can be entered as follows after --generate_projections: format='bdb':apix=5.2:CTF=True:boxsize=100, or format='hdf', etc., where format is bdb or hdf, apix (pixel size) is a float, CTF is True or False, and boxsize denotes the dimension of the box (assumed to be a square). If an optional parameter is not specified, it will default as follows: format='bdb', apix=2.5, CTF=False, boxsize=64.") parser.add_option("--isacgroup", type="int", help="Retrieve original image numbers in the selected ISAC group. See ISAC documentation for details.", default=-1) parser.add_option("--isacselect", action="store_true", help="Retrieve original image numbers of images listed in ISAC output stack of averages. See ISAC documentation for details.", default=False) parser.add_option("--params", type="string", default=None, help="Name of header of parameter, which one depends on specific option") parser.add_option("--adjpw", action="store_true", help="Adjust rotationally averaged power spectrum of an image", default=False) parser.add_option("--rotpw", type="string", default=None, help="Name of the text file to contain rotationally averaged power spectrum of the input image.") parser.add_option("--transformparams", type="string", default=None, help="Transform 3D projection orientation parameters using six 3D parameters (phi, theta,psi,sx,sy,sz). Input: --transformparams=45.,66.,12.,-2,3,-5.5 desired six transformation of the reconstructed structure. Output: file with modified orientation parameters.") # import ctf estimates done using cter parser.add_option("--input", type="string", default= None, help="Input particles.") parser.add_option("--importctf", type="string", default= None, help="Name of the file containing CTF parameters produced by sxcter.") parser.add_option("--defocuserror", type="float", default=1000000.0, help="Exclude micrographs whose relative defocus error as estimated by sxcter is larger than defocuserror percent. The error is computed as (std dev defocus)/defocus*100%") parser.add_option("--astigmatismerror", type="float", default=360.0, help="Set to zero astigmatism for micrographs whose astigmatism angular error as estimated by sxcter is larger than astigmatismerror degrees.") # import ctf estimates done using cter parser.add_option("--scale", type="float", default=-1.0, help="Divide shifts in the input 3D orientation parameters text file by the scale factor.") # generate adaptive mask from an given 3-D volume parser.add_option("--adaptive_mask", action="store_true", help="create adavptive 3-D mask from a given volume", default=False) parser.add_option("--nsigma", type="float", default= 1., help="number of times of sigma of the input volume to obtain the the large density cluster") parser.add_option("--ndilation", type="int", default= 3, help="number of times of dilation applied to the largest cluster of density") parser.add_option("--kernel_size", type="int", default= 11, help="convolution kernel for smoothing the edge of the mask") parser.add_option("--gauss_standard_dev", type="int", default= 9, help="stanadard deviation value to generate Gaussian edge") parser.add_option("--threshold", type="float", default= 9999., help="threshold provided by user to binarize input volume") parser.add_option("--ne", type="int", default= 0, help="number of times to erode the binarized input image") parser.add_option("--nd", type="int", default= 0, help="number of times to dilate the binarized input image") parser.add_option("--postprocess", action="store_true", help="postprocess unfiltered odd, even 3-D volumes",default=False) parser.add_option("--fsc_weighted", action="store_true", help="postprocess unfiltered odd, even 3-D volumes") parser.add_option("--low_pass_filter", action="store_true", default=False, help="postprocess unfiltered odd, even 3-D volumes") parser.add_option("--ff", type="float", default=.25, help="low pass filter stop band frequency in absolute unit") parser.add_option("--aa", type="float", default=.1, help="low pass filter falloff" ) parser.add_option("--mask", type="string", help="input mask file", default=None) parser.add_option("--output", type="string", help="output file name", default="postprocessed.hdf") parser.add_option("--pixel_size", type="float", help="pixel size of the data", default=1.0) parser.add_option("--B_start", type="float", help="starting frequency in Angstrom for B-factor estimation", default=10.) parser.add_option("--FSC_cutoff", type="float", help="stop frequency in Angstrom for B-factor estimation", default=0.143) parser.add_option("--2d", action="store_true", help="postprocess isac 2-D averaged images",default=False) parser.add_option("--window_stack", action="store_true", help="window stack images using a smaller window size", default=False) parser.add_option("--box", type="int", default= 0, help="the new window size ") (options, args) = parser.parse_args() global_def.BATCH = True if options.phase_flip: nargs = len(args) if nargs != 2: print "must provide name of input and output file!" return from EMAN2 import Processor instack = args[0] outstack = args[1] nima = EMUtil.get_image_count(instack) from filter import filt_ctf for i in xrange(nima): img = EMData() img.read_image(instack, i) try: ctf = img.get_attr('ctf') except: print "no ctf information in input stack! Exiting..." return dopad = True sign = 1 binary = 1 # phase flip assert img.get_ysize() > 1 dict = ctf.to_dict() dz = dict["defocus"] cs = dict["cs"] voltage = dict["voltage"] pixel_size = dict["apix"] b_factor = dict["bfactor"] ampcont = dict["ampcont"] dza = dict["dfdiff"] azz = dict["dfang"] if dopad and not img.is_complex(): ip = 1 else: ip = 0 params = {"filter_type": Processor.fourier_filter_types.CTF_, "defocus" : dz, "Cs": cs, "voltage": voltage, "Pixel_size": pixel_size, "B_factor": b_factor, "amp_contrast": ampcont, "dopad": ip, "binary": binary, "sign": sign, "dza": dza, "azz":azz} tmp = Processor.EMFourierFilter(img, params) tmp.set_attr_dict({"ctf": ctf}) tmp.write_image(outstack, i) elif options.changesize: nargs = len(args) if nargs != 2: ERROR("must provide name of input and output file!", "change size", 1) return from utilities import get_im instack = args[0] outstack = args[1] sub_rate = float(options.ratio) nima = EMUtil.get_image_count(instack) from fundamentals import resample for i in xrange(nima): resample(get_im(instack, i), sub_rate).write_image(outstack, i) elif options.isacgroup>-1: nargs = len(args) if nargs != 3: ERROR("Three files needed on input!", "isacgroup", 1) return from utilities import get_im instack = args[0] m=get_im(args[1],int(options.isacgroup)).get_attr("members") l = [] for k in m: l.append(int(get_im(args[0],k).get_attr(options.params))) from utilities import write_text_file write_text_file(l, args[2]) elif options.isacselect: nargs = len(args) if nargs != 2: ERROR("Two files needed on input!", "isacgroup", 1) return from utilities import get_im nima = EMUtil.get_image_count(args[0]) m = [] for k in xrange(nima): m += get_im(args[0],k).get_attr("members") m.sort() from utilities import write_text_file write_text_file(m, args[1]) elif options.pw: nargs = len(args) if nargs < 2: ERROR("must provide name of input and output file!", "pw", 1) return from utilities import get_im, write_text_file from fundamentals import rops_table d = get_im(args[0]) ndim = d.get_ndim() if ndim ==3: pw = rops_table(d) write_text_file(pw, args[1]) else: nx = d.get_xsize() ny = d.get_ysize() if nargs ==3: mask = get_im(args[2]) wn = int(options.wn) if wn == -1: wn = max(nx, ny) else: if( (wn<nx) or (wn<ny) ): ERROR("window size cannot be smaller than the image size","pw",1) n = EMUtil.get_image_count(args[0]) from utilities import model_blank, model_circle, pad from EMAN2 import periodogram p = model_blank(wn,wn) for i in xrange(n): d = get_im(args[0], i) if nargs==3: d *=mask st = Util.infomask(d, None, True) d -= st[0] p += periodogram(pad(d, wn, wn, 1, 0.)) p /= n p.write_image(args[1]) elif options.adjpw: if len(args) < 3: ERROR("filt_by_rops input target output fl aa (the last two are optional parameters of a low-pass filter)","adjpw",1) return img_stack = args[0] from math import sqrt from fundamentals import rops_table, fft from utilities import read_text_file, get_im from filter import filt_tanl, filt_table if( args[1][-3:] == 'txt'): rops_dst = read_text_file( args[1] ) else: rops_dst = rops_table(get_im( args[1] )) out_stack = args[2] if(len(args) >4): fl = float(args[3]) aa = float(args[4]) else: fl = -1.0 aa = 0.0 nimage = EMUtil.get_image_count( img_stack ) for i in xrange(nimage): img = fft(get_im(img_stack, i) ) rops_src = rops_table(img) assert len(rops_dst) == len(rops_src) table = [0.0]*len(rops_dst) for j in xrange( len(rops_dst) ): table[j] = sqrt( rops_dst[j]/rops_src[j] ) if( fl > 0.0): img = filt_tanl(img, fl, aa) img = fft(filt_table(img, table)) img.write_image(out_stack, i) elif options.rotpw != None: if len(args) != 1: ERROR("Only one input permitted","rotpw",1) return from utilities import write_text_file, get_im from fundamentals import rops_table from math import log10 t = rops_table(get_im(args[0])) x = range(len(t)) r = [0.0]*len(x) for i in x: r[i] = log10(t[i]) write_text_file([t,r,x],options.rotpw) elif options.transformparams != None: if len(args) != 2: ERROR("Please provide names of input and output files with orientation parameters","transformparams",1) return from utilities import read_text_row, write_text_row transf = [0.0]*6 spl=options.transformparams.split(',') for i in xrange(len(spl)): transf[i] = float(spl[i]) write_text_row( rotate_shift_params(read_text_row(args[0]), transf) , args[1]) elif options.makedb != None: nargs = len(args) if nargs != 1: print "must provide exactly one argument denoting database key under which the input params will be stored" return dbkey = args[0] print "database key under which params will be stored: ", dbkey gbdb = js_open_dict("e2boxercache/gauss_box_DB.json") parmstr = 'dummy:'+options.makedb[0] (processorname, param_dict) = parsemodopt(parmstr) dbdict = {} for pkey in param_dict: if (pkey == 'invert_contrast') or (pkey == 'use_variance'): if param_dict[pkey] == 'True': dbdict[pkey] = True else: dbdict[pkey] = False else: dbdict[pkey] = param_dict[pkey] gbdb[dbkey] = dbdict elif options.generate_projections: nargs = len(args) if nargs != 3: ERROR("Must provide name of input structure(s) from which to generate projections, name of output projection stack, and prefix for output micrographs."\ "sxprocess - generate projections",1) return inpstr = args[0] outstk = args[1] micpref = args[2] parmstr = 'dummy:'+options.generate_projections[0] (processorname, param_dict) = parsemodopt(parmstr) parm_CTF = False parm_format = 'bdb' parm_apix = 2.5 if 'CTF' in param_dict: if param_dict['CTF'] == 'True': parm_CTF = True if 'format' in param_dict: parm_format = param_dict['format'] if 'apix' in param_dict: parm_apix = float(param_dict['apix']) boxsize = 64 if 'boxsize' in param_dict: boxsize = int(param_dict['boxsize']) print "pixel size: ", parm_apix, " format: ", parm_format, " add CTF: ", parm_CTF, " box size: ", boxsize scale_mult = 2500 sigma_add = 1.5 sigma_proj = 30.0 sigma2_proj = 17.5 sigma_gauss = 0.3 sigma_mic = 30.0 sigma2_mic = 17.5 sigma_gauss_mic = 0.3 if 'scale_mult' in param_dict: scale_mult = float(param_dict['scale_mult']) if 'sigma_add' in param_dict: sigma_add = float(param_dict['sigma_add']) if 'sigma_proj' in param_dict: sigma_proj = float(param_dict['sigma_proj']) if 'sigma2_proj' in param_dict: sigma2_proj = float(param_dict['sigma2_proj']) if 'sigma_gauss' in param_dict: sigma_gauss = float(param_dict['sigma_gauss']) if 'sigma_mic' in param_dict: sigma_mic = float(param_dict['sigma_mic']) if 'sigma2_mic' in param_dict: sigma2_mic = float(param_dict['sigma2_mic']) if 'sigma_gauss_mic' in param_dict: sigma_gauss_mic = float(param_dict['sigma_gauss_mic']) from filter import filt_gaussl, filt_ctf from utilities import drop_spider_doc, even_angles, model_gauss, delete_bdb, model_blank,pad,model_gauss_noise,set_params2D, set_params_proj from projection import prep_vol,prgs seed(14567) delta = 29 angles = even_angles(delta, 0.0, 89.9, 0.0, 359.9, "S") nangle = len(angles) modelvol = [] nvlms = EMUtil.get_image_count(inpstr) from utilities import get_im for k in xrange(nvlms): modelvol.append(get_im(inpstr,k)) nx = modelvol[0].get_xsize() if nx != boxsize: ERROR("Requested box dimension does not match dimension of the input model.", \ "sxprocess - generate projections",1) nvol = 10 volfts = [[] for k in xrange(nvlms)] for k in xrange(nvlms): for i in xrange(nvol): sigma = sigma_add + random() # 1.5-2.5 addon = model_gauss(sigma, boxsize, boxsize, boxsize, sigma, sigma, 38, 38, 40 ) scale = scale_mult * (0.5+random()) vf, kb = prep_vol(modelvol[k] + scale*addon) volfts[k].append(vf) del vf, modelvol if parm_format == "bdb": stack_data = "bdb:"+outstk delete_bdb(stack_data) else: stack_data = outstk + ".hdf" Cs = 2.0 pixel = parm_apix voltage = 120.0 ampcont = 10.0 ibd = 4096/2-boxsize iprj = 0 width = 240 xstart = 8 + boxsize/2 ystart = 8 + boxsize/2 rowlen = 17 from random import randint params = [] for idef in xrange(3, 8): irow = 0 icol = 0 mic = model_blank(4096, 4096) defocus = idef * 0.5#0.2 if parm_CTF: astampl=defocus*0.15 astangl=50.0 ctf = generate_ctf([defocus, Cs, voltage, pixel, ampcont, 0.0, astampl, astangl]) for i in xrange(nangle): for k in xrange(12): dphi = 8.0*(random()-0.5) dtht = 8.0*(random()-0.5) psi = 360.0*random() phi = angles[i][0]+dphi tht = angles[i][1]+dtht s2x = 4.0*(random()-0.5) s2y = 4.0*(random()-0.5) params.append([phi, tht, psi, s2x, s2y]) ivol = iprj % nvol #imgsrc = randint(0,nvlms-1) imgsrc = iprj % nvlms proj = prgs(volfts[imgsrc][ivol], kb, [phi, tht, psi, -s2x, -s2y]) x = xstart + irow * width y = ystart + icol * width mic += pad(proj, 4096, 4096, 1, 0.0, x-2048, y-2048, 0) proj = proj + model_gauss_noise( sigma_proj, nx, nx ) if parm_CTF: proj = filt_ctf(proj, ctf) proj.set_attr_dict({"ctf":ctf, "ctf_applied":0}) proj = proj + filt_gaussl(model_gauss_noise(sigma2_proj, nx, nx), sigma_gauss) proj.set_attr("origimgsrc",imgsrc) proj.set_attr("test_id", iprj) # flags describing the status of the image (1 = true, 0 = false) set_params2D(proj, [0.0, 0.0, 0.0, 0, 1.0]) set_params_proj(proj, [phi, tht, psi, s2x, s2y]) proj.write_image(stack_data, iprj) icol += 1 if icol == rowlen: icol = 0 irow += 1 iprj += 1 mic += model_gauss_noise(sigma_mic,4096,4096) if parm_CTF: #apply CTF mic = filt_ctf(mic, ctf) mic += filt_gaussl(model_gauss_noise(sigma2_mic, 4096, 4096), sigma_gauss_mic) mic.write_image(micpref + "%1d.hdf" % (idef-3), 0) drop_spider_doc("params.txt", params) elif options.importctf != None: print ' IMPORTCTF ' from utilities import read_text_row,write_text_row from random import randint import subprocess grpfile = 'groupid%04d'%randint(1000,9999) ctfpfile = 'ctfpfile%04d'%randint(1000,9999) cterr = [options.defocuserror/100.0, options.astigmatismerror] ctfs = read_text_row(options.importctf) for kk in xrange(len(ctfs)): root,name = os.path.split(ctfs[kk][-1]) ctfs[kk][-1] = name[:-4] if(options.input[:4] != 'bdb:'): ERROR('Sorry, only bdb files implemented','importctf',1) d = options.input[4:] #try: str = d.index('*') #except: str = -1 from string import split import glob uu = os.path.split(d) uu = os.path.join(uu[0],'EMAN2DB',uu[1]+'.bdb') flist = glob.glob(uu) for i in xrange(len(flist)): root,name = os.path.split(flist[i]) root = root[:-7] name = name[:-4] fil = 'bdb:'+os.path.join(root,name) sourcemic = EMUtil.get_all_attributes(fil,'ptcl_source_image') nn = len(sourcemic) gctfp = [] groupid = [] for kk in xrange(nn): junk,name2 = os.path.split(sourcemic[kk]) name2 = name2[:-4] ctfp = [-1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] for ll in xrange(len(ctfs)): if(name2 == ctfs[ll][-1]): # found correct if(ctfs[ll][8]/ctfs[ll][0] <= cterr[0]): # acceptable defocus error ctfp = ctfs[ll][:8] if(ctfs[ll][10] > cterr[1] ): # error of astigmatism exceed the threshold, set astigmatism to zero. ctfp[6] = 0.0 ctfp[7] = 0.0 gctfp.append(ctfp) groupid.append(kk) break if(len(groupid) > 0): write_text_row(groupid, grpfile) write_text_row(gctfp, ctfpfile) cmd = "{} {} {} {}".format('e2bdb.py',fil,'--makevstack=bdb:'+root+'G'+name,'--list='+grpfile) #print cmd subprocess.call(cmd, shell=True) cmd = "{} {} {} {}".format('sxheader.py','bdb:'+root+'G'+name,'--params=ctf','--import='+ctfpfile) #print cmd subprocess.call(cmd, shell=True) else: print ' >>> Group ',name,' skipped.' cmd = "{} {} {}".format("rm -f",grpfile,ctfpfile) subprocess.call(cmd, shell=True) elif options.scale > 0.0: from utilities import read_text_row,write_text_row scale = options.scale nargs = len(args) if nargs != 2: print "Please provide names of input and output file!" return p = read_text_row(args[0]) for i in xrange(len(p)): p[i][3] /= scale p[i][4] /= scale write_text_row(p, args[1]) elif options.adaptive_mask: from utilities import get_im from morphology import adaptive_mask, binarize, erosion, dilation nsigma = options.nsigma ndilation = options.ndilation kernel_size = options.kernel_size gauss_standard_dev = options.gauss_standard_dev nargs = len(args) if nargs ==0: print " Create 3D mask from a given volume, either automatically or from the user provided threshold." elif nargs > 2: print "Too many inputs are given, try again!" return else: inputvol = get_im(args[0]) input_path, input_file_name = os.path.split(args[0]) input_file_name_root,ext=os.path.splitext(input_file_name) if nargs == 2: mask_file_name = args[1] else: mask_file_name = "adaptive_mask_for_"+input_file_name_root+".hdf" # Only hdf file is output. if options.threshold !=9999.: mask3d = binarize(inputvol, options.threshold) for i in xrange(options.ne): mask3d = erosion(mask3d) for i in xrange(options.nd): mask3d = dilation(mask3d) else: mask3d = adaptive_mask(inputvol, nsigma, ndilation, kernel_size, gauss_standard_dev) mask3d.write_image(mask_file_name) elif options.postprocess: from utilities import get_im from fundamentals import rot_avg_table from morphology import compute_bfactor,power from statistics import fsc from filter import filt_table, filt_gaussinv from EMAN2 import periodogram e1 = get_im(args[0],0) if e1.get_zsize()==1: nimage = EMUtil.get_image_count(args[0]) if options.mask !=None: m = get_im(options.mask) else: m = None for i in xrange(nimage): e1 = get_im(args[0],i) if m: e1 *=m guinerline = rot_avg_table(power(periodogram(e1),.5)) freq_max = 1/(2.*pixel_size) freq_min = 1./options.B_start b,junk=compute_bfactor(guinerline, freq_min, freq_max, pixel_size) tmp = b/pixel_size**2 sigma_of_inverse=sqrt(2./tmp) e1 = filt_gaussinv(e1,sigma_of_inverse) if options.low_pass_filter: from filter import filt_tanl e1 =filt_tanl(e1,options.ff, options.aa) e1.write_image(options.output) else: nargs = len(args) e1 = get_im(args[0]) if nargs >1: e2 = get_im(args[1]) if options.mask !=None: m = get_im(options.mask) else: m =None pixel_size = options.pixel_size from math import sqrt if m !=None: e1 *=m if nargs >1 :e2 *=m if options.fsc_weighted: frc = fsc(e1,e2,1) ## FSC is done on masked two images #### FSC weighting sqrt((2.*fsc)/(1+fsc)); fil = len(frc[1])*[None] for i in xrange(len(fil)): if frc[1][i]>=options.FSC_cutoff: tmp = frc[1][i] else: tmp = 0.0 fil[i] = sqrt(2.*tmp/(1.+tmp)) if nargs>1: e1 +=e2 if options.fsc_weighted: e1=filt_table(e1,fil) guinerline = rot_avg_table(power(periodogram(e1),.5)) freq_max = 1/(2.*pixel_size) freq_min = 1./options.B_start b,junk = compute_bfactor(guinerline, freq_min, freq_max, pixel_size) tmp = b/pixel_size**2 sigma_of_inverse=sqrt(2./tmp) e1 = filt_gaussinv(e1,sigma_of_inverse) if options.low_pass_filter: from filter import filt_tanl e1 =filt_tanl(e1,options.ff, options.aa) e1.write_image(options.output) elif options.window_stack: nargs = len(args) if nargs ==0: print " Reduce image size of a stack" return else: output_stack_name = None inputstack = args[0] if nargs ==2:output_stack_name = args[1] input_path,input_file_name=os.path.split(inputstack) input_file_name_root,ext=os.path.splitext(input_file_name) if input_file_name_root[0:3]=="bdb":stack_is_bdb= True else: stack_is_bdb= False if output_stack_name is None: if stack_is_bdb: output_stack_name ="bdb:reduced_"+input_file_name_root[4:] else:output_stack_name = "reduced_"+input_file_name_root+".hdf" # Only hdf file is output. nimage = EMUtil.get_image_count(inputstack) from fundamentals import window2d for i in xrange(nimage): image = EMData() image.read_image(inputstack,i) w = window2d(image,options.box,options.box) w.write_image(output_stack_name,i) else: ERROR("Please provide option name","sxprocess.py",1)