Esempio n. 1
0
    def test_combinemaps_Halfset(self):
        testargs_new = [
            path.join(ABSOLUTE_BIN_PATH, "sp_process.py"), "--combinemaps",
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "11_MERIDIEN", "vol_0_unfil_028.hdf"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "11_MERIDIEN", "vol_1_unfil_028.hdf"),
            "--output_dir=" + self.new_output_folder, "--pixel_size=1.14",
            "--do_adaptive_mask", "--threshold=0.02", "--mtf=" +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "FalconIImtf.txt")
        ]
        testargs_old = [
            path.join(ABSOLUTE_OLDBIN_PATH, "sp_process.py"), "--combinemaps",
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "11_MERIDIEN", "vol_0_unfil_028.hdf"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "11_MERIDIEN", "vol_1_unfil_028.hdf"),
            "--output_dir=" + self.old_output_folder, "--pixel_size=1.14",
            "--do_adaptive_mask", "--threshold=0.02", "--mtf=" +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "FalconIImtf.txt")
        ]

        with patch.object(sys, 'argv', testargs_new):
            fu.main()
        with patch.object(sys, 'argv', testargs_old):
            oldfu.main()

        return_new = get_im(path.join(self.new_output_folder, self.fname))
        return_old = get_im(path.join(self.old_output_folder, self.fname))
        self.assertTrue(
            array_equal(return_new.get_3dview(), return_old.get_3dview()))
Esempio n. 2
0
    def test_calls(self):
        old_final = "old_final.hdf"
        new_final = "new_final.hdf"

        testargs_new = [
            str(MPI_PATH) + " -np " + str(NUM_PROC) + " " +
            path.join(ABSOLUTE_BIN_PATH, "sp_filterlocal.py") + " " +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "12_POSTREFINER", "vol_combined.hdf") + " " +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "17_LOCAL_RES", "localres.hdf") + " " +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "12_POSTREFINER", "vol_adaptive_mask.hdf") + " " +
            new_final + " --radius=145 --MPI"
        ]

        testargs_old = [
            str(MPI_PATH) + " -np " + str(NUM_PROC) + " " +
            path.join(ABSOLUTE_OLDBIN_PATH, "sp_filterlocal.py") + " " +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "12_POSTREFINER", "vol_combined.hdf") + " " +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "17_LOCAL_RES", "localres.hdf") + " " +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "12_POSTREFINER", "vol_adaptive_mask.hdf") + " " +
            old_final + " --radius=145 --MPI"
        ]

        a = subprocess.run(args=testargs_new,
                           shell=True,
                           stderr=subprocess.STDOUT)

        b = subprocess.run(args=testargs_old,
                           shell=True,
                           stderr=subprocess.STDOUT)

        return_old = get_im(old_final)
        return_new = get_im(new_final)
        self.assertTrue(
            array_equal(return_old.get_3dview(), return_new.get_3dview()))
        self.assertTrue(
            allclose(
                return_new.get_3dview().flatten().tolist()[4034427:4034527], [
                    3.201385334250517e-05, -0.0033583296462893486,
                    -0.0033261957578361034, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0
                ],
                atol=0.01))
        remove_list_of_file([old_final, new_final])
Esempio n. 3
0
    def test_(self):
        out_dir_old = "oldviper"
        out_dir_new = "newviper"

        filename_vol = "volf.hdf"
        filename_refvol = "refvol2.hdf"

        testcommand_new = (
            MPI_PATH + " -np " + str(NUM_PROC) + " " +
            path.join(ABSOLUTE_OLDBIN_PATH, "sp_viper.py") + " " +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "04_ISAC", "class_averages.hdf") + " " + out_dir_old +
            " --sym=c5")

        testcommand_old = (
            MPI_PATH + " -np " + str(NUM_PROC) + " " +
            path.join(ABSOLUTE_BIN_PATH, "sp_viper.py") + " " +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "04_ISAC", "class_averages.hdf") + " " + out_dir_new +
            " --sym=c5")

        subprocess.run(args=[testcommand_new],
                       shell=True,
                       stderr=subprocess.STDOUT)
        subprocess.run(args=[testcommand_old],
                       shell=True,
                       stderr=subprocess.STDOUT)

        return_new_avg = get_im(
            path.join(ABSOLUTE_PATH, out_dir_new, filename_vol))
        return_new_var = get_im(
            path.join(ABSOLUTE_PATH, out_dir_new, filename_refvol))

        return_old_avg = get_im(
            path.join(ABSOLUTE_PATH, out_dir_old, filename_vol))
        return_old_var = get_im(
            path.join(ABSOLUTE_PATH, out_dir_old, filename_refvol))

        self.assertTrue(
            allclose(return_new_avg.get_3dview(),
                     return_old_avg.get_3dview(),
                     atol=TOLERANCE))
        self.assertTrue(
            allclose(return_old_var.get_3dview(),
                     return_new_var.get_3dview(),
                     atol=TOLERANCE))

        remove_dir(out_dir_new)
        remove_dir(out_dir_old)
Esempio n. 4
0
def read_volume(path_vol_1,
                path_vol_2,
                path_mask=None,
                resolution=None,
                pixel_size=None):
    """
	Reads the 3D density map
	:param path_vol_1: Path to the density map
	:param path_mask: If provided, the volume is masked.
	:return: Volume for density map
	"""
    print("Read volumes, masking them and prepare them for reprojections")
    mask_vol = None
    vol2 = None
    if path_mask is not None:
        mask_vol = sp_utilities.get_im(path_mask)

    vol1 = prepare_volume(path_vol_1, mask_vol, resolution, pixel_size)
    if path_vol_2:
        vol2 = prepare_volume(path_vol_2, mask_vol, resolution, pixel_size)

    if mask_vol:
        mask_vol = sp_projection.prep_vol(mask_vol, interpolation_method=1)

    return vol1, vol2, mask_vol
Esempio n. 5
0
def resample_prepare(prjfile, nvol, snr, CTF, npad):
    from sp_utilities import get_im
    nx = get_im(prjfile, 0).get_xsize()
    fftvols = [None] * nvol
    wgtvols = [None] * nvol
    rectors = [None] * nvol
    for i in range(nvol):
        fftvols[i] = EMData()
        wgtvols[i] = EMData()
        if CTF:
            params = {
                "size": nx,
                "npad": npad,
                "snr": snr,
                "weight": wgtvols[i],
                "fftvol": fftvols[i]
            }
            rectors[i] = Reconstructors.get("nn4_ctf", params)
        else:
            params = {
                "size": nx,
                "npad": npad,
                "snr": snr,
                "weight": wgtvols[i],
                "fftvol": fftvols[i]
            }
            rectors[i] = Reconstructors.get("nn4", params)

        rectors[i].setup()

    return rectors, fftvols, wgtvols
Esempio n. 6
0
    def test_(self):
        testargs_new = [
            path.join(ABSOLUTE_BIN_PATH, "sp_mask.py"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "09_ADJUSTMENT", "vol3d_ref_moon_eliminated.hdf"),
            "--pixel_size=1.14", self.new_output_folder, "--mol_mass=1400",
            "--edge_width=10"
        ]
        testargs_old = [
            path.join(ABSOLUTE_OLDBIN_PATH, "sp_mask.py"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "09_ADJUSTMENT", "vol3d_ref_moon_eliminated.hdf"),
            "--pixel_size=1.14", self.old_output_folder, "--mol_mass=1400",
            "--edge_width=10"
        ]
        with patch.object(sys, 'argv', testargs_new):
            fu.main()
        with patch.object(sys, 'argv', testargs_old):
            oldfu.main()

        old_value = get_im(path.join(self.old_output_folder, self.filename))
        new_value = get_im(path.join(self.new_output_folder, self.filename))
        self.assertTrue(
            array_equal(old_value.get_3dview(), new_value.get_3dview()))
        self.assertTrue(
            allclose(
                new_value.get_3dview().flatten().tolist()[3022799:3022899], [
                    0.00024921807926148176, 0.0005635463166981936,
                    0.0005635463166981936, 0.00024921807926148176, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
                ]))
Esempio n. 7
0
def genbuf(prjfile, bufprefix, beg, end, CTF, npad, verbose=0):

    if (verbose == 1):
        finfo = open(os.path.join(outdir, "progress.txt"), "w")
    else:
        finfo = None

    start_time = time()
    istore = newfile_store(bufprefix, npad, CTF)
    for i in range(beg, end):
        prj = get_im(prjfile, i)
        istore.add_image(prj, prj.get_attr("xform.projection"))
        if (not (finfo is None) and ((i % 100 == 99 or i == end - 1))):
            finfo.write("%6d buffered, time: %10.3f\n" %
                        (i + 1, time() - start_time))
            finfo.flush()
Esempio n. 8
0
def cml_export_struc(stack, outdir, irun, Ori):
    from sp_projection import plot_angles
    from sp_utilities import set_params_proj, get_im

    global g_n_prj

    pagls = []
    for i in range(g_n_prj):
        data = get_im(stack, i)
        p = [Ori[4 * i], Ori[4 * i + 1], Ori[4 * i + 2], 0.0, 0.0]
        set_params_proj(data, p)
        data.write_image(outdir + '/structure_%03i.hdf' % irun, i)

        # prepare angles to plot
        pagls.append([Ori[4 * i], Ori[4 * i + 1], Ori[4 * i + 2]])

    # plot angles
    im = plot_angles(pagls)
    im.write_image(outdir + '/plot_agls_%03i.hdf' % irun)
Esempio n. 9
0
def do_volume_mask(ref_data):
    """
		1. - volume
		2. - Tracker, see meridien
		3. - current iteration number
	"""

    # Retrieve the function specific input arguments from ref_data
    vol = ref_data[0]
    Tracker = ref_data[1]
    mainiteration = ref_data[2]

    if Tracker["constants"]["mask3D"] is None:
        vol = sp_morphology.cosinemask(vol,
                                       radius=Tracker["constants"]["radius"])
    else:
        EMAN2_cppwrap.Util.mul_img(
            vol, sp_utilities.get_im(Tracker["constants"]["mask3D"]))

    return vol
Esempio n. 10
0
def prepare_volume(volume_path, mask=None, resolution=None, pixel_size=None):
    """
	Prepares the volume for reprojections
	:param volume_path: Path to volume file
	:param mask: Particle mask
	:param resolution: Resolution of the current reconstruction.
	:return:
	"""
    vol = sp_utilities.get_im(volume_path)
    if resolution:
        vol = vol.process(
            "filter.lowpass.gauss",
            {
                "cutoff_freq": old_div(1.0, resolution),
                "apix": pixel_size
            },
        )
    if mask:
        vol = vol * mask
    vol = sp_projection.prep_vol(vol, interpolation_method=1)
    return vol
Esempio n. 11
0
def generate_helimic(refvol,
                     outdir,
                     pixel,
                     CTF=False,
                     Cs=2.0,
                     voltage=200.0,
                     ampcont=10.0,
                     nonoise=False,
                     rand_seed=14567):

    from sp_utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im
    from random import random
    from sp_projection import prgs, prep_vol
    from sp_filter import filt_gaussl, filt_ctf
    from EMAN2 import EMAN2Ctf

    if os.path.exists(outdir):
        ERROR(
            "Output directory exists, please change the name and restart the program"
        )
        return

    os.mkdir(outdir)
    seed(rand_seed)
    Util.set_randnum_seed(rand_seed)
    angles = []
    for i in range(3):
        angles.append([0.0 + 60.0 * i, 90.0 - i * 5, 0.0, 0.0, 0.0])

    nangle = len(angles)

    volfts = get_im(refvol)
    nx = volfts.get_xsize()
    ny = volfts.get_ysize()
    nz = volfts.get_zsize()
    volfts, kbx, kby, kbz = prep_vol(volfts)
    iprj = 0
    width = 500
    xstart = 0
    ystart = 0

    for idef in range(3, 6):
        mic = model_blank(2048, 2048)
        #defocus = idef*0.2
        defocus = idef * 0.6  ##@ming
        if CTF:
            #ctf = EMAN2Ctf()
            #ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
            from sp_utilities import generate_ctf
            ctf = generate_ctf(
                [defocus, 2, 200, 1.84, 0.0, ampcont, defocus * 0.2, 80]
            )  ##@ming   the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
        i = idef - 4
        for k in range(1):
            psi = 90 + 10 * i
            proj = prgs(
                volfts, kbz,
                [angles[idef - 3][0], angles[idef - 3][1], psi, 0.0, 0.0], kbx,
                kby)
            proj = Util.window(proj, 320, nz)
            mic += pad(proj, 2048, 2048, 1, 0.0, 750 * i, 20 * i, 0)

        if not nonoise: mic += model_gauss_noise(30.0, 2048, 2048)
        if CTF:
            #apply CTF
            mic = filt_ctf(mic, ctf)

        if not nonoise:
            mic += filt_gaussl(model_gauss_noise(17.5, 2048, 2048), 0.3)

        mic.write_image("%s/mic%1d.hdf" % (outdir, idef - 3), 0)
Esempio n. 12
0
def mode_meridien(reconfile,
                  classavgstack,
                  classdocs,
                  partangles,
                  selectdoc,
                  maxshift,
                  outerrad,
                  outanglesdoc,
                  outaligndoc,
                  interpolation_method=1,
                  outliers=None,
                  goodclassparttemplate=None,
                  alignopt='apsh',
                  ringstep=1,
                  log=None,
                  verbose=False):

    # Resample reference
    recondata = EMAN2.EMData(reconfile)
    idim = recondata['nx']
    reconprep = prep_vol(recondata,
                         npad=2,
                         interpolation_method=interpolation_method)

    # Initialize output angles
    outangleslist = []
    outalignlist = []

    # Read class lists
    classdoclist = glob.glob(classdocs)
    partangleslist = read_text_row(partangles)

    # Loop through class lists
    for classdoc in classdoclist:  # [classdoclist[32]]:  #
        # Strip out three-digit filenumber
        classexample = os.path.splitext(classdoc)
        classnum = int(classexample[0][-3:])

        # Initial average
        [avg_phi_init, avg_theta_init] = average_angles(partangleslist,
                                                        classdoc,
                                                        selectdoc=selectdoc)

        # Look for outliers
        if outliers:
            [avg_phi_final, avg_theta_final] = average_angles(
                partangleslist,
                classdoc,
                selectdoc=selectdoc,
                init_angles=[avg_phi_init, avg_theta_init],
                threshold=outliers,
                goodpartdoc=goodclassparttemplate.format(classnum),
                log=log,
                verbose=verbose)
        else:
            [avg_phi_final, avg_theta_final] = [avg_phi_init, avg_theta_init]

        # Compute re-projection
        refprjreal = prgl(reconprep, [avg_phi_final, avg_theta_final, 0, 0, 0],
                          interpolation_method=1,
                          return_real=True)

        # Align to class average
        classavg = get_im(classavgstack, classnum)

        # Alignment using self-correlation function
        if alignopt == 'scf':
            ang_align2d, sxs, sys, mirrorflag, peak = align2d_scf(classavg,
                                                                  refprjreal,
                                                                  maxshift,
                                                                  maxshift,
                                                                  ou=outerrad)

        # Weird results
        elif alignopt == 'align2d':
            # Set search range
            currshift = 0
            txrng = tyrng = search_range(idim, outerrad, currshift, maxshift)

            # Perform alignment
            ang_align2d, sxs, sys, mirrorflag, peak = align2d(
                classavg, refprjreal, txrng, tyrng, last_ring=outerrad)

        # Direct3 (angles seemed to be quantized)
        elif alignopt == 'direct3':
            [[ang_align2d, sxs, sys, mirrorflag,
              peak]] = align2d_direct3([classavg],
                                       refprjreal,
                                       maxshift,
                                       maxshift,
                                       ou=outerrad)

        # APSH-like alignment (default)
        else:
            [[ang_align2d, sxs, sys, mirrorflag,
              scale]] = apsh(refprjreal,
                             classavg,
                             outerradius=outerrad,
                             maxshift=maxshift,
                             ringstep=ringstep)

        outalignlist.append([ang_align2d, sxs, sys, mirrorflag, 1])
        msg = "Particle list %s: ang_align2d=%s sx=%s sy=%s mirror=%s\n" % (
            classdoc, ang_align2d, sxs, sys, mirrorflag)
        print_log_msg(msg, log, verbose)

        # Check for mirroring
        if mirrorflag == 1:
            tempeulers = list(
                compose_transform3(avg_phi_final, avg_theta_final, 0, 0, 0, 0,
                                   1, 0, 180, 0, 0, 0, 0, 1))
            combinedparams = list(
                compose_transform3(tempeulers[0], tempeulers[1], tempeulers[2],
                                   tempeulers[3], tempeulers[4], 0, 1, 0, 0,
                                   -ang_align2d, 0, 0, 0, 1))
        else:
            combinedparams = list(
                compose_transform3(avg_phi_final, avg_theta_final, 0, 0, 0, 0,
                                   1, 0, 0, -ang_align2d, 0, 0, 0, 1))
        # compose_transform3: returns phi,theta,psi, tx,ty,tz, scale

        outangleslist.append(combinedparams)
    # End class-loop

    write_text_row(outangleslist, outanglesdoc)
    write_text_row(outalignlist, outaligndoc)
    print_log_msg(
        'Wrote alignment parameters to %s and %s\n' %
        (outanglesdoc, outaligndoc), log, verbose)

    del recondata  # Clean up
Esempio n. 13
0
def compare_projs(reconfile,
                  classavgstack,
                  inputanglesdoc,
                  outdir,
                  interpolation_method=1,
                  log=None,
                  verbose=False):
    """
	Make comparison stack between class averages (even-numbered (starts from 0)) and re-projections (odd-numbered).
	
	Arguments:
		reconfile : Input volume from which to generate re-projections
		classavgstack ; Input image stack
		inputanglesdoc : Input Euler angles doc
		outdir ; Output directory
		interpolation_method : Interpolation method: nearest neighbor (nn, 0), trilinear (1, default), gridding (-1)
		log : Logger object
		verbose : (boolean) Whether to write additional information to screen
	Returns:
		compstack : Stack of comparisons between input image stack (even-numbered (starts from 0)) and input volume (odd-numbered)
	"""

    recondata = EMAN2.EMData(reconfile)
    nx = recondata.get_xsize()

    # Resample reference
    reconprep = prep_vol(recondata,
                         npad=2,
                         interpolation_method=interpolation_method)

    ccclist = []

    #  Here you need actual radius to compute proper ccc's, but if you do, you have to deal with translations, PAP
    mask = model_circle(nx // 2 - 2, nx, nx)
    mask.write_image(os.path.join(outdir, 'maskalign.hdf'))
    compstack = os.path.join(outdir, 'comp-proj-reproj.hdf')

    # Number of images may have changed
    nimg1 = EMAN2.EMUtil.get_image_count(classavgstack)
    angleslist = read_text_row(inputanglesdoc)

    for imgnum in range(nimg1):
        # Get class average
        classimg = get_im(classavgstack, imgnum)

        # Compute re-projection
        prjimg = prgl(reconprep,
                      angleslist[imgnum],
                      interpolation_method=1,
                      return_real=False)

        # Calculate 1D power spectra
        rops_dst = rops_table(classimg * mask)
        rops_src = rops_table(prjimg)

        #  Set power spectrum of reprojection to the data.
        #  Since data has an envelope, it would make more sense to set data to reconstruction,
        #  but to do it one would have to know the actual resolution of the data.
        #  you can check sxprocess.py --adjpw to see how this is done properly  PAP
        table = [0.0] * len(rops_dst)  # initialize table
        for j in range(len(rops_dst)):
            table[j] = sqrt(rops_dst[j] / rops_src[j])
        prjimg = fft(filt_table(
            prjimg,
            table))  # match FFT amplitudes of re-projection and class average

        cccoeff = ccc(prjimg, classimg, mask)
        #print imgnum, cccoeff
        classimg.set_attr_dict({'cross-corr': cccoeff})
        prjimg.set_attr_dict({'cross-corr': cccoeff})

        montagestack = []
        montagestack.append(prjimg)
        montagestack.append(classimg)
        comparison_pair = montage2(montagestack, ncol=2, marginwidth=1)
        comparison_pair.write_image(compstack, imgnum)

        ccclist.append(cccoeff)
    del angleslist
    meanccc = sum(ccclist) / nimg1
    print_log_msg("Average CCC is %s\n" % meanccc, log, verbose)

    nimg2 = EMAN2.EMUtil.get_image_count(compstack)

    for imgnum in range(nimg2):  # xrange will be deprecated in Python3
        prjimg = get_im(compstack, imgnum)
        meanccc1 = prjimg.get_attr_default('mean-cross-corr', -1.0)
        prjimg.set_attr_dict({'mean-cross-corr': meanccc})
        write_header(compstack, prjimg, imgnum)

    return compstack
Esempio n. 14
0
    def test_(self):
        testargs_new = [
            path.join(ABSOLUTE_BIN_PATH, "localRes.py"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "11_MERIDIEN", "vol_0_unfil_028.hdf"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "11_MERIDIEN", "vol_1_unfil_028.hdf"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "12_POSTREFINER", "vol_adaptive_mask.hdf"),
            self.out_dir_new, "--out_ang_res", "--apix=1.14"
        ]
        testargs_old = [
            path.join(ABSOLUTE_OLDBIN_PATH, "sp_process.py"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "11_MERIDIEN", "vol_0_unfil_028.hdf"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "11_MERIDIEN", "vol_1_unfil_028.hdf"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "12_POSTREFINER", "vol_adaptive_mask.hdf"),
            self.out_dir_old, "--out_ang_res", "--apix=1.14"
        ]

        with patch.object(sys, 'argv', testargs_new):
            fu.main()
        with patch.object(sys, 'argv', testargs_old):
            oldfu.main()

        old_localRes_ang = get_im(path.join(self.out_dir_old, self.fname_ang))
        old_localRes = get_im(path.join(self.out_dir_old, self.fname))
        new_localRes_ang = get_im(path.join(self.out_dir_new, self.fname_ang))
        new_localRes = get_im(path.join(self.out_dir_new, self.fname))
        self.assertTrue(
            allclose(old_localRes_ang.get_3dview(),
                     new_localRes_ang.get_3dview(),
                     atol=0.05))
        self.assertTrue(
            allclose(old_localRes.get_3dview(),
                     new_localRes.get_3dview(),
                     atol=0.05))
        self.assertTrue(
            allclose(
                new_localRes_ang.get_3dview().flatten().tolist()
                [4034427:4034527], [
                    6.859487056732178, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
                ],
                atol=0.05))
        self.assertTrue(
            allclose(
                new_localRes.get_3dview().flatten().tolist()[4034427:4034527],
                [
                    0.1661931872367859, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
                ],
                atol=0.05))
Esempio n. 15
0
    def test_symmetrize(self):
        old_output_folder = "3DvariabilitySimmetry_sym_Old"
        new_output_folder = "3DvariabilitySimmetry_sym_New"
        remove_dir(new_output_folder)
        remove_dir(old_output_folder)
        filename = "sdata"
        testargs_new = (path.join(ABSOLUTE_BIN_PATH, "sp_3dvariability.py") +
                        " " + "--symmetrize" + " " + "'bdb:" + path.join(
                            ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                            "06_SUBSTACK_ANO#isac_substack'") + " " +
                        "--output_dir=" + new_output_folder + " " + "--sym=c5")
        testargs_old = (
            path.join(ABSOLUTE_OLDBIN_PATH, "sp_3dvariability.py") + " " +
            "--symmetrize" + " " + "'bdb:" +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "06_SUBSTACK_ANO#isac_substack'") + " " +
            "--output_dir=" + old_output_folder + " " + "--sym=c5")

        subprocess.run(args=[testargs_new], shell=True, capture_output=True)
        subprocess.run(args=[testargs_old], shell=True, capture_output=True)

        return_old = get_im('bdb:{0}#{1}'.format(old_output_folder, filename))
        return_new = get_im('bdb:{0}#{1}'.format(new_output_folder, filename))
        self.assertTrue(
            allclose(return_new.get_2dview(),
                     return_old.get_2dview(),
                     atol=0.1))
        self.assertTrue(
            allclose(return_new.get_2dview().flatten()[0:100], [
                -0.38441434502601624, -0.4573197066783905,
                -0.39960816502571106, -0.10431570559740067,
                0.018583765253424644, 0.37694355845451355, 0.4353419840335846,
                0.06498070061206818, -0.38467326760292053, -0.7100721001625061,
                -0.7571740746498108, -0.9328497648239136, -0.8550546169281006,
                -0.21433517336845398, 0.34148913621902466, 0.30916473269462585,
                0.39394184947013855, 0.4447155296802521, -0.032833874225616455,
                0.4121330976486206, 0.638403594493866, 0.34945985674858093,
                0.2751978039741516, -0.14872148633003235, -0.5307353138923645,
                -0.6574574112892151, -0.8945914506912231, -1.107301950454712,
                -1.3676011562347412, -1.2225056886672974, -1.2273787260055542,
                -0.7121877074241638, 0.06733409315347672, 0.29879996180534363,
                0.6046097874641418, 0.496036559343338, 0.7214235663414001,
                0.7652679681777954, 0.9319247603416443, 0.11259084939956665,
                -0.3054805397987366, -0.8183746933937073, -1.4462037086486816,
                -1.4775571823120117, -1.2116808891296387, -0.911469042301178,
                -0.9934141039848328, -0.9891195297241211, -1.0073580741882324,
                -1.234678864479065, -1.4292954206466675, -1.5580313205718994,
                -1.446936845779419, -1.67134428024292, -2.0199873447418213,
                -2.0543317794799805, -2.2354137897491455, -1.99867844581604,
                -1.4405670166015625, -1.7088592052459717, -2.060110569000244,
                -1.6585056781768799, -1.0386717319488525, -0.4783007502555847,
                -0.12102964520454407, 0.16874085366725922,
                -0.12371158599853516, 0.01841016113758087, -0.6442297101020813,
                -0.6972493529319763, -0.1051267459988594, 0.016293810680508614,
                -0.10689342767000198, -0.472159206867218, -0.4886413514614105,
                -0.09828135371208191, -0.7409976720809937, -0.8893253207206726,
                -1.1477444171905518, -0.35942375659942627,
                0.030692437663674355, 0.5380961298942566, 1.06252121925354,
                0.6961002349853516, 0.7563707232475281, 1.0197871923446655,
                0.9021824598312378, 0.34891727566719055, 0.402149498462677,
                0.634938657283783, 0.6044892072677612, 0.24482420086860657,
                -0.4132026731967926, -0.9945327043533325, -1.260424256324768,
                -1.6913681030273438, -1.7999876737594604, -1.5891680717468262,
                -1.3196191787719727, -0.9893324375152588
            ],
                     atol=0.1))
        remove_dir(new_output_folder)
        remove_dir(old_output_folder)
Esempio n. 16
0
def main():
    def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
        # the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D
        if mirror:
            m = 1
            alpha, sx, sy, scalen = sp_utilities.compose_transform2(
                0, s2x, s2y, 1.0, 540.0 - psi, 0, 0, 1.0)
        else:
            m = 0
            alpha, sx, sy, scalen = sp_utilities.compose_transform2(
                0, s2x, s2y, 1.0, 360.0 - psi, 0, 0, 1.0)
        return alpha, sx, sy, m

    progname = optparse.os.path.basename(sys.argv[0])
    usage = (
        progname +
        " prj_stack  --ave2D= --var2D=  --ave3D= --var3D= --img_per_grp= --fl=  --aa=   --sym=symmetry --CTF"
    )
    parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)

    parser.add_option("--output_dir",
                      type="string",
                      default="./",
                      help="Output directory")
    parser.add_option(
        "--ave2D",
        type="string",
        default=False,
        help="Write to the disk a stack of 2D averages",
    )
    parser.add_option(
        "--var2D",
        type="string",
        default=False,
        help="Write to the disk a stack of 2D variances",
    )
    parser.add_option(
        "--ave3D",
        type="string",
        default=False,
        help="Write to the disk reconstructed 3D average",
    )
    parser.add_option(
        "--var3D",
        type="string",
        default=False,
        help="Compute 3D variability (time consuming!)",
    )
    parser.add_option(
        "--img_per_grp",
        type="int",
        default=100,
        help="Number of neighbouring projections.(Default is 100)",
    )
    parser.add_option(
        "--no_norm",
        action="store_true",
        default=False,
        help="Do not use normalization.(Default is to apply normalization)",
    )
    # parser.add_option("--radius", 	    type="int"         ,	default=-1   ,				help="radius for 3D variability" )
    parser.add_option(
        "--npad",
        type="int",
        default=2,
        help=
        "Number of time to pad the original images.(Default is 2 times padding)",
    )
    parser.add_option("--sym",
                      type="string",
                      default="c1",
                      help="Symmetry. (Default is no symmetry)")
    parser.add_option(
        "--fl",
        type="float",
        default=0.0,
        help=
        "Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)",
    )
    parser.add_option(
        "--aa",
        type="float",
        default=0.02,
        help=
        "Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)",
    )
    parser.add_option(
        "--CTF",
        action="store_true",
        default=False,
        help="Use CFT correction.(Default is no CTF correction)",
    )
    # parser.add_option("--MPI" , 		action="store_true",	default=False,				help="use MPI version")
    # parser.add_option("--radiuspca", 	type="int"         ,	default=-1   ,				help="radius for PCA" )
    # parser.add_option("--iter", 		type="int"         ,	default=40   ,				help="maximum number of iterations (stop criterion of reconstruction process)" )
    # parser.add_option("--abs", 		type="float"   ,        default=0.0  ,				help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
    # parser.add_option("--squ", 		type="float"   ,	    default=0.0  ,				help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
    parser.add_option(
        "--VAR",
        action="store_true",
        default=False,
        help="Stack of input consists of 2D variances (Default False)",
    )
    parser.add_option(
        "--decimate",
        type="float",
        default=0.25,
        help="Image decimate rate, a number less than 1. (Default is 0.25)",
    )
    parser.add_option(
        "--window",
        type="int",
        default=0,
        help=
        "Target image size relative to original image size. (Default value is zero.)",
    )
    # parser.add_option("--SND",			action="store_true",	default=False,				help="compute squared normalized differences (Default False)")
    # parser.add_option("--nvec",			type="int"         ,	default=0    ,				help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)")
    parser.add_option(
        "--symmetrize",
        action="store_true",
        default=False,
        help="Prepare input stack for handling symmetry (Default False)",
    )
    parser.add_option("--overhead",
                      type="float",
                      default=0.5,
                      help="python overhead per CPU.")

    (options, args) = parser.parse_args()
    #####
    # from mpi import *

    #  This is code for handling symmetries by the above program.  To be incorporated. PAP 01/27/2015

    # Set up global variables related to bdb cache
    if sp_global_def.CACHE_DISABLE:
        sp_utilities.disable_bdb_cache()

    # Set up global variables related to ERROR function
    sp_global_def.BATCH = True

    # detect if program is running under MPI
    RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in optparse.os.environ
    if RUNNING_UNDER_MPI:
        sp_global_def.MPI = True
    if options.output_dir == "./":
        current_output_dir = optparse.os.path.abspath(options.output_dir)
    else:
        current_output_dir = options.output_dir
    if options.symmetrize:

        if mpi.mpi_comm_size(mpi.MPI_COMM_WORLD) > 1:
            sp_global_def.ERROR(
                "Cannot use more than one CPU for symmetry preparation")

        if not optparse.os.path.exists(current_output_dir):
            optparse.os.makedirs(current_output_dir)
            sp_global_def.write_command(current_output_dir)

        if optparse.os.path.exists(
                optparse.os.path.join(current_output_dir, "log.txt")):
            optparse.os.remove(
                optparse.os.path.join(current_output_dir, "log.txt"))
        log_main = sp_logger.Logger(sp_logger.BaseLogger_Files())
        log_main.prefix = optparse.os.path.join(current_output_dir, "./")

        instack = args[0]
        sym = options.sym.lower()
        if sym == "c1":
            sp_global_def.ERROR(
                "There is no need to symmetrize stack for C1 symmetry")

        line = ""
        for a in sys.argv:
            line += " " + a
        log_main.add(line)

        if instack[:4] != "bdb:":
            # if output_dir =="./": stack = "bdb:data"
            stack = "bdb:" + current_output_dir + "/data"
            sp_utilities.delete_bdb(stack)
            junk = sp_utilities.cmdexecute("sp_cpy.py  " + instack + "  " +
                                           stack)
        else:
            stack = instack

        qt = EMAN2_cppwrap.EMUtil.get_all_attributes(stack, "xform.projection")

        na = len(qt)
        ts = sp_utilities.get_symt(sym)
        ks = len(ts)
        angsa = [None] * na

        for k in range(ks):
            # Qfile = "Q%1d"%k
            # if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k)
            Qfile = optparse.os.path.join(current_output_dir, "Q%1d" % k)
            # delete_bdb("bdb:Q%1d"%k)
            sp_utilities.delete_bdb("bdb:" + Qfile)
            # junk = cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
            junk = sp_utilities.cmdexecute("e2bdb.py  " + stack +
                                           "  --makevstack=bdb:" + Qfile)
            # DB = db_open_dict("bdb:Q%1d"%k)
            DB = EMAN2db.db_open_dict("bdb:" + Qfile)
            for i in range(na):
                ut = qt[i] * ts[k]
                DB.set_attr(i, "xform.projection", ut)
                # bt = ut.get_params("spider")
                # angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
            # write_text_row(angsa, 'ptsma%1d.txt'%k)
            # junk = cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
            # junk = cmdexecute("sxheader.py  bdb:Q%1d  --params=xform.projection  --import=ptsma%1d.txt"%(k,k))
            DB.close()
        # if options.output_dir =="./": delete_bdb("bdb:sdata")
        sp_utilities.delete_bdb("bdb:" + current_output_dir + "/" + "sdata")
        # junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
        sdata = "bdb:" + current_output_dir + "/" + "sdata"
        sp_global_def.sxprint(sdata)
        junk = sp_utilities.cmdexecute("e2bdb.py   " + current_output_dir +
                                       "  --makevstack=" + sdata + " --filt=Q")
        # junk = cmdexecute("ls  EMAN2DB/sdata*")
        # a = get_im("bdb:sdata")
        a = sp_utilities.get_im(sdata)
        a.set_attr("variabilitysymmetry", sym)
        # a.write_image("bdb:sdata")
        a.write_image(sdata)

    else:

        myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
        number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
        main_node = 0
        shared_comm = mpi.mpi_comm_split_type(mpi.MPI_COMM_WORLD,
                                              mpi.MPI_COMM_TYPE_SHARED, 0,
                                              mpi.MPI_INFO_NULL)
        myid_on_node = mpi.mpi_comm_rank(shared_comm)
        no_of_processes_per_group = mpi.mpi_comm_size(shared_comm)
        masters_from_groups_vs_everything_else_comm = mpi.mpi_comm_split(
            mpi.MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node)
        color, no_of_groups, balanced_processor_load_on_nodes = sp_utilities.get_colors_and_subsets(
            main_node,
            mpi.MPI_COMM_WORLD,
            myid,
            shared_comm,
            myid_on_node,
            masters_from_groups_vs_everything_else_comm,
        )
        overhead_loading = options.overhead * number_of_proc
        # memory_per_node  = options.memory_per_node
        # if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group
        keepgoing = 1

        current_window = options.window
        current_decimate = options.decimate

        if len(args) == 1:
            stack = args[0]
        else:
            sp_global_def.sxprint("Usage: " + usage)
            sp_global_def.sxprint("Please run '" + progname +
                                  " -h' for detailed options")
            sp_global_def.ERROR(
                "Invalid number of parameters used. Please see usage information above."
            )
            return

        t0 = time.time()
        # obsolete flags
        options.MPI = True
        # options.nvec = 0
        options.radiuspca = -1
        options.iter = 40
        options.abs = 0.0
        options.squ = 0.0

        if options.fl > 0.0 and options.aa == 0.0:
            sp_global_def.ERROR(
                "Fall off has to be given for the low-pass filter", myid=myid)

        # if options.VAR and options.SND:
        # 	ERROR( "Only one of var and SND can be set!",myid=myid )

        if options.VAR and (options.ave2D or options.ave3D or options.var2D):
            sp_global_def.ERROR(
                "When VAR is set, the program cannot output ave2D, ave3D or var2D",
                myid=myid,
            )

        # if options.SND and (options.ave2D or options.ave3D):
        # 	ERROR( "When SND is set, the program cannot output ave2D or ave3D", myid=myid )

        # if options.nvec > 0 :
        # 	ERROR( "PCA option not implemented", myid=myid )

        # if options.nvec > 0 and options.ave3D == None:
        # 	ERROR( "When doing PCA analysis, one must set ave3D", myid=myid )

        if current_decimate > 1.0 or current_decimate < 0.0:
            sp_global_def.ERROR(
                "Decimate rate should be a value between 0.0 and 1.0",
                myid=myid)

        if current_window < 0.0:
            sp_global_def.ERROR(
                "Target window size should be always larger than zero",
                myid=myid)

        if myid == main_node:
            img = sp_utilities.get_image(stack, 0)
            nx = img.get_xsize()
            ny = img.get_ysize()
            if min(nx, ny) < current_window:
                keepgoing = 0
        keepgoing = sp_utilities.bcast_number_to_all(keepgoing, main_node,
                                                     mpi.MPI_COMM_WORLD)
        if keepgoing == 0:
            sp_global_def.ERROR(
                "The target window size cannot be larger than the size of decimated image",
                myid=myid,
            )

        options.sym = options.sym.lower()
        # if global_def.CACHE_DISABLE:
        # 	from utilities import disable_bdb_cache
        # 	disable_bdb_cache()
        # global_def.BATCH = True

        if myid == main_node:
            if not optparse.os.path.exists(current_output_dir):
                optparse.os.makedirs(
                    current_output_dir
                )  # Never delete output_dir in the program!

        img_per_grp = options.img_per_grp
        # nvec        = options.nvec
        radiuspca = options.radiuspca
        # if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
        log_main = sp_logger.Logger(sp_logger.BaseLogger_Files())
        log_main.prefix = optparse.os.path.join(current_output_dir, "./")

        if myid == main_node:
            line = ""
            for a in sys.argv:
                line += " " + a
            log_main.add(line)
            log_main.add("-------->>>Settings given by all options<<<-------")
            log_main.add("Symmetry             : %s" % options.sym)
            log_main.add("Input stack          : %s" % stack)
            log_main.add("Output_dir           : %s" % current_output_dir)

            if options.ave3D:
                log_main.add("Ave3d                : %s" % options.ave3D)
            if options.var3D:
                log_main.add("Var3d                : %s" % options.var3D)
            if options.ave2D:
                log_main.add("Ave2D                : %s" % options.ave2D)
            if options.var2D:
                log_main.add("Var2D                : %s" % options.var2D)
            if options.VAR:
                log_main.add("VAR                  : True")
            else:
                log_main.add("VAR                  : False")
            if options.CTF:
                log_main.add("CTF correction       : True  ")
            else:
                log_main.add("CTF correction       : False ")

            log_main.add("Image per group      : %5d" % options.img_per_grp)
            log_main.add("Image decimate rate  : %4.3f" % current_decimate)
            log_main.add("Low pass filter      : %4.3f" % options.fl)
            current_fl = options.fl
            if current_fl == 0.0:
                current_fl = 0.5
            log_main.add(
                "Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"
                % round((current_fl * current_decimate), 3))
            log_main.add("Window size          : %5d " % current_window)
            log_main.add("sx3dvariability begins")

        symbaselen = 0
        if myid == main_node:
            nima = EMAN2_cppwrap.EMUtil.get_image_count(stack)
            img = sp_utilities.get_image(stack)
            nx = img.get_xsize()
            ny = img.get_ysize()
            nnxo = nx
            nnyo = ny
            if options.sym != "c1":
                imgdata = sp_utilities.get_im(stack)
                try:
                    i = imgdata.get_attr("variabilitysymmetry").lower()
                    if i != options.sym:
                        sp_global_def.ERROR(
                            "The symmetry provided does not agree with the symmetry of the input stack",
                            myid=myid,
                        )
                except:
                    sp_global_def.ERROR(
                        "Input stack is not prepared for symmetry, please follow instructions",
                        myid=myid,
                    )
                i = len(sp_utilities.get_symt(options.sym))
                if (old_div(nima, i)) * i != nima:
                    sp_global_def.ERROR(
                        "The length of the input stack is incorrect for symmetry processing",
                        myid=myid,
                    )
                symbaselen = old_div(nima, i)
            else:
                symbaselen = nima
        else:
            nima = 0
            nx = 0
            ny = 0
            nnxo = 0
            nnyo = 0
        nima = sp_utilities.bcast_number_to_all(nima)
        nx = sp_utilities.bcast_number_to_all(nx)
        ny = sp_utilities.bcast_number_to_all(ny)
        nnxo = sp_utilities.bcast_number_to_all(nnxo)
        nnyo = sp_utilities.bcast_number_to_all(nnyo)
        if current_window > max(nx, ny):
            sp_global_def.ERROR(
                "Window size is larger than the original image size")

        if current_decimate == 1.0:
            if current_window != 0:
                nx = current_window
                ny = current_window
        else:
            if current_window == 0:
                nx = int(nx * current_decimate + 0.5)
                ny = int(ny * current_decimate + 0.5)
            else:
                nx = int(current_window * current_decimate + 0.5)
                ny = nx
        symbaselen = sp_utilities.bcast_number_to_all(symbaselen)

        # check FFT prime number
        is_fft_friendly = nx == sp_fundamentals.smallprime(nx)

        if not is_fft_friendly:
            if myid == main_node:
                log_main.add(
                    "The target image size is not a product of small prime numbers"
                )
                log_main.add("Program adjusts the input settings!")
            ### two cases
            if current_decimate == 1.0:
                nx = sp_fundamentals.smallprime(nx)
                ny = nx
                current_window = nx  # update
                if myid == main_node:
                    log_main.add("The window size is updated to %d." %
                                 current_window)
            else:
                if current_window == 0:
                    nx = sp_fundamentals.smallprime(
                        int(nx * current_decimate + 0.5))
                    current_decimate = old_div(float(nx), nnxo)
                    ny = nx
                    if myid == main_node:
                        log_main.add("The decimate rate is updated to %f." %
                                     current_decimate)
                else:
                    nx = sp_fundamentals.smallprime(
                        int(current_window * current_decimate + 0.5))
                    ny = nx
                    current_window = int(old_div(nx, current_decimate) + 0.5)
                    if myid == main_node:
                        log_main.add("The window size is updated to %d." %
                                     current_window)

        if myid == main_node:
            log_main.add("The target image size is %d" % nx)

        if radiuspca == -1:
            radiuspca = old_div(nx, 2) - 2
        if myid == main_node:
            log_main.add("%-70s:  %d\n" % ("Number of projection", nima))
        img_begin, img_end = sp_applications.MPI_start_end(
            nima, number_of_proc, myid)
        """Multiline Comment0"""
        """
        Comments from adnan, replace index_of_proj to index_of_particle, index_of_proj was not defined
        also varList is not defined not made an empty list there
        """

        if options.VAR:  # 2D variance images have no shifts
            varList = []
            # varList   = EMData.read_images(stack, range(img_begin, img_end))
            for index_of_particle in range(img_begin, img_end):
                image = sp_utilities.get_im(stack, index_of_particle)
                if current_window > 0:
                    varList.append(
                        sp_fundamentals.fdecimate(
                            sp_fundamentals.window2d(image, current_window,
                                                     current_window),
                            nx,
                            ny,
                        ))
                else:
                    varList.append(sp_fundamentals.fdecimate(image, nx, ny))

        else:
            if myid == main_node:
                t1 = time.time()
                proj_angles = []
                aveList = []
                tab = EMAN2_cppwrap.EMUtil.get_all_attributes(
                    stack, "xform.projection")
                for i in range(nima):
                    t = tab[i].get_params("spider")
                    phi = t["phi"]
                    theta = t["theta"]
                    psi = t["psi"]
                    x = theta
                    if x > 90.0:
                        x = 180.0 - x
                    x = x * 10000 + psi
                    proj_angles.append([x, t["phi"], t["theta"], t["psi"], i])
                t2 = time.time()
                log_main.add(
                    "%-70s:  %d\n" %
                    ("Number of neighboring projections", img_per_grp))
                log_main.add("...... Finding neighboring projections\n")
                log_main.add("Number of images per group: %d" % img_per_grp)
                log_main.add("Now grouping projections")
                proj_angles.sort()
                proj_angles_list = numpy.full((nima, 4),
                                              0.0,
                                              dtype=numpy.float32)
                for i in range(nima):
                    proj_angles_list[i][0] = proj_angles[i][1]
                    proj_angles_list[i][1] = proj_angles[i][2]
                    proj_angles_list[i][2] = proj_angles[i][3]
                    proj_angles_list[i][3] = proj_angles[i][4]
            else:
                proj_angles_list = 0
            proj_angles_list = sp_utilities.wrap_mpi_bcast(
                proj_angles_list, main_node, mpi.MPI_COMM_WORLD)
            proj_angles = []
            for i in range(nima):
                proj_angles.append([
                    proj_angles_list[i][0],
                    proj_angles_list[i][1],
                    proj_angles_list[i][2],
                    int(proj_angles_list[i][3]),
                ])
            del proj_angles_list
            proj_list, mirror_list = sp_utilities.nearest_proj(
                proj_angles, img_per_grp, range(img_begin, img_end))
            all_proj = []
            for im in proj_list:
                for jm in im:
                    all_proj.append(proj_angles[jm][3])
            all_proj = list(set(all_proj))
            index = {}
            for i in range(len(all_proj)):
                index[all_proj[i]] = i
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            if myid == main_node:
                log_main.add("%-70s:  %.2f\n" %
                             ("Finding neighboring projections lasted [s]",
                              time.time() - t2))
                log_main.add("%-70s:  %d\n" %
                             ("Number of groups processed on the main node",
                              len(proj_list)))
                log_main.add("Grouping projections took:  %12.1f [m]" %
                             (old_div((time.time() - t2), 60.0)))
                log_main.add("Number of groups on main node: ", len(proj_list))
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)

            if myid == main_node:
                log_main.add("...... Calculating the stack of 2D variances \n")
            # Memory estimation. There are two memory consumption peaks
            # peak 1. Compute ave, var;
            # peak 2. Var volume reconstruction;
            # proj_params = [0.0]*(nima*5)
            aveList = []
            varList = []
            # if nvec > 0: eigList = [[] for i in range(nvec)]
            dnumber = len(
                all_proj)  # all neighborhood set for assigned to myid
            pnumber = len(proj_list) * 2.0 + img_per_grp  # aveList and varList
            tnumber = dnumber + pnumber
            vol_size2 = old_div(nx**3 * 4.0 * 8, 1.0e9)
            vol_size1 = old_div(2.0 * nnxo**3 * 4.0 * 8, 1.0e9)
            proj_size = old_div(nnxo * nnyo * len(proj_list) * 4.0 * 2.0,
                                1.0e9)  # both aveList and varList
            orig_data_size = old_div(nnxo * nnyo * 4.0 * tnumber, 1.0e9)
            reduced_data_size = old_div(nx * nx * 4.0 * tnumber, 1.0e9)
            full_data = numpy.full((number_of_proc, 2),
                                   -1.0,
                                   dtype=numpy.float16)
            full_data[myid] = orig_data_size, reduced_data_size
            if myid != main_node:
                sp_utilities.wrap_mpi_send(full_data, main_node,
                                           mpi.MPI_COMM_WORLD)
            if myid == main_node:
                for iproc in range(number_of_proc):
                    if iproc != main_node:
                        dummy = sp_utilities.wrap_mpi_recv(
                            iproc, mpi.MPI_COMM_WORLD)
                        full_data[numpy.where(dummy > -1)] = dummy[numpy.where(
                            dummy > -1)]
                del dummy
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            full_data = sp_utilities.wrap_mpi_bcast(full_data, main_node,
                                                    mpi.MPI_COMM_WORLD)
            # find the CPU with heaviest load
            minindx = numpy.argsort(full_data, 0)
            heavy_load_myid = minindx[-1][1]
            total_mem = sum(full_data)
            if myid == main_node:
                if current_window == 0:
                    log_main.add(
                        "Nx:   current image size = %d. Decimated by %f from %d"
                        % (nx, current_decimate, nnxo))
                else:
                    log_main.add(
                        "Nx:   current image size = %d. Windowed to %d, and decimated by %f from %d"
                        % (nx, current_window, current_decimate, nnxo))
                log_main.add("Nproj:       number of particle images.")
                log_main.add("Navg:        number of 2D average images.")
                log_main.add("Nvar:        number of 2D variance images.")
                log_main.add(
                    "Img_per_grp: user defined image per group for averaging = %d"
                    % img_per_grp)
                log_main.add(
                    "Overhead:    total python overhead memory consumption   = %f"
                    % overhead_loading)
                log_main.add(
                    "Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"
                    % (total_mem[1] + overhead_loading))
            del full_data
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            if myid == heavy_load_myid:
                log_main.add(
                    "Begin reading and preprocessing images on processor. Wait... "
                )
                ttt = time.time()
            # imgdata = EMData.read_images(stack, all_proj)
            imgdata = [None for im in range(len(all_proj))]
            for index_of_proj in range(len(all_proj)):
                # image = get_im(stack, all_proj[index_of_proj])
                if current_window > 0:
                    imgdata[index_of_proj] = sp_fundamentals.fdecimate(
                        sp_fundamentals.window2d(
                            sp_utilities.get_im(stack,
                                                all_proj[index_of_proj]),
                            current_window,
                            current_window,
                        ),
                        nx,
                        ny,
                    )
                else:
                    imgdata[index_of_proj] = sp_fundamentals.fdecimate(
                        sp_utilities.get_im(stack, all_proj[index_of_proj]),
                        nx, ny)

                if current_decimate > 0.0 and options.CTF:
                    ctf = imgdata[index_of_proj].get_attr("ctf")
                    ctf.apix = old_div(ctf.apix, current_decimate)
                    imgdata[index_of_proj].set_attr("ctf", ctf)

                if myid == heavy_load_myid and index_of_proj % 100 == 0:
                    log_main.add(
                        " ...... %6.2f%% " %
                        (old_div(index_of_proj, float(len(all_proj))) * 100.0))
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            if myid == heavy_load_myid:
                log_main.add("All_proj preprocessing cost %7.2f m" % (old_div(
                    (time.time() - ttt), 60.0)))
                log_main.add("Wait untill reading on all CPUs done...")
            """Multiline Comment1"""
            if not options.no_norm:
                mask = sp_utilities.model_circle(old_div(nx, 2) - 2, nx, nx)
            if myid == heavy_load_myid:
                log_main.add("Start computing 2D aveList and varList. Wait...")
                ttt = time.time()
            inner = old_div(nx, 2) - 4
            outer = inner + 2
            xform_proj_for_2D = [None for i in range(len(proj_list))]
            for i in range(len(proj_list)):
                ki = proj_angles[proj_list[i][0]][3]
                if ki >= symbaselen:
                    continue
                mi = index[ki]
                dpar = EMAN2_cppwrap.Util.get_transform_params(
                    imgdata[mi], "xform.projection", "spider")
                phiM, thetaM, psiM, s2xM, s2yM = (
                    dpar["phi"],
                    dpar["theta"],
                    dpar["psi"],
                    -dpar["tx"] * current_decimate,
                    -dpar["ty"] * current_decimate,
                )
                grp_imgdata = []
                for j in range(img_per_grp):
                    mj = index[proj_angles[proj_list[i][j]][3]]
                    cpar = EMAN2_cppwrap.Util.get_transform_params(
                        imgdata[mj], "xform.projection", "spider")
                    alpha, sx, sy, mirror = params_3D_2D_NEW(
                        cpar["phi"],
                        cpar["theta"],
                        cpar["psi"],
                        -cpar["tx"] * current_decimate,
                        -cpar["ty"] * current_decimate,
                        mirror_list[i][j],
                    )
                    if thetaM <= 90:
                        if mirror == 0:
                            alpha, sx, sy, scale = sp_utilities.compose_transform2(
                                alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0,
                                0.0, 1.0)
                        else:
                            alpha, sx, sy, scale = sp_utilities.compose_transform2(
                                alpha,
                                sx,
                                sy,
                                1.0,
                                180 - (phiM - cpar["phi"]),
                                0.0,
                                0.0,
                                1.0,
                            )
                    else:
                        if mirror == 0:
                            alpha, sx, sy, scale = sp_utilities.compose_transform2(
                                alpha, sx, sy, 1.0, -(phiM - cpar["phi"]), 0.0,
                                0.0, 1.0)
                        else:
                            alpha, sx, sy, scale = sp_utilities.compose_transform2(
                                alpha,
                                sx,
                                sy,
                                1.0,
                                -(180 - (phiM - cpar["phi"])),
                                0.0,
                                0.0,
                                1.0,
                            )
                    imgdata[mj].set_attr(
                        "xform.align2d",
                        EMAN2_cppwrap.Transform({
                            "type": "2D",
                            "alpha": alpha,
                            "tx": sx,
                            "ty": sy,
                            "mirror": mirror,
                            "scale": 1.0,
                        }),
                    )
                    grp_imgdata.append(imgdata[mj])
                if not options.no_norm:
                    for k in range(img_per_grp):
                        ave, std, minn, maxx = EMAN2_cppwrap.Util.infomask(
                            grp_imgdata[k], mask, False)
                        grp_imgdata[k] -= ave
                        grp_imgdata[k] = old_div(grp_imgdata[k], std)
                if options.fl > 0.0:
                    for k in range(img_per_grp):
                        grp_imgdata[k] = sp_filter.filt_tanl(
                            grp_imgdata[k], options.fl, options.aa)

                #  Because of background issues, only linear option works.
                if options.CTF:
                    ave, var = sp_statistics.aves_wiener(
                        grp_imgdata, SNR=1.0e5, interpolation_method="linear")
                else:
                    ave, var = sp_statistics.ave_var(grp_imgdata)
                # Switch to std dev
                # threshold is not really needed,it is just in case due to numerical accuracy something turns out negative.
                var = sp_morphology.square_root(sp_morphology.threshold(var))

                sp_utilities.set_params_proj(ave,
                                             [phiM, thetaM, 0.0, 0.0, 0.0])
                sp_utilities.set_params_proj(var,
                                             [phiM, thetaM, 0.0, 0.0, 0.0])

                aveList.append(ave)
                varList.append(var)
                xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0]
                """Multiline Comment2"""
                if (myid == heavy_load_myid) and (i % 100 == 0):
                    log_main.add(" ......%6.2f%%  " %
                                 (old_div(i, float(len(proj_list))) * 100.0))
            del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index
            if not options.no_norm:
                del mask
            if myid == main_node:
                del tab
            #  At this point, all averages and variances are computed
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)

            if myid == heavy_load_myid:
                log_main.add("Computing aveList and varList took %12.1f [m]" %
                             (old_div((time.time() - ttt), 60.0)))

            xform_proj_for_2D = sp_utilities.wrap_mpi_gatherv(
                xform_proj_for_2D, main_node, mpi.MPI_COMM_WORLD)
            if myid == main_node:
                sp_utilities.write_text_row(
                    [str(entry) for entry in xform_proj_for_2D],
                    optparse.os.path.join(current_output_dir, "params.txt"),
                )
            del xform_proj_for_2D
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            if options.ave2D:
                if myid == main_node:
                    log_main.add("Compute ave2D ... ")
                    km = 0
                    for i in range(number_of_proc):
                        if i == main_node:
                            for im in range(len(aveList)):
                                aveList[im].write_image(
                                    optparse.os.path.join(
                                        current_output_dir, options.ave2D),
                                    km,
                                )
                                km += 1
                        else:
                            nl = mpi.mpi_recv(
                                1,
                                mpi.MPI_INT,
                                i,
                                sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
                                mpi.MPI_COMM_WORLD,
                            )
                            nl = int(nl[0])
                            for im in range(nl):
                                ave = sp_utilities.recv_EMData(
                                    i, im + i + 70000)
                                """Multiline Comment3"""
                                tmpvol = sp_fundamentals.fpol(ave, nx, nx, 1)
                                tmpvol.write_image(
                                    optparse.os.path.join(
                                        current_output_dir, options.ave2D),
                                    km,
                                )
                                km += 1
                else:
                    mpi.mpi_send(
                        len(aveList),
                        1,
                        mpi.MPI_INT,
                        main_node,
                        sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
                        mpi.MPI_COMM_WORLD,
                    )
                    for im in range(len(aveList)):
                        sp_utilities.send_EMData(aveList[im], main_node,
                                                 im + myid + 70000)
                        """Multiline Comment4"""
                if myid == main_node:
                    sp_applications.header(
                        optparse.os.path.join(current_output_dir,
                                              options.ave2D),
                        params="xform.projection",
                        fimport=optparse.os.path.join(current_output_dir,
                                                      "params.txt"),
                    )
                mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            if options.ave3D:
                t5 = time.time()
                if myid == main_node:
                    log_main.add("Reconstruct ave3D ... ")
                ave3D = sp_reconstruction.recons3d_4nn_MPI(
                    myid, aveList, symmetry=options.sym, npad=options.npad)
                sp_utilities.bcast_EMData_to_all(ave3D, myid)
                if myid == main_node:
                    if current_decimate != 1.0:
                        ave3D = sp_fundamentals.resample(
                            ave3D, old_div(1.0, current_decimate))
                    ave3D = sp_fundamentals.fpol(
                        ave3D, nnxo, nnxo,
                        nnxo)  # always to the orignal image size
                    sp_utilities.set_pixel_size(ave3D, 1.0)
                    ave3D.write_image(
                        optparse.os.path.join(current_output_dir,
                                              options.ave3D))
                    log_main.add("Ave3D reconstruction took %12.1f [m]" %
                                 (old_div((time.time() - t5), 60.0)))
                    log_main.add("%-70s:  %s\n" %
                                 ("The reconstructed ave3D is saved as ",
                                  options.ave3D))

            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList
            """Multiline Comment5"""

            if options.ave3D:
                del ave3D
            if options.var2D:
                if myid == main_node:
                    log_main.add("Compute var2D...")
                    km = 0
                    for i in range(number_of_proc):
                        if i == main_node:
                            for im in range(len(varList)):
                                tmpvol = sp_fundamentals.fpol(
                                    varList[im], nx, nx, 1)
                                tmpvol.write_image(
                                    optparse.os.path.join(
                                        current_output_dir, options.var2D),
                                    km,
                                )
                                km += 1
                        else:
                            nl = mpi.mpi_recv(
                                1,
                                mpi.MPI_INT,
                                i,
                                sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
                                mpi.MPI_COMM_WORLD,
                            )
                            nl = int(nl[0])
                            for im in range(nl):
                                ave = sp_utilities.recv_EMData(
                                    i, im + i + 70000)
                                tmpvol = sp_fundamentals.fpol(ave, nx, nx, 1)
                                tmpvol.write_image(
                                    optparse.os.path.join(
                                        current_output_dir, options.var2D),
                                    km,
                                )
                                km += 1
                else:
                    mpi.mpi_send(
                        len(varList),
                        1,
                        mpi.MPI_INT,
                        main_node,
                        sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
                        mpi.MPI_COMM_WORLD,
                    )
                    for im in range(len(varList)):
                        sp_utilities.send_EMData(
                            varList[im], main_node,
                            im + myid + 70000)  # What with the attributes??
                mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
                if myid == main_node:
                    sp_applications.header(
                        optparse.os.path.join(current_output_dir,
                                              options.var2D),
                        params="xform.projection",
                        fimport=optparse.os.path.join(current_output_dir,
                                                      "params.txt"),
                    )
                mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        if options.var3D:
            if myid == main_node:
                log_main.add("Reconstruct var3D ...")
            t6 = time.time()
            # radiusvar = options.radius
            # if( radiusvar < 0 ):  radiusvar = nx//2 -3
            res = sp_reconstruction.recons3d_4nn_MPI(myid,
                                                     varList,
                                                     symmetry=options.sym,
                                                     npad=options.npad)
            # res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
            if myid == main_node:
                if current_decimate != 1.0:
                    res = sp_fundamentals.resample(
                        res, old_div(1.0, current_decimate))
                res = sp_fundamentals.fpol(res, nnxo, nnxo, nnxo)
                sp_utilities.set_pixel_size(res, 1.0)
                res.write_image(os.path.join(current_output_dir,
                                             options.var3D))
                log_main.add(
                    "%-70s:  %s\n" %
                    ("The reconstructed var3D is saved as ", options.var3D))
                log_main.add("Var3D reconstruction took %f12.1 [m]" % (old_div(
                    (time.time() - t6), 60.0)))
                log_main.add("Total computation time %f12.1 [m]" % (old_div(
                    (time.time() - t0), 60.0)))
                log_main.add("sx3dvariability finishes")

        if RUNNING_UNDER_MPI:
            sp_global_def.MPI = False

        sp_global_def.BATCH = False
Esempio n. 17
0
def main():
    arglist = []
    for arg in sys.argv:
        arglist.append(arg)
    progname = optparse.os.path.basename(arglist[0])
    usage = progname + """ inputvolume  locresvolume maskfile outputfile   --radius --falloff  --MPI

	    Locally filer a volume based on local resolution volume (sxlocres.py) within area outlined by the maskfile
	"""
    parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)

    parser.add_option(
        "--radius",
        type="int",
        default=-1,
        help=
        "if there is no maskfile, sphere with r=radius will be used, by default the radius is nx/2-1"
    )
    parser.add_option("--falloff",
                      type="float",
                      default=0.1,
                      help="falloff of tanl filter (default 0.1)")
    parser.add_option("--MPI",
                      action="store_true",
                      default=False,
                      help="use MPI version")

    (options, args) = parser.parse_args(arglist[1:])

    if len(args) < 3 or len(args) > 4:
        sp_global_def.sxprint("See usage " + usage)
        sp_global_def.ERROR(
            "Wrong number of parameters. Please see usage information above.")
        return

    if sp_global_def.CACHE_DISABLE:
        pass  #IMPORTIMPORTIMPORT from sp_utilities import disable_bdb_cache
        sp_utilities.disable_bdb_cache()

    if options.MPI:
        number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
        myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
        main_node = 0

        if (myid == main_node):
            #print sys.argv
            vi = sp_utilities.get_im(sys.argv[1])
            ui = sp_utilities.get_im(sys.argv[2])
            #print   Util.infomask(ui, None, True)
            radius = options.radius
            nx = vi.get_xsize()
            ny = vi.get_ysize()
            nz = vi.get_zsize()
            dis = [nx, ny, nz]
        else:
            falloff = 0.0
            radius = 0
            dis = [0, 0, 0]
            vi = None
            ui = None
        dis = sp_utilities.bcast_list_to_all(dis, myid, source_node=main_node)

        if (myid != main_node):
            nx = int(dis[0])
            ny = int(dis[1])
            nz = int(dis[2])
        radius = sp_utilities.bcast_number_to_all(radius, main_node)
        if len(args) == 3:
            if (radius == -1):
                radius = min(nx, ny, nz) // 2 - 1
            m = sp_utilities.model_circle(radius, nx, ny, nz)
            outvol = args[2]

        elif len(args) == 4:
            if (myid == main_node):
                m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
            else:
                m = sp_utilities.model_blank(nx, ny, nz)
            outvol = args[3]
            sp_utilities.bcast_EMData_to_all(m, myid, main_node)

        pass  #IMPORTIMPORTIMPORT from sp_filter import filterlocal
        filteredvol = sp_filter.filterlocal(ui, vi, m, options.falloff, myid,
                                            main_node, number_of_proc)

        if (myid == 0):
            filteredvol.write_image(outvol)

    else:
        vi = sp_utilities.get_im(args[0])
        ui = sp_utilities.get_im(
            args[1]
        )  # resolution volume, values are assumed to be from 0 to 0.5

        nn = vi.get_xsize()

        falloff = options.falloff

        if len(args) == 3:
            radius = options.radius
            if (radius == -1):
                radius = nn // 2 - 1
            m = sp_utilities.model_circle(radius, nn, nn, nn)
            outvol = args[2]

        elif len(args) == 4:
            m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
            outvol = args[3]

        sp_fundamentals.fftip(vi)  # this is the volume to be filtered

        #  Round all resolution numbers to two digits
        for x in range(nn):
            for y in range(nn):
                for z in range(nn):
                    ui.set_value_at_fast(x, y, z,
                                         round(ui.get_value_at(x, y, z), 2))
        st = EMAN2_cppwrap.Util.infomask(ui, m, True)

        filteredvol = sp_utilities.model_blank(nn, nn, nn)
        cutoff = max(st[2] - 0.01, 0.0)
        while (cutoff < st[3]):
            cutoff = round(cutoff + 0.01, 2)
            pt = EMAN2_cppwrap.Util.infomask(
                sp_morphology.threshold_outside(ui, cutoff - 0.00501,
                                                cutoff + 0.005), m, True)
            if (pt[0] != 0.0):
                vovo = sp_fundamentals.fft(
                    sp_filter.filt_tanl(vi, cutoff, falloff))
                for x in range(nn):
                    for y in range(nn):
                        for z in range(nn):
                            if (m.get_value_at(x, y, z) > 0.5):
                                if (round(ui.get_value_at(x, y, z),
                                          2) == cutoff):
                                    filteredvol.set_value_at_fast(
                                        x, y, z, vovo.get_value_at(x, y, z))

        sp_global_def.write_command(optparse.os.path.dirname(outvol))
        filteredvol.write_image(outvol)
Esempio n. 18
0
    def test_(self):
        testargs_new = [
            path.join(ABSOLUTE_BIN_PATH, "sp_ctf_refine.py"), "manual",
            "bdb:" + path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                               "06_SUBSTACK_ANO#isac_substack"),
            self.new_output_folder,
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "12_POSTREFINER", "vol_combined.hdf"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "11_MERIDIEN", "final_params_028.txt")
        ]

        print("2nd round")

        testargs_old = [
            path.join(ABSOLUTE_OLDBIN_PATH, "sp_ctf_refine.py"), "manual",
            "bdb:" + path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                               "06_SUBSTACK_ANO#isac_substack"),
            self.old_output_folder,
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "12_POSTREFINER", "vol_combined.hdf"),
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "11_MERIDIEN", "final_params_028.txt")
        ]
        with patch.object(sys, 'argv', testargs_new):
            fu._main_()
        with patch.object(sys, 'argv', testargs_old):
            oldfu._main_()
        fpath_new = 'bdb:{0}#{1}'.format(self.new_output_folder, self.bdb_name)
        fpath_old = 'bdb:{0}#{1}'.format(self.old_output_folder, self.bdb_name)
        return_old = get_im(fpath_old)
        return_new = get_im(fpath_new)
        self.assertTrue(
            array_equal(return_new.get_3dview(), return_old.get_3dview()))
        self.assertTrue(
            array_equal(return_new.get_3dview().flatten().tolist()[0:100], [
                -0.38441434502601624, -0.4573197066783905,
                -0.39960816502571106, -0.10431570559740067,
                0.018583765253424644, 0.37694355845451355, 0.4353419840335846,
                0.06498070061206818, -0.38467326760292053, -0.7100721001625061,
                -0.7571740746498108, -0.9328497648239136, -0.8550546169281006,
                -0.21433517336845398, 0.34148913621902466, 0.30916473269462585,
                0.39394184947013855, 0.4447155296802521, -0.032833874225616455,
                0.4121330976486206, 0.638403594493866, 0.34945985674858093,
                0.2751978039741516, -0.14872148633003235, -0.5307353138923645,
                -0.6574574112892151, -0.8945914506912231, -1.107301950454712,
                -1.3676011562347412, -1.2225056886672974, -1.2273787260055542,
                -0.7121877074241638, 0.06733409315347672, 0.29879996180534363,
                0.6046097874641418, 0.496036559343338, 0.7214235663414001,
                0.7652679681777954, 0.9319247603416443, 0.11259084939956665,
                -0.3054805397987366, -0.8183746933937073, -1.4462037086486816,
                -1.4775571823120117, -1.2116808891296387, -0.911469042301178,
                -0.9934141039848328, -0.9891195297241211, -1.0073580741882324,
                -1.234678864479065, -1.4292954206466675, -1.5580313205718994,
                -1.446936845779419, -1.67134428024292, -2.0199873447418213,
                -2.0543317794799805, -2.2354137897491455, -1.99867844581604,
                -1.4405670166015625, -1.7088592052459717, -2.060110569000244,
                -1.6585056781768799, -1.0386717319488525, -0.4783007502555847,
                -0.12102964520454407, 0.16874085366725922,
                -0.12371158599853516, 0.01841016113758087, -0.6442297101020813,
                -0.6972493529319763, -0.1051267459988594, 0.016293810680508614,
                -0.10689342767000198, -0.472159206867218, -0.4886413514614105,
                -0.09828135371208191, -0.7409976720809937, -0.8893253207206726,
                -1.1477444171905518, -0.35942375659942627,
                0.030692437663674355, 0.5380961298942566, 1.06252121925354,
                0.6961002349853516, 0.7563707232475281, 1.0197871923446655,
                0.9021824598312378, 0.34891727566719055, 0.402149498462677,
                0.634938657283783, 0.6044892072677612, 0.24482420086860657,
                -0.4132026731967926, -0.9945327043533325, -1.260424256324768,
                -1.6913681030273438, -1.7999876737594604, -1.5891680717468262,
                -1.3196191787719727, -0.9893324375152588
            ]))
Esempio n. 19
0
def combine_isac_params(isac_dir,
                        classavgstack,
                        chains_params_file,
                        old_combined_parts,
                        classdoc,
                        combined_params_file,
                        log=False,
                        verbose=False):
    """
	Combines initial and all_params from ISAC.
	
	Arguments:
		isac_dir : ISAC directory
		classavgstack : Input image stack
		chains_params_file : Input alignment parameters applied to averages in sp_chains
		old_combined_parts
		classdoc
		combined_params_file : Output combined alignment parameters
		log : instance of Logger class
		verbose : (boolean) Whether to write to screen
	"""

    from sp_utilities import combine_params2, read_text_row

    # File-handling
    init_params_file = os.path.join(isac_dir, "2dalignment",
                                    "initial2Dparams.txt")
    all_params_file = os.path.join(isac_dir, "all_parameters.txt")
    init_params_list = read_text_row(init_params_file)
    all_params = read_text_row(all_params_file)
    isac_shrink_path = os.path.join(isac_dir, "README_shrink_ratio.txt")
    isac_shrink_file = open(isac_shrink_path, "r")
    isac_shrink_lines = isac_shrink_file.readlines()
    isac_shrink_ratio = float(isac_shrink_lines[5])
    """
	Three cases:
		1) Using class_averages.hdf
		2) Using ordered_class_averages.hdf, but chains_params.txt doesn't exist
		3) Using ordered_class_averages.hdf and chains_params.txt 
	"""

    msg = "Combining alignment parameters from %s and %s, dividing by %s, and writing to %s" % \
     (init_params_file, all_params_file, isac_shrink_ratio, combined_params_file)

    # Check if using ordered class averages and whether chains_params exists
    if os.path.basename(classavgstack) == 'ordered_class_averages.hdf':
        if not os.path.exists(chains_params_file):
            msg += "WARNING: '%s' does not exist. " % chains_params_file
            msg += "         Using '%s' but alignment parameters correspond to 'class_averages.hdf'.\n" % classavgstack
        else:
            msg = "Combining alignment parameters from %s, %s, and %s, dividing by %s, and writing to %s" % \
             (init_params_file, all_params_file, chains_params_file, isac_shrink_ratio, combined_params_file)

    print_log_msg(msg, log, verbose)

    if os.path.basename(
            classavgstack) == 'ordered_class_averages.hdf' and os.path.exists(
                chains_params_file):
        chains_params_list = read_text_row(chains_params_file)
        old_combined_list = read_text_row(old_combined_parts)
        num_classes = EMUtil.get_image_count(classavgstack)
        tmp_combined = []

        # Loop through classes
        for class_num in range(num_classes):
            # Extract members
            image = get_im(classavgstack, class_num)
            members = sorted(image.get_attr("members"))
            old_class_list = read_text_row(classdoc.format(class_num))
            new_class_list = []

            # Loop through particles
            for idx, im in enumerate(members):
                tmp_par = combine_params2(
                    init_params_list[im][0], init_params_list[im][1],
                    init_params_list[im][2], init_params_list[im][3],
                    all_params[im][0], all_params[im][1] / isac_shrink_ratio,
                    all_params[im][2] / isac_shrink_ratio, all_params[im][3])

                # Combine with class-average parameters
                P = combine_params2(tmp_par[0], tmp_par[1], tmp_par[2],
                                    tmp_par[3],
                                    chains_params_list[class_num][2],
                                    chains_params_list[class_num][3],
                                    chains_params_list[class_num][4],
                                    chains_params_list[class_num][5])

                tmp_combined.append([im, P[0], P[1], P[2], P[3]])

                # Need to update class number in class docs
                old_part_num = old_class_list[idx]

                try:
                    new_part_num = old_combined_list.index(old_part_num)
                except ValueError:
                    print(
                        "Couldn't find particle: class_num %s, old_part_num %s, new_part_num %s"
                        % (class_num, old_part_num[0], new_part_num))

                new_class_list.append(new_part_num)
            # End particle-loop

            # Overwrite pre-existing class doc
            write_text_row(new_class_list, classdoc.format(class_num))
        # End class-loop

        # Sort by particle number
        combined_params = sorted(tmp_combined, key=itemgetter(0))

        # Remove first column
        for row in combined_params:
            del row[0]

    # Not applying alignments of ordered_class_averages
    else:
        combined_params = []

        # Loop through images
        for im in range(len(all_params)):
            P = combine_params2(
                init_params_list[im][0], init_params_list[im][1],
                init_params_list[im][2], init_params_list[im][3],
                all_params[im][0], all_params[im][1] / isac_shrink_ratio,
                all_params[im][2] / isac_shrink_ratio, all_params[im][3])
            combined_params.append([P[0], P[1], P[2], P[3], 1.0])

    write_text_row(combined_params, combined_params_file)
    print_log_msg(
        'Wrote %s entries to %s\n' %
        (len(combined_params), combined_params_file), log, verbose)

    return combined_params
Esempio n. 20
0
def shiftali_MPI(stack,
                 maskfile=None,
                 maxit=100,
                 CTF=False,
                 snr=1.0,
                 Fourvar=False,
                 search_rng=-1,
                 oneDx=False,
                 search_rng_y=-1):

    number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
    myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
    main_node = 0

    ftp = file_type(stack)

    if myid == main_node:
        print_begin_msg("shiftali_MPI")

    max_iter = int(maxit)

    if myid == main_node:
        if ftp == "bdb":
            from EMAN2db import db_open_dict
            dummy = db_open_dict(stack, True)
        nima = EMUtil.get_image_count(stack)
    else:
        nima = 0
    nima = bcast_number_to_all(nima, source_node=main_node)
    list_of_particles = list(range(nima))

    image_start, image_end = MPI_start_end(nima, number_of_proc, myid)
    list_of_particles = list_of_particles[image_start:image_end]

    # read nx and ctf_app (if CTF) and broadcast to all nodes
    if myid == main_node:
        ima = EMData()
        ima.read_image(stack, list_of_particles[0], True)
        nx = ima.get_xsize()
        ny = ima.get_ysize()
        if CTF: ctf_app = ima.get_attr_default('ctf_applied', 2)
        del ima
    else:
        nx = 0
        ny = 0
        if CTF: ctf_app = 0
    nx = bcast_number_to_all(nx, source_node=main_node)
    ny = bcast_number_to_all(ny, source_node=main_node)
    if CTF:
        ctf_app = bcast_number_to_all(ctf_app, source_node=main_node)
        if ctf_app > 0:
            ERROR("data cannot be ctf-applied", myid=myid)

    if maskfile == None:
        mrad = min(nx, ny)
        mask = model_circle(mrad // 2 - 2, nx, ny)
    else:
        mask = get_im(maskfile)

    if CTF:
        from sp_filter import filt_ctf
        from sp_morphology import ctf_img
        ctf_abs_sum = EMData(nx, ny, 1, False)
        ctf_2_sum = EMData(nx, ny, 1, False)
    else:
        ctf_2_sum = None

    from sp_global_def import CACHE_DISABLE
    if CACHE_DISABLE:
        data = EMData.read_images(stack, list_of_particles)
    else:
        for i in range(number_of_proc):
            if myid == i:
                data = EMData.read_images(stack, list_of_particles)
            if ftp == "bdb": mpi.mpi_barrier(mpi.MPI_COMM_WORLD)

    for im in range(len(data)):
        data[im].set_attr('ID', list_of_particles[im])
        st = Util.infomask(data[im], mask, False)
        data[im] -= st[0]
        if CTF:
            ctf_params = data[im].get_attr("ctf")
            ctfimg = ctf_img(nx, ctf_params, ny=ny)
            Util.add_img2(ctf_2_sum, ctfimg)
            Util.add_img_abs(ctf_abs_sum, ctfimg)

    if CTF:
        reduce_EMData_to_root(ctf_2_sum, myid, main_node)
        reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
    else:
        ctf_2_sum = None
    if CTF:
        if myid != main_node:
            del ctf_2_sum
            del ctf_abs_sum
        else:
            temp = EMData(nx, ny, 1, False)
            for i in range(0, nx, 2):
                for j in range(ny):
                    temp.set_value_at(i, j, snr)
            Util.add_img(ctf_2_sum, temp)
            del temp

    total_iter = 0

    # apply initial xform.align2d parameters stored in header
    init_params = []
    for im in range(len(data)):
        t = data[im].get_attr('xform.align2d')
        init_params.append(t)
        p = t.get_params("2d")
        data[im] = rot_shift2D(data[im],
                               p['alpha'],
                               sx=p['tx'],
                               sy=p['ty'],
                               mirror=p['mirror'],
                               scale=p['scale'])

    # fourier transform all images, and apply ctf if CTF
    for im in range(len(data)):
        if CTF:
            ctf_params = data[im].get_attr("ctf")
            data[im] = filt_ctf(fft(data[im]), ctf_params)
        else:
            data[im] = fft(data[im])

    sx_sum = 0
    sy_sum = 0
    sx_sum_total = 0
    sy_sum_total = 0
    shift_x = [0.0] * len(data)
    shift_y = [0.0] * len(data)
    ishift_x = [0.0] * len(data)
    ishift_y = [0.0] * len(data)

    for Iter in range(max_iter):
        if myid == main_node:
            start_time = time()
            print_msg("Iteration #%4d\n" % (total_iter))
        total_iter += 1
        avg = EMData(nx, ny, 1, False)
        for im in data:
            Util.add_img(avg, im)

        reduce_EMData_to_root(avg, myid, main_node)

        if myid == main_node:
            if CTF:
                tavg = Util.divn_filter(avg, ctf_2_sum)
            else:
                tavg = Util.mult_scalar(avg, 1.0 / float(nima))
        else:
            tavg = EMData(nx, ny, 1, False)

        if Fourvar:
            bcast_EMData_to_all(tavg, myid, main_node)
            vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)

        if myid == main_node:
            if Fourvar:
                tavg = fft(Util.divn_img(fft(tavg), vav))
                vav_r = Util.pack_complex_to_real(vav)

            # normalize and mask tavg in real space
            tavg = fft(tavg)
            stat = Util.infomask(tavg, mask, False)
            tavg -= stat[0]
            Util.mul_img(tavg, mask)
            # For testing purposes: shift tavg to some random place and see if the centering is still correct
            #tavg = rot_shift3D(tavg,sx=3,sy=-4)
            tavg = fft(tavg)

        if Fourvar: del vav
        bcast_EMData_to_all(tavg, myid, main_node)

        sx_sum = 0
        sy_sum = 0
        if search_rng > 0: nwx = 2 * search_rng + 1
        else: nwx = nx

        if search_rng_y > 0: nwy = 2 * search_rng_y + 1
        else: nwy = ny

        not_zero = 0
        for im in range(len(data)):
            if oneDx:
                ctx = Util.window(ccf(data[im], tavg), nwx, 1)
                p1 = peak_search(ctx)
                p1_x = -int(p1[0][3])
                ishift_x[im] = p1_x
                sx_sum += p1_x
            else:
                p1 = peak_search(Util.window(ccf(data[im], tavg), nwx, nwy))
                p1_x = -int(p1[0][4])
                p1_y = -int(p1[0][5])
                ishift_x[im] = p1_x
                ishift_y[im] = p1_y
                sx_sum += p1_x
                sy_sum += p1_y

            if not_zero == 0:
                if (not (ishift_x[im] == 0.0)) or (not (ishift_y[im] == 0.0)):
                    not_zero = 1

        sx_sum = mpi.mpi_reduce(sx_sum, 1, mpi.MPI_INT, mpi.MPI_SUM, main_node,
                                mpi.MPI_COMM_WORLD)

        if not oneDx:
            sy_sum = mpi.mpi_reduce(sy_sum, 1, mpi.MPI_INT, mpi.MPI_SUM,
                                    main_node, mpi.MPI_COMM_WORLD)

        if myid == main_node:
            sx_sum_total = int(sx_sum[0])
            if not oneDx:
                sy_sum_total = int(sy_sum[0])
        else:
            sx_sum_total = 0
            sy_sum_total = 0

        sx_sum_total = bcast_number_to_all(sx_sum_total, source_node=main_node)

        if not oneDx:
            sy_sum_total = bcast_number_to_all(sy_sum_total,
                                               source_node=main_node)

        sx_ave = round(float(sx_sum_total) / nima)
        sy_ave = round(float(sy_sum_total) / nima)
        for im in range(len(data)):
            p1_x = ishift_x[im] - sx_ave
            p1_y = ishift_y[im] - sy_ave
            params2 = {
                "filter_type": Processor.fourier_filter_types.SHIFT,
                "x_shift": p1_x,
                "y_shift": p1_y,
                "z_shift": 0.0
            }
            data[im] = Processor.EMFourierFilter(data[im], params2)
            shift_x[im] += p1_x
            shift_y[im] += p1_y
        # stop if all shifts are zero
        not_zero = mpi.mpi_reduce(not_zero, 1, mpi.MPI_INT, mpi.MPI_SUM,
                                  main_node, mpi.MPI_COMM_WORLD)
        if myid == main_node:
            not_zero_all = int(not_zero[0])
        else:
            not_zero_all = 0
        not_zero_all = bcast_number_to_all(not_zero_all, source_node=main_node)

        if myid == main_node:
            print_msg("Time of iteration = %12.2f\n" % (time() - start_time))
            start_time = time()

        if not_zero_all == 0: break

    #for im in xrange(len(data)): data[im] = fft(data[im])  This should not be required as only header information is used
    # combine shifts found with the original parameters
    for im in range(len(data)):
        t0 = init_params[im]
        t1 = Transform()
        t1.set_params({
            "type": "2D",
            "alpha": 0,
            "scale": t0.get_scale(),
            "mirror": 0,
            "tx": shift_x[im],
            "ty": shift_y[im]
        })
        # combine t0 and t1
        tt = t1 * t0
        data[im].set_attr("xform.align2d", tt)

    # write out headers and STOP, under MPI writing has to be done sequentially
    mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
    par_str = ["xform.align2d", "ID"]
    if myid == main_node:
        from sp_utilities import file_type
        if (file_type(stack) == "bdb"):
            from sp_utilities import recv_attr_dict_bdb
            recv_attr_dict_bdb(main_node, stack, data, par_str, image_start,
                               image_end, number_of_proc)
        else:
            from sp_utilities import recv_attr_dict
            recv_attr_dict(main_node, stack, data, par_str, image_start,
                           image_end, number_of_proc)

    else:
        send_attr_dict(main_node, data, par_str, image_start, image_end)
    if myid == main_node: print_end_msg("shiftali_MPI")
Esempio n. 21
0
def main():
    progname = os.path.basename(sys.argv[0])
    usage = progname + " prj_stack .. average eigvol output_factcoords --rad=radius --neigvol=number_of_eigvol  --CTF"
    parser = OptionParser(usage, version=SPARXVERSION)
    parser.add_option("--rad", type="int", default=-1, help="radius of mask")
    parser.add_option("--neigvol",
                      type="int",
                      default=-1,
                      help="number of eigvenvectors to use (default all)")
    parser.add_option(
        "--fl",
        type="float",
        default=0.0,
        help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
    parser.add_option(
        "--aa",
        type="float",
        default=0.0,
        help="fall-off of hyperbolic tangent low-pass Fourier filter")
    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="Use CTF")
    parser.add_option("--MPI", action="store_true", help="use MPI")

    (options, args) = parser.parse_args()

    if (len(args) < 4):
        sxprint("Usage: " + usage)
        sxprint("Please run \'" + progname + " -h\' for detailed options")
        ERROR(
            "Invalid number of parameters used. Please see usage information above."
        )
        return

    else:
        stacks = args[0:-3]
        avgvol = args[-3]
        eigvol = args[-2]
        output = args[-1]

        if options.rad < 0:
            ERROR("Mask radius is not given")
            return

        if sp_global_def.CACHE_DISABLE:
            from sp_utilities import disable_bdb_cache
            disable_bdb_cache()

        from sp_utilities import get_im
        sp_global_def.BATCH = True

        if (get_im(stacks[0]).get_zsize() == 1
                and get_im(eigvol).get_zsize() > 1):
            from sp_applications import factcoords_prj
            factcoords_prj(stacks, avgvol, eigvol, output, options.rad,
                           options.neigvol, options.fl, options.aa,
                           options.CTF, options.MPI)
        else:
            from sp_applications import factcoords_vol
            factcoords_vol(stacks, avgvol, eigvol, output, options.rad,
                           options.neigvol, options.fl, options.aa,
                           options.MPI)
        sp_global_def.BATCH = False
Esempio n. 22
0
def main():
    from sp_logger import Logger, BaseLogger_Files
    arglist = []
    i = 0
    while (i < len(sys.argv)):
        if sys.argv[i] == '-p4pg':
            i = i + 2
        elif sys.argv[i] == '-p4wd':
            i = i + 2
        else:
            arglist.append(sys.argv[i])
            i = i + 1
    progname = os.path.basename(arglist[0])
    usage = progname + " stack  outdir  <mask> --focus=3Dmask --radius=outer_radius --delta=angular_step" +\
    "--an=angular_neighborhood --maxit=max_iter  --CTF --sym=c1 --function=user_function --independent=indenpendent_runs  --number_of_images_per_group=number_of_images_per_group  --low_pass_filter=.25  --seed=random_seed"
    parser = OptionParser(usage, version=SPARXVERSION)
    parser.add_option("--focus",
                      type="string",
                      default='',
                      help="bineary 3D mask for focused clustering ")
    parser.add_option(
        "--ir",
        type="int",
        default=1,
        help="inner radius for rotational correlation > 0 (set to 1)")
    parser.add_option(
        "--radius",
        type="int",
        default=-1,
        help=
        "particle radius in pixel for rotational correlation <nx-1 (set to the radius of the particle)"
    )
    parser.add_option("--maxit",
                      type="int",
                      default=25,
                      help="maximum number of iteration")
    parser.add_option(
        "--rs",
        type="int",
        default=1,
        help="step between rings in rotational correlation >0 (set to 1)")
    parser.add_option(
        "--xr",
        type="string",
        default='1',
        help="range for translation search in x direction, search is +/-xr ")
    parser.add_option(
        "--yr",
        type="string",
        default='-1',
        help=
        "range for translation search in y direction, search is +/-yr (default = same as xr)"
    )
    parser.add_option(
        "--ts",
        type="string",
        default='0.25',
        help=
        "step size of the translation search in both directions direction, search is -xr, -xr+ts, 0, xr-ts, xr "
    )
    parser.add_option("--delta",
                      type="string",
                      default='2',
                      help="angular step of reference projections")
    parser.add_option("--an",
                      type="string",
                      default='-1',
                      help="angular neighborhood for local searches")
    parser.add_option(
        "--center",
        type="int",
        default=0,
        help=
        "0 - if you do not want the volume to be centered, 1 - center the volume using cog (default=0)"
    )
    parser.add_option(
        "--nassign",
        type="int",
        default=1,
        help=
        "number of reassignment iterations performed for each angular step (set to 3) "
    )
    parser.add_option(
        "--nrefine",
        type="int",
        default=0,
        help=
        "number of alignment iterations performed for each angular step (set to 0)"
    )
    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="do CTF correction during clustring")
    parser.add_option(
        "--stoprnct",
        type="float",
        default=3.0,
        help="Minimum percentage of assignment change to stop the program")
    parser.add_option("--sym",
                      type="string",
                      default='c1',
                      help="symmetry of the structure ")
    parser.add_option("--function",
                      type="string",
                      default='do_volume_mrk05',
                      help="name of the reference preparation function")
    parser.add_option("--independent",
                      type="int",
                      default=3,
                      help="number of independent run")
    parser.add_option("--number_of_images_per_group",
                      type="int",
                      default=1000,
                      help="number of groups")
    parser.add_option(
        "--low_pass_filter",
        type="float",
        default=-1.0,
        help=
        "absolute frequency of low-pass filter for 3d sorting on the original image size"
    )
    parser.add_option("--nxinit",
                      type="int",
                      default=64,
                      help="initial image size for sorting")
    parser.add_option("--unaccounted",
                      action="store_true",
                      default=False,
                      help="reconstruct the unaccounted images")
    parser.add_option(
        "--seed",
        type="int",
        default=-1,
        help="random seed for create initial random assignment for EQ Kmeans")
    parser.add_option("--smallest_group",
                      type="int",
                      default=500,
                      help="minimum members for identified group")
    parser.add_option("--sausage",
                      action="store_true",
                      default=False,
                      help="way of filter volume")
    parser.add_option("--chunk0",
                      type="string",
                      default='',
                      help="chunk0 for computing margin of error")
    parser.add_option("--chunk1",
                      type="string",
                      default='',
                      help="chunk1 for computing margin of error")
    parser.add_option(
        "--PWadjustment",
        type="string",
        default='',
        help=
        "1-D power spectrum of PDB file used for EM volume power spectrum correction"
    )
    parser.add_option(
        "--protein_shape",
        type="string",
        default='g',
        help=
        "protein shape. It defines protein preferred orientation angles. Currently it has g and f two types "
    )
    parser.add_option(
        "--upscale",
        type="float",
        default=0.5,
        help=" scaling parameter to adjust the power spectrum of EM volumes")
    parser.add_option("--wn",
                      type="int",
                      default=0,
                      help="optimal window size for data processing")
    parser.add_option(
        "--interpolation",
        type="string",
        default="4nn",
        help="3-d reconstruction interpolation method, two options trl and 4nn"
    )

    (options, args) = parser.parse_args(arglist[1:])

    if len(args) < 1 or len(args) > 4:
        sxprint("Usage: " + usage)
        sxprint("Please run \'" + progname + " -h\' for detailed options")
        ERROR(
            "Invalid number of parameters used. Please see usage information above."
        )
        return

    else:

        if len(args) > 2:
            mask_file = args[2]
        else:
            mask_file = None

        orgstack = args[0]
        masterdir = args[1]
        sp_global_def.BATCH = True
        #---initialize MPI related variables
        nproc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
        myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
        mpi_comm = mpi.MPI_COMM_WORLD
        main_node = 0
        # import some utilities
        from sp_utilities import get_im, bcast_number_to_all, cmdexecute, write_text_file, read_text_file, wrap_mpi_bcast, get_params_proj, write_text_row
        from sp_applications import recons3d_n_MPI, mref_ali3d_MPI, Kmref_ali3d_MPI
        from sp_statistics import k_means_match_clusters_asg_new, k_means_stab_bbenum
        from sp_applications import mref_ali3d_EQ_Kmeans, ali3d_mref_Kmeans_MPI
        # Create the main log file
        from sp_logger import Logger, BaseLogger_Files
        if myid == main_node:
            log_main = Logger(BaseLogger_Files())
            log_main.prefix = masterdir + "/"
        else:
            log_main = None
        #--- fill input parameters into dictionary named after Constants
        Constants = {}
        Constants["stack"] = args[0]
        Constants["masterdir"] = masterdir
        Constants["mask3D"] = mask_file
        Constants["focus3Dmask"] = options.focus
        Constants["indep_runs"] = options.independent
        Constants["stoprnct"] = options.stoprnct
        Constants[
            "number_of_images_per_group"] = options.number_of_images_per_group
        Constants["CTF"] = options.CTF
        Constants["maxit"] = options.maxit
        Constants["ir"] = options.ir
        Constants["radius"] = options.radius
        Constants["nassign"] = options.nassign
        Constants["rs"] = options.rs
        Constants["xr"] = options.xr
        Constants["yr"] = options.yr
        Constants["ts"] = options.ts
        Constants["delta"] = options.delta
        Constants["an"] = options.an
        Constants["sym"] = options.sym
        Constants["center"] = options.center
        Constants["nrefine"] = options.nrefine
        #Constants["fourvar"]            		 = options.fourvar
        Constants["user_func"] = options.function
        Constants[
            "low_pass_filter"] = options.low_pass_filter  # enforced low_pass_filter
        #Constants["debug"]              		 = options.debug
        Constants["main_log_prefix"] = args[1]
        #Constants["importali3d"]        		 = options.importali3d
        Constants["myid"] = myid
        Constants["main_node"] = main_node
        Constants["nproc"] = nproc
        Constants["log_main"] = log_main
        Constants["nxinit"] = options.nxinit
        Constants["unaccounted"] = options.unaccounted
        Constants["seed"] = options.seed
        Constants["smallest_group"] = options.smallest_group
        Constants["sausage"] = options.sausage
        Constants["chunk0"] = options.chunk0
        Constants["chunk1"] = options.chunk1
        Constants["PWadjustment"] = options.PWadjustment
        Constants["upscale"] = options.upscale
        Constants["wn"] = options.wn
        Constants["3d-interpolation"] = options.interpolation
        Constants["protein_shape"] = options.protein_shape
        # -----------------------------------------------------
        #
        # Create and initialize Tracker dictionary with input options
        Tracker = {}
        Tracker["constants"] = Constants
        Tracker["maxit"] = Tracker["constants"]["maxit"]
        Tracker["radius"] = Tracker["constants"]["radius"]
        #Tracker["xr"]             = ""
        #Tracker["yr"]             = "-1"  # Do not change!
        #Tracker["ts"]             = 1
        #Tracker["an"]             = "-1"
        #Tracker["delta"]          = "2.0"
        #Tracker["zoom"]           = True
        #Tracker["nsoft"]          = 0
        #Tracker["local"]          = False
        #Tracker["PWadjustment"]   = Tracker["constants"]["PWadjustment"]
        Tracker["upscale"] = Tracker["constants"]["upscale"]
        #Tracker["upscale"]        = 0.5
        Tracker[
            "applyctf"] = False  #  Should the data be premultiplied by the CTF.  Set to False for local continuous.
        #Tracker["refvol"]         = None
        Tracker["nxinit"] = Tracker["constants"]["nxinit"]
        #Tracker["nxstep"]         = 32
        Tracker["icurrentres"] = -1
        #Tracker["ireachedres"]    = -1
        #Tracker["lowpass"]        = 0.4
        #Tracker["falloff"]        = 0.2
        #Tracker["inires"]         = options.inires  # Now in A, convert to absolute before using
        Tracker["fuse_freq"] = 50  # Now in A, convert to absolute before using
        #Tracker["delpreviousmax"] = False
        #Tracker["anger"]          = -1.0
        #Tracker["shifter"]        = -1.0
        #Tracker["saturatecrit"]   = 0.95
        #Tracker["pixercutoff"]    = 2.0
        #Tracker["directory"]      = ""
        #Tracker["previousoutputdir"] = ""
        #Tracker["eliminated-outliers"] = False
        #Tracker["mainiteration"]  = 0
        #Tracker["movedback"]      = False
        #Tracker["state"]          = Tracker["constants"]["states"][0]
        #Tracker["global_resolution"] =0.0
        Tracker["orgstack"] = orgstack
        #--------------------------------------------------------------------
        # import from utilities
        from sp_utilities import sample_down_1D_curve, get_initial_ID, remove_small_groups, print_upper_triangular_matrix, print_a_line_with_timestamp
        from sp_utilities import print_dict, get_resolution_mrk01, partition_to_groups, partition_independent_runs, get_outliers
        from sp_utilities import merge_groups, save_alist, margin_of_error, get_margin_of_error, do_two_way_comparison, select_two_runs, get_ali3d_params
        from sp_utilities import counting_projections, unload_dict, load_dict, get_stat_proj, create_random_list, get_number_of_groups, recons_mref
        from sp_utilities import apply_low_pass_filter, get_groups_from_partition, get_number_of_groups, get_complementary_elements_total, update_full_dict
        from sp_utilities import count_chunk_members, set_filter_parameters_from_adjusted_fsc, get_two_chunks_from_stack
        ####------------------------------------------------------------------
        #
        # Get the pixel size; if none, set to 1.0, and the original image size
        from sp_utilities import get_shrink_data_huang
        if (myid == main_node):
            line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
            sxprint((line + "Initialization of 3-D sorting"))
            a = get_im(orgstack)
            nnxo = a.get_xsize()
            if (Tracker["nxinit"] > nnxo):
                sp_global_def.ERROR(
                    "Image size less than minimum permitted $d" %
                    Tracker["nxinit"])
                nnxo = -1
            else:
                if Tracker["constants"]["CTF"]:
                    i = a.get_attr('ctf')
                    pixel_size = i.apix
                    fq = pixel_size / Tracker["fuse_freq"]
                else:
                    pixel_size = 1.0
                    #  No pixel size, fusing computed as 5 Fourier pixels
                    fq = 5.0 / nnxo
                    del a
        else:
            nnxo = 0
            fq = 0.0
            pixel_size = 1.0
        nnxo = bcast_number_to_all(nnxo, source_node=main_node)
        if (nnxo < 0):
            return
        pixel_size = bcast_number_to_all(pixel_size, source_node=main_node)
        fq = bcast_number_to_all(fq, source_node=main_node)
        if Tracker["constants"]["wn"] == 0:
            Tracker["constants"]["nnxo"] = nnxo
        else:
            Tracker["constants"]["nnxo"] = Tracker["constants"]["wn"]
            nnxo = Tracker["constants"]["nnxo"]
        Tracker["constants"]["pixel_size"] = pixel_size
        Tracker["fuse_freq"] = fq
        del fq, nnxo, pixel_size
        if (Tracker["constants"]["radius"] < 1):
            Tracker["constants"][
                "radius"] = Tracker["constants"]["nnxo"] // 2 - 2
        elif ((2 * Tracker["constants"]["radius"] + 2) >
              Tracker["constants"]["nnxo"]):
            sp_global_def.ERROR("Particle radius set too large!", myid=myid)


####-----------------------------------------------------------------------------------------
# Master directory
        if myid == main_node:
            if masterdir == "":
                timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
                masterdir = "master_sort3d" + timestring
            li = len(masterdir)
            cmd = "{} {}".format("mkdir -p", masterdir)
            os.system(cmd)
        else:
            li = 0
        li = mpi.mpi_bcast(li, 1, mpi.MPI_INT, main_node,
                           mpi.MPI_COMM_WORLD)[0]
        if li > 0:
            masterdir = mpi.mpi_bcast(masterdir, li, mpi.MPI_CHAR, main_node,
                                      mpi.MPI_COMM_WORLD)
            import string
            masterdir = string.join(masterdir, "")
        if myid == main_node:
            print_dict(Tracker["constants"],
                       "Permanent settings of 3-D sorting program")
        ######### create a vstack from input stack to the local stack in masterdir
        # stack name set to default
        Tracker["constants"]["stack"] = "bdb:" + masterdir + "/rdata"
        Tracker["constants"]["ali3d"] = os.path.join(masterdir,
                                                     "ali3d_init.txt")
        Tracker["constants"]["ctf_params"] = os.path.join(
            masterdir, "ctf_params.txt")
        Tracker["constants"]["partstack"] = Tracker["constants"][
            "ali3d"]  # also serves for refinement
        if myid == main_node:
            total_stack = EMUtil.get_image_count(Tracker["orgstack"])
        else:
            total_stack = 0
        total_stack = bcast_number_to_all(total_stack, source_node=main_node)
        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        from time import sleep
        while not os.path.exists(masterdir):
            sxprint("Node ", myid, "  waiting...")
            sleep(5)
        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        if myid == main_node:
            log_main.add("Sphire sort3d ")
            log_main.add("the sort3d master directory is " + masterdir)
        #####
        ###----------------------------------------------------------------------------------
        # Initial data analysis and handle two chunk files
        from random import shuffle
        # Compute the resolution
        #### make chunkdir dictionary for computing margin of error
        import sp_user_functions
        user_func = sp_user_functions.factory[Tracker["constants"]
                                              ["user_func"]]
        chunk_dict = {}
        chunk_list = []
        if myid == main_node:
            chunk_one = read_text_file(Tracker["constants"]["chunk0"])
            chunk_two = read_text_file(Tracker["constants"]["chunk1"])
        else:
            chunk_one = 0
            chunk_two = 0
        chunk_one = wrap_mpi_bcast(chunk_one, main_node)
        chunk_two = wrap_mpi_bcast(chunk_two, main_node)
        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        ######################## Read/write bdb: data on main node ############################
        if myid == main_node:
            if (orgstack[:4] == "bdb:"):
                cmd = "{} {} {}".format(
                    "e2bdb.py", orgstack,
                    "--makevstack=" + Tracker["constants"]["stack"])
            else:
                cmd = "{} {} {}".format("sp_cpy.py", orgstack,
                                        Tracker["constants"]["stack"])
            junk = cmdexecute(cmd)
            cmd = "{} {} {}".format(
                "sp_header.py  --params=xform.projection",
                "--export=" + Tracker["constants"]["ali3d"], orgstack)
            junk = cmdexecute(cmd)
            cmd = "{} {} {}".format(
                "sp_header.py  --params=ctf",
                "--export=" + Tracker["constants"]["ctf_params"], orgstack)
            junk = cmdexecute(cmd)
        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        ########-----------------------------------------------------------------------------
        Tracker["total_stack"] = total_stack
        Tracker["constants"]["total_stack"] = total_stack
        Tracker["shrinkage"] = float(
            Tracker["nxinit"]) / Tracker["constants"]["nnxo"]
        Tracker[
            "radius"] = Tracker["constants"]["radius"] * Tracker["shrinkage"]
        if Tracker["constants"]["mask3D"]:
            Tracker["mask3D"] = os.path.join(masterdir, "smask.hdf")
        else:
            Tracker["mask3D"] = None
        if Tracker["constants"]["focus3Dmask"]:
            Tracker["focus3D"] = os.path.join(masterdir, "sfocus.hdf")
        else:
            Tracker["focus3D"] = None
        if myid == main_node:
            if Tracker["constants"]["mask3D"]:
                mask_3D = get_shrink_3dmask(Tracker["nxinit"],
                                            Tracker["constants"]["mask3D"])
                mask_3D.write_image(Tracker["mask3D"])
            if Tracker["constants"]["focus3Dmask"]:
                mask_3D = get_shrink_3dmask(
                    Tracker["nxinit"], Tracker["constants"]["focus3Dmask"])
                st = Util.infomask(mask_3D, None, True)
                if (st[0] == 0.0):
                    ERROR(
                        "Incorrect focused mask, after binarize all values zero"
                    )
                mask_3D.write_image(Tracker["focus3D"])
                del mask_3D
        if Tracker["constants"]["PWadjustment"] != '':
            PW_dict = {}
            nxinit_pwsp = sample_down_1D_curve(
                Tracker["constants"]["nxinit"], Tracker["constants"]["nnxo"],
                Tracker["constants"]["PWadjustment"])
            Tracker["nxinit_PW"] = os.path.join(masterdir, "spwp.txt")
            if myid == main_node:
                write_text_file(nxinit_pwsp, Tracker["nxinit_PW"])
            PW_dict[Tracker["constants"]
                    ["nnxo"]] = Tracker["constants"]["PWadjustment"]
            PW_dict[Tracker["constants"]["nxinit"]] = Tracker["nxinit_PW"]
            Tracker["PW_dict"] = PW_dict
        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        #-----------------------From two chunks to FSC, and low pass filter-----------------------------------------###
        for element in chunk_one:
            chunk_dict[element] = 0
        for element in chunk_two:
            chunk_dict[element] = 1
        chunk_list = [chunk_one, chunk_two]
        Tracker["chunk_dict"] = chunk_dict
        Tracker["P_chunk0"] = len(chunk_one) / float(total_stack)
        Tracker["P_chunk1"] = len(chunk_two) / float(total_stack)
        ### create two volumes to estimate resolution
        if myid == main_node:
            for index in range(2):
                write_text_file(
                    chunk_list[index],
                    os.path.join(masterdir, "chunk%01d.txt" % index))
        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        vols = []
        for index in range(2):
            data, old_shifts = get_shrink_data_huang(
                Tracker,
                Tracker["constants"]["nxinit"],
                os.path.join(masterdir, "chunk%01d.txt" % index),
                Tracker["constants"]["partstack"],
                myid,
                main_node,
                nproc,
                preshift=True)
            vol = recons3d_4nn_ctf_MPI(myid=myid,
                                       prjlist=data,
                                       symmetry=Tracker["constants"]["sym"],
                                       finfo=None)
            if myid == main_node:
                vol.write_image(os.path.join(masterdir, "vol%d.hdf" % index))
            vols.append(vol)
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        if myid == main_node:
            low_pass, falloff, currentres = get_resolution_mrk01(
                vols, Tracker["constants"]["radius"],
                Tracker["constants"]["nxinit"], masterdir, Tracker["mask3D"])
            if low_pass > Tracker["constants"]["low_pass_filter"]:
                low_pass = Tracker["constants"]["low_pass_filter"]
        else:
            low_pass = 0.0
            falloff = 0.0
            currentres = 0.0
        bcast_number_to_all(currentres, source_node=main_node)
        bcast_number_to_all(low_pass, source_node=main_node)
        bcast_number_to_all(falloff, source_node=main_node)
        Tracker["currentres"] = currentres
        Tracker["falloff"] = falloff
        if Tracker["constants"]["low_pass_filter"] == -1.0:
            Tracker["low_pass_filter"] = min(
                .45, low_pass / Tracker["shrinkage"])  # no better than .45
        else:
            Tracker["low_pass_filter"] = min(
                .45,
                Tracker["constants"]["low_pass_filter"] / Tracker["shrinkage"])
        Tracker["lowpass"] = Tracker["low_pass_filter"]
        Tracker["falloff"] = .1
        Tracker["global_fsc"] = os.path.join(masterdir, "fsc.txt")
        ############################################################################################
        if myid == main_node:
            log_main.add("The command-line inputs are as following:")
            log_main.add(
                "**********************************************************")
        for a in sys.argv:
            if myid == main_node: log_main.add(a)
        if myid == main_node:
            log_main.add("number of cpus used in this run is %d" %
                         Tracker["constants"]["nproc"])
            log_main.add(
                "**********************************************************")
        from sp_filter import filt_tanl
        ### START 3-D sorting
        if myid == main_node:
            log_main.add("----------3-D sorting  program------- ")
            log_main.add(
                "current resolution %6.3f for images of original size in terms of absolute frequency"
                % Tracker["currentres"])
            log_main.add("equivalent to %f Angstrom resolution" %
                         (Tracker["constants"]["pixel_size"] /
                          Tracker["currentres"] / Tracker["shrinkage"]))
            log_main.add("the user provided enforced low_pass_filter is %f" %
                         Tracker["constants"]["low_pass_filter"])
            #log_main.add("equivalent to %f Angstrom resolution"%(Tracker["constants"]["pixel_size"]/Tracker["constants"]["low_pass_filter"]))
            for index in range(2):
                filt_tanl(
                    get_im(os.path.join(masterdir, "vol%01d.hdf" % index)),
                    Tracker["low_pass_filter"],
                    Tracker["falloff"]).write_image(
                        os.path.join(masterdir, "volf%01d.hdf" % index))
        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        from sp_utilities import get_input_from_string
        delta = get_input_from_string(Tracker["constants"]["delta"])
        delta = delta[0]
        from sp_utilities import even_angles
        n_angles = even_angles(delta, 0, 180)
        this_ali3d = Tracker["constants"]["ali3d"]
        sampled = get_stat_proj(Tracker, delta, this_ali3d)
        if myid == main_node:
            nc = 0
            for a in sampled:
                if len(sampled[a]) > 0:
                    nc += 1
            log_main.add("total sampled direction %10d  at angle step %6.3f" %
                         (len(n_angles), delta))
            log_main.add(
                "captured sampled directions %10d percentage covered by data  %6.3f"
                % (nc, float(nc) / len(n_angles) * 100))
        number_of_images_per_group = Tracker["constants"][
            "number_of_images_per_group"]
        if myid == main_node:
            log_main.add("user provided number_of_images_per_group %d" %
                         number_of_images_per_group)
        Tracker["number_of_images_per_group"] = number_of_images_per_group
        number_of_groups = get_number_of_groups(total_stack,
                                                number_of_images_per_group)
        Tracker["number_of_groups"] = number_of_groups
        generation = 0
        partition_dict = {}
        full_dict = {}
        workdir = os.path.join(masterdir, "generation%03d" % generation)
        Tracker["this_dir"] = workdir
        if myid == main_node:
            log_main.add("---- generation         %5d" % generation)
            log_main.add("number of images per group is set as %d" %
                         number_of_images_per_group)
            log_main.add("the initial number of groups is  %10d " %
                         number_of_groups)
            cmd = "{} {}".format("mkdir", workdir)
            os.system(cmd)
        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        list_to_be_processed = list(range(Tracker["constants"]["total_stack"]))
        Tracker["this_data_list"] = list_to_be_processed
        create_random_list(Tracker)
        #################################
        full_dict = {}
        for iptl in range(Tracker["constants"]["total_stack"]):
            full_dict[iptl] = iptl
        Tracker["full_ID_dict"] = full_dict
        #################################
        for indep_run in range(Tracker["constants"]["indep_runs"]):
            Tracker["this_particle_list"] = Tracker["this_indep_list"][
                indep_run]
            ref_vol = recons_mref(Tracker)
            if myid == main_node:
                log_main.add("independent run  %10d" % indep_run)
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            Tracker["this_data_list"] = list_to_be_processed
            Tracker["total_stack"] = len(Tracker["this_data_list"])
            Tracker["this_particle_text_file"] = os.path.join(
                workdir,
                "independent_list_%03d.txt" % indep_run)  # for get_shrink_data
            if myid == main_node:
                write_text_file(Tracker["this_data_list"],
                                Tracker["this_particle_text_file"])
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            outdir = os.path.join(workdir, "EQ_Kmeans%03d" % indep_run)
            ref_vol = apply_low_pass_filter(ref_vol, Tracker)
            mref_ali3d_EQ_Kmeans(ref_vol, outdir,
                                 Tracker["this_particle_text_file"], Tracker)
            partition_dict[indep_run] = Tracker["this_partition"]
        Tracker["partition_dict"] = partition_dict
        Tracker["total_stack"] = len(Tracker["this_data_list"])
        Tracker["this_total_stack"] = Tracker["total_stack"]
        ###############################
        do_two_way_comparison(Tracker)
        ###############################
        ref_vol_list = []
        from time import sleep
        number_of_ref_class = []
        for igrp in range(len(Tracker["two_way_stable_member"])):
            Tracker["this_data_list"] = Tracker["two_way_stable_member"][igrp]
            Tracker["this_data_list_file"] = os.path.join(
                workdir, "stable_class%d.txt" % igrp)
            if myid == main_node:
                write_text_file(Tracker["this_data_list"],
                                Tracker["this_data_list_file"])
            data, old_shifts = get_shrink_data_huang(
                Tracker,
                Tracker["nxinit"],
                Tracker["this_data_list_file"],
                Tracker["constants"]["partstack"],
                myid,
                main_node,
                nproc,
                preshift=True)
            volref = recons3d_4nn_ctf_MPI(myid=myid,
                                          prjlist=data,
                                          symmetry=Tracker["constants"]["sym"],
                                          finfo=None)
            ref_vol_list.append(volref)
            number_of_ref_class.append(len(Tracker["this_data_list"]))
            if myid == main_node:
                log_main.add("group  %d  members %d " %
                             (igrp, len(Tracker["this_data_list"])))
        Tracker["number_of_ref_class"] = number_of_ref_class
        nx_of_image = ref_vol_list[0].get_xsize()
        if Tracker["constants"]["PWadjustment"]:
            Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
        else:
            Tracker["PWadjustment"] = Tracker["constants"][
                "PWadjustment"]  # no PW adjustment
        if myid == main_node:
            for iref in range(len(ref_vol_list)):
                refdata = [None] * 4
                refdata[0] = ref_vol_list[iref]
                refdata[1] = Tracker
                refdata[2] = Tracker["constants"]["myid"]
                refdata[3] = Tracker["constants"]["nproc"]
                volref = user_func(refdata)
                volref.write_image(os.path.join(workdir, "volf_stable.hdf"),
                                   iref)
        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        Tracker["this_data_list"] = Tracker["this_accounted_list"]
        outdir = os.path.join(workdir, "Kmref")
        empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI(
            ref_vol_list, outdir, Tracker["this_accounted_text"], Tracker)
        Tracker["this_unaccounted_list"] = get_complementary_elements(
            list_to_be_processed, final_list)
        if myid == main_node:
            log_main.add("the number of particles not processed is %d" %
                         len(Tracker["this_unaccounted_list"]))
            write_text_file(Tracker["this_unaccounted_list"],
                            Tracker["this_unaccounted_text"])
        update_full_dict(Tracker["this_unaccounted_list"], Tracker)
        #######################################
        number_of_groups = len(res_groups)
        vol_list = []
        number_of_ref_class = []
        for igrp in range(number_of_groups):
            data, old_shifts = get_shrink_data_huang(
                Tracker,
                Tracker["constants"]["nnxo"],
                os.path.join(outdir, "Class%d.txt" % igrp),
                Tracker["constants"]["partstack"],
                myid,
                main_node,
                nproc,
                preshift=True)
            volref = recons3d_4nn_ctf_MPI(myid=myid,
                                          prjlist=data,
                                          symmetry=Tracker["constants"]["sym"],
                                          finfo=None)
            vol_list.append(volref)

            if (myid == main_node):
                npergroup = len(
                    read_text_file(os.path.join(outdir, "Class%d.txt" % igrp)))
            else:
                npergroup = 0
            npergroup = bcast_number_to_all(npergroup, main_node)
            number_of_ref_class.append(npergroup)

        Tracker["number_of_ref_class"] = number_of_ref_class

        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        nx_of_image = vol_list[0].get_xsize()
        if Tracker["constants"]["PWadjustment"]:
            Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
        else:
            Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]

        if myid == main_node:
            for ivol in range(len(vol_list)):
                refdata = [None] * 4
                refdata[0] = vol_list[ivol]
                refdata[1] = Tracker
                refdata[2] = Tracker["constants"]["myid"]
                refdata[3] = Tracker["constants"]["nproc"]
                volref = user_func(refdata)
                volref.write_image(
                    os.path.join(workdir, "volf_of_Classes.hdf"), ivol)
                log_main.add("number of unaccounted particles  %10d" %
                             len(Tracker["this_unaccounted_list"]))
                log_main.add("number of accounted particles  %10d" %
                             len(Tracker["this_accounted_list"]))

        Tracker["this_data_list"] = Tracker[
            "this_unaccounted_list"]  # reset parameters for the next round calculation
        Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
        Tracker["this_total_stack"] = Tracker["total_stack"]
        number_of_groups = get_number_of_groups(
            len(Tracker["this_unaccounted_list"]), number_of_images_per_group)
        Tracker["number_of_groups"] = number_of_groups
        while number_of_groups >= 2:
            generation += 1
            partition_dict = {}
            workdir = os.path.join(masterdir, "generation%03d" % generation)
            Tracker["this_dir"] = workdir
            if myid == main_node:
                log_main.add("*********************************************")
                log_main.add("-----    generation             %5d    " %
                             generation)
                log_main.add("number of images per group is set as %10d " %
                             number_of_images_per_group)
                log_main.add("the number of groups is  %10d " %
                             number_of_groups)
                log_main.add(" number of particles for clustering is %10d" %
                             Tracker["total_stack"])
                cmd = "{} {}".format("mkdir", workdir)
                os.system(cmd)
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            create_random_list(Tracker)
            for indep_run in range(Tracker["constants"]["indep_runs"]):
                Tracker["this_particle_list"] = Tracker["this_indep_list"][
                    indep_run]
                ref_vol = recons_mref(Tracker)
                if myid == main_node:
                    log_main.add("independent run  %10d" % indep_run)
                    outdir = os.path.join(workdir, "EQ_Kmeans%03d" % indep_run)
                Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
                #ref_vol=apply_low_pass_filter(ref_vol,Tracker)
                mref_ali3d_EQ_Kmeans(ref_vol, outdir,
                                     Tracker["this_unaccounted_text"], Tracker)
                partition_dict[indep_run] = Tracker["this_partition"]
                Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
                Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
                Tracker["partition_dict"] = partition_dict
                Tracker["this_total_stack"] = Tracker["total_stack"]
            total_list_of_this_run = Tracker["this_unaccounted_list"]
            ###############################
            do_two_way_comparison(Tracker)
            ###############################
            ref_vol_list = []
            number_of_ref_class = []
            for igrp in range(len(Tracker["two_way_stable_member"])):
                Tracker["this_data_list"] = Tracker["two_way_stable_member"][
                    igrp]
                Tracker["this_data_list_file"] = os.path.join(
                    workdir, "stable_class%d.txt" % igrp)
                if myid == main_node:
                    write_text_file(Tracker["this_data_list"],
                                    Tracker["this_data_list_file"])
                mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
                data, old_shifts = get_shrink_data_huang(
                    Tracker,
                    Tracker["constants"]["nxinit"],
                    Tracker["this_data_list_file"],
                    Tracker["constants"]["partstack"],
                    myid,
                    main_node,
                    nproc,
                    preshift=True)
                volref = recons3d_4nn_ctf_MPI(
                    myid=myid,
                    prjlist=data,
                    symmetry=Tracker["constants"]["sym"],
                    finfo=None)
                #volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1)
                if myid == main_node:
                    volref.write_image(os.path.join(workdir, "vol_stable.hdf"),
                                       iref)
                #volref = resample(volref,Tracker["shrinkage"])
                ref_vol_list.append(volref)
                number_of_ref_class.append(len(Tracker["this_data_list"]))
                mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            Tracker["number_of_ref_class"] = number_of_ref_class
            Tracker["this_data_list"] = Tracker["this_accounted_list"]
            outdir = os.path.join(workdir, "Kmref")
            empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI(
                ref_vol_list, outdir, Tracker["this_accounted_text"], Tracker)
            # calculate the 3-D structure of original image size for each group
            number_of_groups = len(res_groups)
            Tracker["this_unaccounted_list"] = get_complementary_elements(
                total_list_of_this_run, final_list)
            if myid == main_node:
                log_main.add("the number of particles not processed is %d" %
                             len(Tracker["this_unaccounted_list"]))
                write_text_file(Tracker["this_unaccounted_list"],
                                Tracker["this_unaccounted_text"])
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            update_full_dict(Tracker["this_unaccounted_list"], Tracker)
            vol_list = []
            for igrp in range(number_of_groups):
                data, old_shifts = get_shrink_data_huang(
                    Tracker,
                    Tracker["constants"]["nnxo"],
                    os.path.join(outdir, "Class%d.txt" % igrp),
                    Tracker["constants"]["partstack"],
                    myid,
                    main_node,
                    nproc,
                    preshift=True)
                volref = recons3d_4nn_ctf_MPI(
                    myid=myid,
                    prjlist=data,
                    symmetry=Tracker["constants"]["sym"],
                    finfo=None)
                vol_list.append(volref)

            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            nx_of_image = ref_vol_list[0].get_xsize()
            if Tracker["constants"]["PWadjustment"]:
                Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
            else:
                Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]

            if myid == main_node:
                for ivol in range(len(vol_list)):
                    refdata = [None] * 4
                    refdata[0] = vol_list[ivol]
                    refdata[1] = Tracker
                    refdata[2] = Tracker["constants"]["myid"]
                    refdata[3] = Tracker["constants"]["nproc"]
                    volref = user_func(refdata)
                    volref.write_image(
                        os.path.join(workdir, "volf_of_Classes.hdf"), ivol)
                log_main.add("number of unaccounted particles  %10d" %
                             len(Tracker["this_unaccounted_list"]))
                log_main.add("number of accounted particles  %10d" %
                             len(Tracker["this_accounted_list"]))
            del vol_list
            mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
            number_of_groups = get_number_of_groups(
                len(Tracker["this_unaccounted_list"]),
                number_of_images_per_group)
            Tracker["number_of_groups"] = number_of_groups
            Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
            Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
        if Tracker["constants"]["unaccounted"]:
            data, old_shifts = get_shrink_data_huang(
                Tracker,
                Tracker["constants"]["nnxo"],
                Tracker["this_unaccounted_text"],
                Tracker["constants"]["partstack"],
                myid,
                main_node,
                nproc,
                preshift=True)
            volref = recons3d_4nn_ctf_MPI(myid=myid,
                                          prjlist=data,
                                          symmetry=Tracker["constants"]["sym"],
                                          finfo=None)
            nx_of_image = volref.get_xsize()
            if Tracker["constants"]["PWadjustment"]:
                Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
            else:
                Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
            if (myid == main_node):
                refdata = [None] * 4
                refdata[0] = volref
                refdata[1] = Tracker
                refdata[2] = Tracker["constants"]["myid"]
                refdata[3] = Tracker["constants"]["nproc"]
                volref = user_func(refdata)
                #volref    = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1)
                volref.write_image(
                    os.path.join(workdir, "volf_unaccounted.hdf"))
        # Finish program
        if myid == main_node: log_main.add("sxsort3d finishes")
        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
        return
Esempio n. 23
0
def main():
    progname = os.path.basename(sys.argv[0])
    usage = progname + """ Input Output [options]
	
	Generate three micrographs, each micrograph contains one projection of a long filament.
	Input: Reference Volume, output directory 
	Output: Three micrographs stored in output directory		
				 
		sxhelical_demo.py tmp.hdf  mic --generate_micrograph --CTF --apix=1.84	
	
	Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
	
		sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
	
	Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
	
		sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
	
	Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
		
		sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
	
	Generate run through example script for helicon
	
		sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
	"""
    parser = OptionParser(usage, version=SPARXVERSION)

    # helicise the Atom coordinates

    # generate micrographs of helical filament
    parser.add_option(
        "--generate_micrograph",
        action="store_true",
        default=False,
        help=
        "Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory"
    )
    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="Use CTF correction")
    parser.add_option("--apix",
                      type="float",
                      default=-1,
                      help="pixel size in Angstroms")
    parser.add_option(
        "--rand_seed",
        type="int",
        default=14567,
        help=
        "the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs."
    )
    parser.add_option("--Cs",
                      type="float",
                      default=2.0,
                      help="Microscope Cs (spherical aberation)")
    parser.add_option("--voltage",
                      type="float",
                      default=200.0,
                      help="Microscope voltage in KV")
    parser.add_option("--ac",
                      type="float",
                      default=10.0,
                      help="Amplitude contrast (percentage, default=10)")
    parser.add_option("--nonoise",
                      action="store_true",
                      default=False,
                      help="Do not add noise to the micrograph.")

    # generate initial volume
    parser.add_option("--generate_noisycyl",
                      action="store_true",
                      default=False,
                      help="Generate initial volume of noisy cylinder.")
    parser.add_option(
        "--boxsize",
        type="string",
        default="100,100,200",
        help=
        "String containing x , y, z dimensions (separated by comma) in pixels")
    parser.add_option("--rad",
                      type="int",
                      default=35,
                      help="Radius of initial volume in pixels")

    # generate 2D mask
    parser.add_option("--generate_mask",
                      action="store_true",
                      default=False,
                      help="Generate 2D rectangular mask.")
    parser.add_option(
        "--masksize",
        type="string",
        default="200,200",
        help=
        "String containing x and y dimensions (separated by comma) in pixels")
    parser.add_option("--maskwidth",
                      type="int",
                      default=60,
                      help="Width of rectangular mask")

    # Apply 2D alignment parameters to input stack and output new images to output stack
    parser.add_option(
        "--applyparams",
        action="store_true",
        default=False,
        help=
        "Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack"
    )

    # Generate run script
    parser.add_option("--generate_script",
                      action="store_true",
                      default=False,
                      help="Generate script for helicon run through example")
    parser.add_option("--filename",
                      type="string",
                      default="runhelicon",
                      help="Name of run script to generate")
    parser.add_option("--seg_ny",
                      type="int",
                      default=180,
                      help="y-dimension of segment used for refinement")
    parser.add_option(
        "--ptcl_dist",
        type="int",
        default=15,
        help=
        "Distance in pixels between adjacent segments windowed from same filament"
    )
    parser.add_option(
        "--fract",
        type="float",
        default=0.35,
        help="Fraction of the volume used for applying helical symmetry.")

    (options, args) = parser.parse_args()
    if len(args) > 3:
        sxprint("usage: " + usage)
        sxprint("Please run '" + progname + " -h' for detailed options")
        ERROR(
            "Invalid number of parameters. Please see usage information above."
        )
        return

    else:
        if options.generate_script:
            generate_runscript(options.filename, options.seg_ny,
                               options.ptcl_dist, options.fract)

        if options.generate_micrograph:
            if options.apix <= 0:
                ERROR("Please enter pixel size.")
                return

            generate_helimic(args[0], args[1], options.apix, options.CTF,
                             options.Cs, options.voltage, options.ac,
                             options.nonoise, options.rand_seed)

        if options.generate_noisycyl:

            from sp_utilities import model_cylinder, model_gauss_noise
            outvol = args[0]
            boxdims = options.boxsize.split(',')

            if len(boxdims) < 1 or len(boxdims) > 3:
                ERROR(
                    "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize=\'100,100,200\'"
                )
                return

            nx = int(boxdims[0])

            if len(boxdims) == 1:
                ny = nx
                nz = nx
            else:
                ny = int(boxdims[1])
                if len(boxdims) == 3:
                    nz = int(boxdims[2])

            (model_cylinder(options.rad, nx, ny, nz) *
             model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol)

        if options.generate_mask:
            from sp_utilities import model_blank, pad
            outvol = args[0]
            maskdims = options.masksize.split(',')

            if len(maskdims) < 1 or len(maskdims) > 2:
                ERROR(
                    "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize=\'200,200\'"
                )
                return

            nx = int(maskdims[0])

            if len(maskdims) == 1:
                ny = nx
            else:
                ny = int(maskdims[1])

            mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1,
                       0.0)
            mask.write_image(outvol)

        if options.applyparams:
            from sp_utilities import get_im, get_params2D, set_params2D
            from sp_fundamentals import cyclic_shift
            stack = args[0]
            newstack = args[1]
            mask = get_im(args[2])
            nima = EMUtil.get_image_count(stack)
            for im in range(nima):
                prj = get_im(stack, im)
                alpha, sx, sy, mirror, scale = get_params2D(prj)
                prj = cyclic_shift(prj, int(sx))
                set_params2D(prj, [0.0, 0., 0.0, 0, 1])
                stat = Util.infomask(prj, mask, False)
                prj = (prj - stat[0]) / stat[1]
                ctf_params = prj.get_attr("ctf")
                prj.set_attr('ctf_applied', 0)
                prj.write_image(newstack, im)
Esempio n. 24
0
def cml_open_proj(stack, ir, ou, lf, hf, dpsi=1):
    from sp_projection import cml_sinogram
    from sp_utilities import model_circle, get_params_proj, model_blank, get_im
    from sp_fundamentals import fftip
    from sp_filter import filt_tanh

    # number of projections
    if type(stack) == type(""): nprj = EMUtil.get_image_count(stack)
    else: nprj = len(stack)
    Prj = []  # list of projections
    Ori = [
        -1
    ] * 4 * nprj  # orientation intial (phi, theta, psi, index) for each projection

    for i in range(nprj):
        image = get_im(stack, i)

        # read initial angles if given
        try:
            Ori[4 * i], Ori[4 * i +
                            1], Ori[4 * i +
                                    2], s2x, s2y = get_params_proj(image)
        except:
            pass

        if (i == 0):
            nx = image.get_xsize()
            if (ou < 1): ou = nx // 2 - 1
            diameter = int(2 * ou)
            mask2D = model_circle(ou, nx, nx)
            if ir > 0: mask2D -= model_circle(ir, nx, nx)

        # normalize under the mask
        [mean_a, sigma, imin, imax] = Util.infomask(image, mask2D, True)
        image -= mean_a
        Util.mul_scalar(image, 1.0 / sigma)
        Util.mul_img(image, mask2D)

        # sinogram
        sino = cml_sinogram(image, diameter, dpsi)

        # prepare the cut positions in order to filter (lf: low freq; hf: high freq)
        ihf = min(int(2 * hf * diameter), diameter + (diameter + 1) % 2)
        ihf = ihf + (ihf + 1) % 2  # index ihf must be odd to take the img part
        ilf = max(int(2 * lf * diameter), 0)
        ilf = ilf + ilf % 2  # index ilf must be even to fall in the real part
        bdf = ihf - ilf + 1

        # process lines
        nxe = sino.get_xsize()
        nye = sino.get_ysize()
        prj = model_blank(bdf, 2 * nye)
        pp = model_blank(nxe, 2 * nye)
        for li in range(nye):
            # get the line li
            line = Util.window(sino, nxe, 1, 1, 0, li - nye // 2, 0)
            # u2 (not improve the results)
            #line = filt_tanh(line, ou / float(nx), ou / float(nx))
            # normalize this line
            [mean_l, sigma_l, imin, imax] = Util.infomask(line, None, True)
            line = (line - mean_l) / sigma_l
            # fft
            fftip(line)
            # filter (cut part of coef) and create mirror line
            Util.cml_prepare_line(prj, line, ilf, ihf, li, nye)

        # store the projection
        Prj.append(prj)

    return Prj, Ori
Esempio n. 25
0
def main():
    """
	Main function.

	Arguments:
	None

	Returns:
	None
	"""

    command_args = parse_command_line()

    # Import volume
    sp_global_def.sxprint("Import volume.")
    input_vol = sp_utilities.get_im(command_args.input_volume)

    # Sanity checks
    sanity_checks(command_args, input_vol)

    try:
        os.makedirs(command_args.output_dir)
    except OSError:
        sp_global_def.sxprint(
            "Output directory already exists. No need to create it.")
    else:
        sp_global_def.sxprint("Created output directory.")
    sp_global_def.write_command(command_args.output_dir)
    output_prefix = os.path.join(command_args.output_dir, command_args.prefix)

    # Filter volume if specified
    if command_args.low_pass_filter_resolution is not None:
        sp_global_def.sxprint("Filter volume to {0}A.".format(
            command_args.low_pass_filter_resolution))
        input_vol = sp_filter.filt_tanl(
            input_vol,
            old_div(command_args.pixel_size,
                    command_args.low_pass_filter_resolution),
            command_args.low_pass_filter_falloff,
        )
        input_vol.write_image(output_prefix + "_filtered_volume.hdf")
    else:
        sp_global_def.sxprint("Skip filter volume.")

    # Create a mask based on the filtered volume
    sp_global_def.sxprint("Create mask")
    density_threshold = -9999.0
    nsigma = 1.0
    if command_args.mol_mass:
        density_threshold = input_vol.find_3d_threshold(
            command_args.mol_mass, command_args.pixel_size)
        sp_global_def.sxprint(
            "Mask molecular mass translated into binary threshold: ",
            density_threshold)
    elif command_args.threshold:
        density_threshold = command_args.threshold
    elif command_args.nsigma:
        nsigma = command_args.nsigma
    else:
        assert False

    if command_args.edge_type == "cosine":
        mode = "C"
    elif command_args.edge_type == "gaussian":
        mode = "G"
    else:
        assert False

    mask_first = sp_morphology.adaptive_mask_scipy(
        input_vol,
        nsigma=nsigma,
        threshold=density_threshold,
        ndilation=command_args.ndilation,
        nerosion=command_args.nerosion,
        edge_width=command_args.edge_width,
        allow_disconnected=command_args.allow_disconnected,
        mode=mode,
        do_approx=command_args.do_old,
        do_fill=command_args.fill_mask,
        do_print=True,
    )

    # Create a second mask based on the filtered volume
    s_mask = None
    s_density_threshold = 1
    s_nsigma = 1.0
    if command_args.second_mask is not None:
        sp_global_def.sxprint("Prepare second mask")
        s_mask = sp_utilities.get_im(command_args.second_mask)
        s_density_threshold = -9999.0
        s_nsigma = 1.0
        if command_args.s_mol_mass:
            s_density_threshold = input_vol.find_3d_threshold(
                command_args.s_mol_mass, command_args.s_pixel_size)
            sp_global_def.sxprint(
                "Second mask molecular mass translated into binary threshold: ",
                s_density_threshold,
            )
        elif command_args.s_threshold:
            s_density_threshold = command_args.s_threshold
        elif command_args.s_nsigma:
            s_nsigma = command_args.s_nsigma
        else:
            assert False
    elif command_args.second_mask_shape is not None:
        sp_global_def.sxprint("Prepare second mask")
        nx = mask_first.get_xsize()
        ny = mask_first.get_ysize()
        nz = mask_first.get_zsize()
        if command_args.second_mask_shape == "cube":
            s_nx = command_args.s_nx
            s_ny = command_args.s_ny
            s_nz = command_args.s_nz
            s_mask = sp_utilities.model_blank(s_nx, s_ny, s_nz, 1)
        elif command_args.second_mask_shape == "cylinder":
            s_radius = command_args.s_radius
            s_nx = command_args.s_nx
            s_ny = command_args.s_ny
            s_nz = command_args.s_nz
            try:
                s_mask = sp_utilities.model_cylinder(s_radius, s_nx, s_ny,
                                                     s_nz)
            except RuntimeError as e:
                sp_global_def.sxprint(
                    "An error occured! Please check the error log")
                raise
        elif command_args.second_mask_shape == "sphere":
            s_radius = command_args.s_radius
            s_nx = command_args.s_nx
            s_ny = command_args.s_ny
            s_nz = command_args.s_nz
            try:
                s_mask = sp_utilities.model_circle(s_radius, s_nx, s_ny, s_nz)
            except RuntimeError as e:
                sp_global_def.sxprint(
                    "An error occured! Please check the error log")
                raise
        else:
            assert False
        s_mask = sp_utilities.pad(s_mask, nx, ny, nz, 0)

    if s_mask is not None:
        sp_global_def.sxprint("Create second mask")

        if command_args.s_edge_type == "cosine":
            mode = "C"
        elif command_args.s_edge_type == "gaussian":
            mode = "G"
        else:
            assert False

        s_mask = sp_morphology.adaptive_mask_scipy(
            s_mask,
            nsigma=s_nsigma,
            threshold=s_density_threshold,
            ndilation=command_args.s_ndilation,
            nerosion=command_args.s_nerosion,
            edge_width=command_args.s_edge_width,
            allow_disconnected=command_args.s_allow_disconnected,
            mode=mode,
            do_approx=command_args.s_do_old,
            do_fill=command_args.s_fill_mask,
            do_print=True,
        )
        if command_args.s_invert:
            s_mask = 1 - s_mask
        sp_global_def.sxprint("Write outputs.")
        mask_first.write_image(output_prefix + "_mask_first.hdf")
        s_mask.write_image(output_prefix + "_mask_second.hdf")
        masked_combined = mask_first * s_mask
        masked_combined.write_image(output_prefix + "_mask.hdf")
    else:
        sp_global_def.sxprint("Write outputs.")
        mask_first.write_image(output_prefix + "_mask.hdf")
Esempio n. 26
0
def main():
    arglist = []
    for arg in sys.argv:
        arglist.append(arg)
    progname = os.path.basename(arglist[0])
    usage = progname + """ firstvolume  secondvolume  maskfile  directory  --prefix  --wn  --step  --cutoff  --radius  --fsc  --res_overall  --out_ang_res  --apix  --MPI

	Compute local resolution in real space within area outlined by the maskfile and within regions wn x wn x wn
	"""
    parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)

    parser.add_option("--prefix",
                      type="str",
                      default='localres',
                      help="Prefix for the output files. (default localres)")
    parser.add_option(
        "--wn",
        type="int",
        default=7,
        help=
        "Size of window within which local real-space FSC is computed. (default 7)"
    )
    parser.add_option(
        "--step",
        type="float",
        default=1.0,
        help="Shell step in Fourier size in pixels. (default 1.0)")
    parser.add_option("--cutoff",
                      type="float",
                      default=0.143,
                      help="Resolution cut-off for FSC. (default 0.143)")
    parser.add_option(
        "--radius",
        type="int",
        default=-1,
        help=
        "If there is no maskfile, sphere with r=radius will be used. By default, the radius is nx/2-wn (default -1)"
    )
    parser.add_option(
        "--fsc",
        type="string",
        default=None,
        help=
        "Save overall FSC curve (might be truncated). By default, the program does not save the FSC curve. (default none)"
    )
    parser.add_option(
        "--res_overall",
        type="float",
        default=-1.0,
        help=
        "Overall resolution at the cutoff level estimated by the user [abs units]. (default None)"
    )
    parser.add_option(
        "--out_ang_res",
        action="store_true",
        default=False,
        help=
        "Additionally creates a local resolution file in Angstroms. (default False)"
    )
    parser.add_option(
        "--apix",
        type="float",
        default=1.0,
        help=
        "Pixel size in Angstrom. Effective only with --out_ang_res options. (default 1.0)"
    )
    parser.add_option("--MPI",
                      action="store_true",
                      default=False,
                      help="Use MPI version.")

    (options, args) = parser.parse_args(arglist[1:])

    if len(args) < 3 or len(args) > 4:
        sxprint("Usage: " + usage)
        ERROR(
            "Invalid number of parameters used. Please see usage information above."
        )
        return

    if sp_global_def.CACHE_DISABLE:
        sp_utilities.disable_bdb_cache()

    res_overall = options.res_overall

    if options.MPI:

        number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
        myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
        main_node = 0
        sp_global_def.MPI = True
        cutoff = options.cutoff

        nk = int(options.wn)

        if (myid == main_node):
            #print sys.argv
            vi = sp_utilities.get_im(sys.argv[1])
            ui = sp_utilities.get_im(sys.argv[2])

            nx = vi.get_xsize()
            ny = vi.get_ysize()
            nz = vi.get_zsize()
            dis = [nx, ny, nz]
        else:
            dis = [0, 0, 0, 0]

        sp_global_def.BATCH = True

        dis = sp_utilities.bcast_list_to_all(dis, myid, source_node=main_node)

        if (myid != main_node):
            nx = int(dis[0])
            ny = int(dis[1])
            nz = int(dis[2])

            vi = sp_utilities.model_blank(nx, ny, nz)
            ui = sp_utilities.model_blank(nx, ny, nz)

        if len(args) == 3:
            m = sp_utilities.model_circle((min(nx, ny, nz) - nk) // 2, nx, ny,
                                          nz)
            outdir = args[2]

        elif len(args) == 4:
            if (myid == main_node):
                m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
            else:
                m = sp_utilities.model_blank(nx, ny, nz)
            outdir = args[3]
        if os.path.exists(outdir) and myid == 0:
            sp_global_def.ERROR('Output directory already exists!')
        elif myid == 0:
            os.makedirs(outdir)
        sp_global_def.write_command(outdir)
        sp_utilities.bcast_EMData_to_all(m, myid, main_node)
        """Multiline Comment0"""
        freqvol, resolut = sp_statistics.locres(vi, ui, m, nk, cutoff,
                                                options.step, myid, main_node,
                                                number_of_proc)

        if (myid == 0):
            # Remove outliers based on the Interquartile range
            output_volume(freqvol, resolut, options.apix, outdir,
                          options.prefix, options.fsc, options.out_ang_res, nx,
                          ny, nz, res_overall)

    else:
        cutoff = options.cutoff
        vi = sp_utilities.get_im(args[0])
        ui = sp_utilities.get_im(args[1])

        nn = vi.get_xsize()
        nx = nn
        ny = nn
        nz = nn
        nk = int(options.wn)

        if len(args) == 3:
            m = sp_utilities.model_circle((nn - nk) // 2, nn, nn, nn)
            outdir = args[2]

        elif len(args) == 4:
            m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
            outdir = args[3]
        if os.path.exists(outdir):
            sp_global_def.ERROR('Output directory already exists!')
        else:
            os.makedirs(outdir)
        sp_global_def.write_command(outdir)

        mc = sp_utilities.model_blank(nn, nn, nn, 1.0) - m

        vf = sp_fundamentals.fft(vi)
        uf = sp_fundamentals.fft(ui)
        """Multiline Comment1"""
        lp = int(nn / 2 / options.step + 0.5)
        step = 0.5 / lp

        freqvol = sp_utilities.model_blank(nn, nn, nn)
        resolut = []
        for i in range(1, lp):
            fl = step * i
            fh = fl + step
            #print(lp,i,step,fl,fh)
            v = sp_fundamentals.fft(sp_filter.filt_tophatb(vf, fl, fh))
            u = sp_fundamentals.fft(sp_filter.filt_tophatb(uf, fl, fh))
            tmp1 = EMAN2_cppwrap.Util.muln_img(v, v)
            tmp2 = EMAN2_cppwrap.Util.muln_img(u, u)

            do = EMAN2_cppwrap.Util.infomask(
                sp_morphology.square_root(
                    sp_morphology.threshold(
                        EMAN2_cppwrap.Util.muln_img(tmp1, tmp2))), m, True)[0]

            tmp3 = EMAN2_cppwrap.Util.muln_img(u, v)
            dp = EMAN2_cppwrap.Util.infomask(tmp3, m, True)[0]
            resolut.append([i, (fl + fh) / 2.0, dp / do])

            tmp1 = EMAN2_cppwrap.Util.box_convolution(tmp1, nk)
            tmp2 = EMAN2_cppwrap.Util.box_convolution(tmp2, nk)
            tmp3 = EMAN2_cppwrap.Util.box_convolution(tmp3, nk)

            EMAN2_cppwrap.Util.mul_img(tmp1, tmp2)

            tmp1 = sp_morphology.square_root(sp_morphology.threshold(tmp1))

            EMAN2_cppwrap.Util.mul_img(tmp1, m)
            EMAN2_cppwrap.Util.add_img(tmp1, mc)

            EMAN2_cppwrap.Util.mul_img(tmp3, m)
            EMAN2_cppwrap.Util.add_img(tmp3, mc)

            EMAN2_cppwrap.Util.div_img(tmp3, tmp1)

            EMAN2_cppwrap.Util.mul_img(tmp3, m)
            freq = (fl + fh) / 2.0
            bailout = True
            for x in range(nn):
                for y in range(nn):
                    for z in range(nn):
                        if (m.get_value_at(x, y, z) > 0.5):
                            if (freqvol.get_value_at(x, y, z) == 0.0):
                                if (tmp3.get_value_at(x, y, z) < cutoff):
                                    freqvol.set_value_at(x, y, z, freq)
                                    bailout = False
                                else:
                                    bailout = False
            if (bailout): break
        #print(len(resolut))
        # remove outliers
        output_volume(freqvol, resolut, options.apix, outdir, options.prefix,
                      options.fsc, options.out_ang_res, nx, ny, nz,
                      res_overall)
Esempio n. 27
0
    def test_(self):
        out_dir_old = "oldmeridien"
        out_dir_new = "newmeridien"

        mkdir(out_dir_old)
        mkdir(out_dir_new)
        testcommand_old = (
            MPI_PATH + " -np " + str(NUM_PROC) + " " +
            path.join(ABSOLUTE_OLDBIN_PATH, "sp_meridien.py") + " " +
            " 'bdb:" +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "06_SUBSTACK_ANO#isac_substack'") + " " +
            self.out_dir_old + " " +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "09_ADJUSTMENT", "vol3d_ref_moon_eliminated_mask.hdf") +
            " --initialshifts" + " --radius=145" + " --mask3D=" +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "10_MASK", "sp_mask_mask.hdf") + " --symmetry=c5")

        testcommand_new = (
            MPI_PATH + " -np " + str(NUM_PROC) + " " +
            path.join(ABSOLUTE_BIN_PATH, "sp_meridien.py") + " 'bdb:" +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "06_SUBSTACK_ANO#isac_substack'") + " " +
            self.out_dir_new + " " +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "09_ADJUSTMENT", "vol3d_ref_moon_eliminated_mask.hdf") +
            " --initialshifts" + " --radius=145" + " --mask3D=" +
            path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
                      "10_MASK", "sp_mask_mask.hdf") + " --symmetry=c5")

        # as they are ordered alphanumerically the first is always vol0

        subprocess.run(args=[testcommand_old],
                       shell=True,
                       stderr=subprocess.STDOUT)
        subprocess.run(args=[testcommand_new],
                       shell=True,
                       stderr=subprocess.STDOUT)

        hdfFilesOld = [
            f for f in listdir(path.join(ABSOLUTE_PATH, self.out_dir_old))
            if path.isfile(path.join(ABSOLUTE_PATH, self.out_dir_old, f))
            and f.split(".")[1] == "hdf"
        ]
        hdfFilesNew = [
            f for f in listdir(path.join(ABSOLUTE_PATH, self.out_dir_old))
            if path.isfile(path.join(ABSOLUTE_PATH, self.out_dir_new, f))
            and f.split(".")[1] == "hdf"
        ]
        old_v0 = get_im(
            path.join(ABSOLUTE_PATH, self.out_dir_old, hdfFilesOld[0]))
        old_v1 = get_im(
            path.join(ABSOLUTE_PATH, self.out_dir_old, hdfFilesOld[1]))
        new_v0 = get_im(
            path.join(ABSOLUTE_PATH, self.out_dir_new, hdfFilesNew[0]))
        new_v1 = get_im(
            path.join(ABSOLUTE_PATH, self.out_dir_new, hdfFilesNew[1]))
        self.assertTrue(
            allclose(old_v0.get_3dview(), new_v0.get_3dview(), atol=TOLERANCE))
        self.assertTrue(
            allclose(old_v1.get_3dview(), new_v1.get_3dview(), atol=TOLERANCE))
        self.assertTrue(
            allclose(new_v0.get_3dview().flatten().tolist()[0:100], [
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539,
                0.000977565417997539, 0.000977565417997539
            ],
                     atol=0.005))
        self.assertTrue(
            allclose(new_v1.get_3dview().flatten().tolist()[0:100], [
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855,
                0.0011777321342378855, 0.0011777321342378855
            ],
                     atol=0.005))
Esempio n. 28
0
def main():
    global Tracker, Blockdata
    progname = os.path.basename(sys.argv[0])
    usage = progname + " --output_dir=output_dir  --isac_dir=output_dir_of_isac "
    parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
    parser.add_option(
        "--pw_adjustment",
        type="string",
        default="analytical_model",
        help=
        "adjust power spectrum of 2-D averages to an analytic model. Other opions: no_adjustment; bfactor; a text file of 1D rotationally averaged PW",
    )
    #### Four options for --pw_adjustment:
    # 1> analytical_model(default);
    # 2> no_adjustment;
    # 3> bfactor;
    # 4> adjust_to_given_pw2(user has to provide a text file that contains 1D rotationally averaged PW)

    # options in common
    parser.add_option(
        "--isac_dir",
        type="string",
        default="",
        help="ISAC run output directory, input directory for this command",
    )
    parser.add_option(
        "--output_dir",
        type="string",
        default="",
        help="output directory where computed averages are saved",
    )
    parser.add_option(
        "--pixel_size",
        type="float",
        default=-1.0,
        help=
        "pixel_size of raw images. one can put 1.0 in case of negative stain data",
    )
    parser.add_option(
        "--fl",
        type="float",
        default=-1.0,
        help=
        "low pass filter, = -1.0, not applied; =0.0, using FH1 (initial resolution), = 1.0 using FH2 (resolution after local alignment), or user provided value in absolute freqency [0.0:0.5]",
    )
    parser.add_option("--stack",
                      type="string",
                      default="",
                      help="data stack used in ISAC")
    parser.add_option("--radius", type="int", default=-1, help="radius")
    parser.add_option("--xr",
                      type="float",
                      default=-1.0,
                      help="local alignment search range")
    # parser.add_option("--ts",                    type   ="float",          default =1.0,    help= "local alignment search step")
    parser.add_option(
        "--fh",
        type="float",
        default=-1.0,
        help="local alignment high frequencies limit",
    )
    # parser.add_option("--maxit",                 type   ="int",            default =5,      help= "local alignment iterations")
    parser.add_option("--navg",
                      type="int",
                      default=1000000,
                      help="number of aveages")
    parser.add_option(
        "--local_alignment",
        action="store_true",
        default=False,
        help="do local alignment",
    )
    parser.add_option(
        "--noctf",
        action="store_true",
        default=False,
        help=
        "no ctf correction, useful for negative stained data. always ctf for cryo data",
    )
    parser.add_option(
        "--B_start",
        type="float",
        default=45.0,
        help=
        "start frequency (Angstrom) of power spectrum for B_factor estimation",
    )
    parser.add_option(
        "--Bfactor",
        type="float",
        default=-1.0,
        help=
        "User defined bactors (e.g. 25.0[A^2]). By default, the program automatically estimates B-factor. ",
    )

    (options, args) = parser.parse_args(sys.argv[1:])

    adjust_to_analytic_model = (True if options.pw_adjustment
                                == "analytical_model" else False)
    no_adjustment = True if options.pw_adjustment == "no_adjustment" else False
    B_enhance = True if options.pw_adjustment == "bfactor" else False
    adjust_to_given_pw2 = (
        True if not (adjust_to_analytic_model or no_adjustment or B_enhance)
        else False)

    # mpi
    nproc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
    myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)

    Blockdata = {}
    Blockdata["nproc"] = nproc
    Blockdata["myid"] = myid
    Blockdata["main_node"] = 0
    Blockdata["shared_comm"] = mpi.mpi_comm_split_type(
        mpi.MPI_COMM_WORLD, mpi.MPI_COMM_TYPE_SHARED, 0, mpi.MPI_INFO_NULL)
    Blockdata["myid_on_node"] = mpi.mpi_comm_rank(Blockdata["shared_comm"])
    Blockdata["no_of_processes_per_group"] = mpi.mpi_comm_size(
        Blockdata["shared_comm"])
    masters_from_groups_vs_everything_else_comm = mpi.mpi_comm_split(
        mpi.MPI_COMM_WORLD,
        Blockdata["main_node"] == Blockdata["myid_on_node"],
        Blockdata["myid_on_node"],
    )
    Blockdata["color"], Blockdata[
        "no_of_groups"], balanced_processor_load_on_nodes = sp_utilities.get_colors_and_subsets(
            Blockdata["main_node"],
            mpi.MPI_COMM_WORLD,
            Blockdata["myid"],
            Blockdata["shared_comm"],
            Blockdata["myid_on_node"],
            masters_from_groups_vs_everything_else_comm,
        )
    #  We need two nodes for processing of volumes
    Blockdata["node_volume"] = [
        Blockdata["no_of_groups"] - 3,
        Blockdata["no_of_groups"] - 2,
        Blockdata["no_of_groups"] - 1,
    ]  # For 3D stuff take three last nodes
    #  We need two CPUs for processing of volumes, they are taken to be main CPUs on each volume
    #  We have to send the two myids to all nodes so we can identify main nodes on two selected groups.
    Blockdata["nodes"] = [
        Blockdata["node_volume"][0] * Blockdata["no_of_processes_per_group"],
        Blockdata["node_volume"][1] * Blockdata["no_of_processes_per_group"],
        Blockdata["node_volume"][2] * Blockdata["no_of_processes_per_group"],
    ]
    # End of Blockdata: sorting requires at least three nodes, and the used number of nodes be integer times of three
    sp_global_def.BATCH = True
    sp_global_def.MPI = True

    if adjust_to_given_pw2:
        checking_flag = 0
        if Blockdata["myid"] == Blockdata["main_node"]:
            if not os.path.exists(options.pw_adjustment):
                checking_flag = 1
        checking_flag = sp_utilities.bcast_number_to_all(
            checking_flag, Blockdata["main_node"], mpi.MPI_COMM_WORLD)

        if checking_flag == 1:
            sp_global_def.ERROR("User provided power spectrum does not exist",
                                myid=Blockdata["myid"])

    Tracker = {}
    Constants = {}
    Constants["isac_dir"] = options.isac_dir
    Constants["masterdir"] = options.output_dir
    Constants["pixel_size"] = options.pixel_size
    Constants["orgstack"] = options.stack
    Constants["radius"] = options.radius
    Constants["xrange"] = options.xr
    Constants["FH"] = options.fh
    Constants["low_pass_filter"] = options.fl
    # Constants["maxit"]                        = options.maxit
    Constants["navg"] = options.navg
    Constants["B_start"] = options.B_start
    Constants["Bfactor"] = options.Bfactor

    if adjust_to_given_pw2:
        Constants["modelpw"] = options.pw_adjustment
    Tracker["constants"] = Constants
    # -------------------------------------------------------------
    #
    # Create and initialize Tracker dictionary with input options  # State Variables

    # <<<---------------------->>>imported functions<<<---------------------------------------------

    # x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1)
    # y_range =  x_range

    ####-----------------------------------------------------------
    # Create Master directory and associated subdirectories
    line = time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()) + " =>"
    if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]:
        masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen")
    else:
        masterdir = Tracker["constants"]["masterdir"]

    if Blockdata["myid"] == Blockdata["main_node"]:
        msg = "Postprocessing ISAC 2D averages starts"
        sp_global_def.sxprint(line, "Postprocessing ISAC 2D averages starts")
        if not masterdir:
            timestring = time.strftime("_%d_%b_%Y_%H_%M_%S", time.localtime())
            masterdir = "sharpen_" + Tracker["constants"]["isac_dir"]
            os.makedirs(masterdir)
        else:
            if os.path.exists(masterdir):
                sp_global_def.sxprint("%s already exists" % masterdir)
            else:
                os.makedirs(masterdir)
        sp_global_def.write_command(masterdir)
        subdir_path = os.path.join(masterdir, "ali2d_local_params_avg")
        if not os.path.exists(subdir_path):
            os.mkdir(subdir_path)
        subdir_path = os.path.join(masterdir, "params_avg")
        if not os.path.exists(subdir_path):
            os.mkdir(subdir_path)
        li = len(masterdir)
    else:
        li = 0
    li = mpi.mpi_bcast(li, 1, mpi.MPI_INT, Blockdata["main_node"],
                       mpi.MPI_COMM_WORLD)[0]
    masterdir = mpi.mpi_bcast(masterdir, li, mpi.MPI_CHAR,
                              Blockdata["main_node"], mpi.MPI_COMM_WORLD)
    masterdir = b"".join(masterdir).decode('latin1')
    Tracker["constants"]["masterdir"] = masterdir
    log_main = sp_logger.Logger(sp_logger.BaseLogger_Files())
    log_main.prefix = Tracker["constants"]["masterdir"] + "/"

    while not os.path.exists(Tracker["constants"]["masterdir"]):
        sp_global_def.sxprint(
            "Node ",
            Blockdata["myid"],
            "  waiting...",
            Tracker["constants"]["masterdir"],
        )
        time.sleep(1)
    mpi.mpi_barrier(mpi.MPI_COMM_WORLD)

    if Blockdata["myid"] == Blockdata["main_node"]:
        init_dict = {}
        sp_global_def.sxprint(Tracker["constants"]["isac_dir"])
        Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"],
                                            "2dalignment")
        core = sp_utilities.read_text_row(
            os.path.join(Tracker["directory"], "initial2Dparams.txt"))
        for im in range(len(core)):
            init_dict[im] = core[im]
        del core
    else:
        init_dict = 0
    init_dict = sp_utilities.wrap_mpi_bcast(init_dict,
                                            Blockdata["main_node"],
                                            communicator=mpi.MPI_COMM_WORLD)
    ###
    do_ctf = True
    if options.noctf:
        do_ctf = False
    if Blockdata["myid"] == Blockdata["main_node"]:
        if do_ctf:
            sp_global_def.sxprint("CTF correction is on")
        else:
            sp_global_def.sxprint("CTF correction is off")
        if options.local_alignment:
            sp_global_def.sxprint("local refinement is on")
        else:
            sp_global_def.sxprint("local refinement is off")
        if B_enhance:
            sp_global_def.sxprint("Bfactor is to be applied on averages")
        elif adjust_to_given_pw2:
            sp_global_def.sxprint(
                "PW of averages is adjusted to a given 1D PW curve")
        elif adjust_to_analytic_model:
            sp_global_def.sxprint(
                "PW of averages is adjusted to analytical model")
        else:
            sp_global_def.sxprint("PW of averages is not adjusted")
        # Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack")
        image = sp_utilities.get_im(Tracker["constants"]["orgstack"], 0)
        Tracker["constants"]["nnxo"] = image.get_xsize()
        if Tracker["constants"]["pixel_size"] == -1.0:
            sp_global_def.sxprint(
                "Pixel size value is not provided by user. extracting it from ctf header entry of the original stack."
            )
            try:
                ctf_params = image.get_attr("ctf")
                Tracker["constants"]["pixel_size"] = ctf_params.apix
            except:
                sp_global_def.ERROR(
                    "Pixel size could not be extracted from the original stack.",
                    myid=Blockdata["myid"],
                )
        ## Now fill in low-pass filter

        isac_shrink_path = os.path.join(Tracker["constants"]["isac_dir"],
                                        "README_shrink_ratio.txt")

        if not os.path.exists(isac_shrink_path):
            sp_global_def.ERROR(
                "%s does not exist in the specified ISAC run output directory"
                % (isac_shrink_path),
                myid=Blockdata["myid"],
            )

        isac_shrink_file = open(isac_shrink_path, "r")
        isac_shrink_lines = isac_shrink_file.readlines()
        isac_shrink_ratio = float(
            isac_shrink_lines[5]
        )  # 6th line: shrink ratio (= [target particle radius]/[particle radius]) used in the ISAC run
        isac_radius = float(
            isac_shrink_lines[6]
        )  # 7th line: particle radius at original pixel size used in the ISAC run
        isac_shrink_file.close()
        print("Extracted parameter values")
        print("ISAC shrink ratio    : {0}".format(isac_shrink_ratio))
        print("ISAC particle radius : {0}".format(isac_radius))
        Tracker["ini_shrink"] = isac_shrink_ratio
    else:
        Tracker["ini_shrink"] = 0.0
    Tracker = sp_utilities.wrap_mpi_bcast(Tracker,
                                          Blockdata["main_node"],
                                          communicator=mpi.MPI_COMM_WORLD)

    # print(Tracker["constants"]["pixel_size"], "pixel_size")
    x_range = max(
        Tracker["constants"]["xrange"],
        int(old_div(1.0, Tracker["ini_shrink"]) + 0.99999),
    )
    a_range = y_range = x_range

    if Blockdata["myid"] == Blockdata["main_node"]:
        parameters = sp_utilities.read_text_row(
            os.path.join(Tracker["constants"]["isac_dir"],
                         "all_parameters.txt"))
    else:
        parameters = 0
    parameters = sp_utilities.wrap_mpi_bcast(parameters,
                                             Blockdata["main_node"],
                                             communicator=mpi.MPI_COMM_WORLD)
    params_dict = {}
    list_dict = {}
    # parepare params_dict

    # navg = min(Tracker["constants"]["navg"]*Blockdata["nproc"], EMUtil.get_image_count(os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf")))
    navg = min(
        Tracker["constants"]["navg"],
        EMAN2_cppwrap.EMUtil.get_image_count(
            os.path.join(Tracker["constants"]["isac_dir"],
                         "class_averages.hdf")),
    )
    global_dict = {}
    ptl_list = []
    memlist = []
    if Blockdata["myid"] == Blockdata["main_node"]:
        sp_global_def.sxprint("Number of averages computed in this run is %d" %
                              navg)
        for iavg in range(navg):
            params_of_this_average = []
            image = sp_utilities.get_im(
                os.path.join(Tracker["constants"]["isac_dir"],
                             "class_averages.hdf"),
                iavg,
            )
            members = sorted(image.get_attr("members"))
            memlist.append(members)
            for im in range(len(members)):
                abs_id = members[im]
                global_dict[abs_id] = [iavg, im]
                P = sp_utilities.combine_params2(
                    init_dict[abs_id][0],
                    init_dict[abs_id][1],
                    init_dict[abs_id][2],
                    init_dict[abs_id][3],
                    parameters[abs_id][0],
                    old_div(parameters[abs_id][1], Tracker["ini_shrink"]),
                    old_div(parameters[abs_id][2], Tracker["ini_shrink"]),
                    parameters[abs_id][3],
                )
                if parameters[abs_id][3] == -1:
                    sp_global_def.sxprint(
                        "WARNING: Image #{0} is an unaccounted particle with invalid 2D alignment parameters and should not be the member of any classes. Please check the consitency of input dataset."
                        .format(abs_id)
                    )  # How to check what is wrong about mirror = -1 (Toshio 2018/01/11)
                params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0])
                ptl_list.append(abs_id)
            params_dict[iavg] = params_of_this_average
            list_dict[iavg] = members
            sp_utilities.write_text_row(
                params_of_this_average,
                os.path.join(
                    Tracker["constants"]["masterdir"],
                    "params_avg",
                    "params_avg_%03d.txt" % iavg,
                ),
            )
        ptl_list.sort()
        init_params = [None for im in range(len(ptl_list))]
        for im in range(len(ptl_list)):
            init_params[im] = [ptl_list[im]] + params_dict[global_dict[
                ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
        sp_utilities.write_text_row(
            init_params,
            os.path.join(Tracker["constants"]["masterdir"],
                         "init_isac_params.txt"),
        )
    else:
        params_dict = 0
        list_dict = 0
        memlist = 0
    params_dict = sp_utilities.wrap_mpi_bcast(params_dict,
                                              Blockdata["main_node"],
                                              communicator=mpi.MPI_COMM_WORLD)
    list_dict = sp_utilities.wrap_mpi_bcast(list_dict,
                                            Blockdata["main_node"],
                                            communicator=mpi.MPI_COMM_WORLD)
    memlist = sp_utilities.wrap_mpi_bcast(memlist,
                                          Blockdata["main_node"],
                                          communicator=mpi.MPI_COMM_WORLD)
    # Now computing!
    del init_dict
    tag_sharpen_avg = 1000
    ## always apply low pass filter to B_enhanced images to suppress noise in high frequencies
    enforced_to_H1 = False
    if B_enhance:
        if Tracker["constants"]["low_pass_filter"] == -1.0:
            enforced_to_H1 = True

    # distribute workload among mpi processes
    image_start, image_end = sp_applications.MPI_start_end(
        navg, Blockdata["nproc"], Blockdata["myid"])

    if Blockdata["myid"] == Blockdata["main_node"]:
        cpu_dict = {}
        for iproc in range(Blockdata["nproc"]):
            local_image_start, local_image_end = sp_applications.MPI_start_end(
                navg, Blockdata["nproc"], iproc)
            for im in range(local_image_start, local_image_end):
                cpu_dict[im] = iproc
    else:
        cpu_dict = 0

    cpu_dict = sp_utilities.wrap_mpi_bcast(cpu_dict,
                                           Blockdata["main_node"],
                                           communicator=mpi.MPI_COMM_WORLD)

    slist = [None for im in range(navg)]
    ini_list = [None for im in range(navg)]
    avg1_list = [None for im in range(navg)]
    avg2_list = [None for im in range(navg)]
    data_list = [None for im in range(navg)]
    plist_dict = {}

    if Blockdata["myid"] == Blockdata["main_node"]:
        if B_enhance:
            sp_global_def.sxprint(
                "Avg ID   B-factor  FH1(Res before ali) FH2(Res after ali)")
        else:
            sp_global_def.sxprint(
                "Avg ID   FH1(Res before ali)  FH2(Res after ali)")

    FH_list = [[0, 0.0, 0.0] for im in range(navg)]
    for iavg in range(image_start, image_end):

        mlist = EMAN2_cppwrap.EMData.read_images(
            Tracker["constants"]["orgstack"], list_dict[iavg])

        for im in range(len(mlist)):
            sp_utilities.set_params2D(mlist[im],
                                      params_dict[iavg][im],
                                      xform="xform.align2d")

        if options.local_alignment:
            new_avg, plist, FH2 = sp_applications.refinement_2d_local(
                mlist,
                Tracker["constants"]["radius"],
                a_range,
                x_range,
                y_range,
                CTF=do_ctf,
                SNR=1.0e10,
            )
            plist_dict[iavg] = plist
            FH1 = -1.0

        else:
            new_avg, frc, plist = compute_average(
                mlist, Tracker["constants"]["radius"], do_ctf)
            FH1 = get_optimistic_res(frc)
            FH2 = -1.0

        FH_list[iavg] = [iavg, FH1, FH2]

        if B_enhance:
            new_avg, gb = apply_enhancement(
                new_avg,
                Tracker["constants"]["B_start"],
                Tracker["constants"]["pixel_size"],
                Tracker["constants"]["Bfactor"],
            )
            sp_global_def.sxprint("  %6d      %6.3f  %4.3f  %4.3f" %
                                  (iavg, gb, FH1, FH2))

        elif adjust_to_given_pw2:
            roo = sp_utilities.read_text_file(Tracker["constants"]["modelpw"],
                                              -1)
            roo = roo[0]  # always on the first column
            new_avg = adjust_pw_to_model(new_avg,
                                         Tracker["constants"]["pixel_size"],
                                         roo)
            sp_global_def.sxprint("  %6d      %4.3f  %4.3f  " %
                                  (iavg, FH1, FH2))

        elif adjust_to_analytic_model:
            new_avg = adjust_pw_to_model(new_avg,
                                         Tracker["constants"]["pixel_size"],
                                         None)
            sp_global_def.sxprint("  %6d      %4.3f  %4.3f   " %
                                  (iavg, FH1, FH2))

        elif no_adjustment:
            pass

        if Tracker["constants"]["low_pass_filter"] != -1.0:
            if Tracker["constants"]["low_pass_filter"] == 0.0:
                low_pass_filter = FH1
            elif Tracker["constants"]["low_pass_filter"] == 1.0:
                low_pass_filter = FH2
                if not options.local_alignment:
                    low_pass_filter = FH1
            else:
                low_pass_filter = Tracker["constants"]["low_pass_filter"]
                if low_pass_filter >= 0.45:
                    low_pass_filter = 0.45
            new_avg = sp_filter.filt_tanl(new_avg, low_pass_filter, 0.02)
        else:  # No low pass filter but if enforced
            if enforced_to_H1:
                new_avg = sp_filter.filt_tanl(new_avg, FH1, 0.02)
        if B_enhance:
            new_avg = sp_fundamentals.fft(new_avg)

        new_avg.set_attr("members", list_dict[iavg])
        new_avg.set_attr("n_objects", len(list_dict[iavg]))
        slist[iavg] = new_avg
        sp_global_def.sxprint(
            time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()) + " =>",
            "Refined average %7d" % iavg,
        )

    ## send to main node to write
    mpi.mpi_barrier(mpi.MPI_COMM_WORLD)

    for im in range(navg):
        # avg
        if (cpu_dict[im] == Blockdata["myid"]
                and Blockdata["myid"] != Blockdata["main_node"]):
            sp_utilities.send_EMData(slist[im], Blockdata["main_node"],
                                     tag_sharpen_avg)

        elif (cpu_dict[im] == Blockdata["myid"]
              and Blockdata["myid"] == Blockdata["main_node"]):
            slist[im].set_attr("members", memlist[im])
            slist[im].set_attr("n_objects", len(memlist[im]))
            slist[im].write_image(
                os.path.join(Tracker["constants"]["masterdir"],
                             "class_averages.hdf"),
                im,
            )

        elif (cpu_dict[im] != Blockdata["myid"]
              and Blockdata["myid"] == Blockdata["main_node"]):
            new_avg_other_cpu = sp_utilities.recv_EMData(
                cpu_dict[im], tag_sharpen_avg)
            new_avg_other_cpu.set_attr("members", memlist[im])
            new_avg_other_cpu.set_attr("n_objects", len(memlist[im]))
            new_avg_other_cpu.write_image(
                os.path.join(Tracker["constants"]["masterdir"],
                             "class_averages.hdf"),
                im,
            )

        if options.local_alignment:
            if cpu_dict[im] == Blockdata["myid"]:
                sp_utilities.write_text_row(
                    plist_dict[im],
                    os.path.join(
                        Tracker["constants"]["masterdir"],
                        "ali2d_local_params_avg",
                        "ali2d_local_params_avg_%03d.txt" % im,
                    ),
                )

            if (cpu_dict[im] == Blockdata["myid"]
                    and cpu_dict[im] != Blockdata["main_node"]):
                sp_utilities.wrap_mpi_send(plist_dict[im],
                                           Blockdata["main_node"],
                                           mpi.MPI_COMM_WORLD)
                sp_utilities.wrap_mpi_send(FH_list, Blockdata["main_node"],
                                           mpi.MPI_COMM_WORLD)

            elif (cpu_dict[im] != Blockdata["main_node"]
                  and Blockdata["myid"] == Blockdata["main_node"]):
                dummy = sp_utilities.wrap_mpi_recv(cpu_dict[im],
                                                   mpi.MPI_COMM_WORLD)
                plist_dict[im] = dummy
                dummy = sp_utilities.wrap_mpi_recv(cpu_dict[im],
                                                   mpi.MPI_COMM_WORLD)
                FH_list[im] = dummy[im]
        else:
            if (cpu_dict[im] == Blockdata["myid"]
                    and cpu_dict[im] != Blockdata["main_node"]):
                sp_utilities.wrap_mpi_send(FH_list, Blockdata["main_node"],
                                           mpi.MPI_COMM_WORLD)

            elif (cpu_dict[im] != Blockdata["main_node"]
                  and Blockdata["myid"] == Blockdata["main_node"]):
                dummy = sp_utilities.wrap_mpi_recv(cpu_dict[im],
                                                   mpi.MPI_COMM_WORLD)
                FH_list[im] = dummy[im]

        mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
    mpi.mpi_barrier(mpi.MPI_COMM_WORLD)

    if options.local_alignment:
        if Blockdata["myid"] == Blockdata["main_node"]:
            ali3d_local_params = [None for im in range(len(ptl_list))]
            for im in range(len(ptl_list)):
                ali3d_local_params[im] = [ptl_list[im]] + plist_dict[
                    global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
            sp_utilities.write_text_row(
                ali3d_local_params,
                os.path.join(Tracker["constants"]["masterdir"],
                             "ali2d_local_params.txt"),
            )
            sp_utilities.write_text_row(
                FH_list,
                os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
    else:
        if Blockdata["myid"] == Blockdata["main_node"]:
            sp_utilities.write_text_row(
                FH_list,
                os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))

    mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
    target_xr = 3
    target_yr = 3
    if Blockdata["myid"] == 0:
        cmd = "{} {} {} {} {} {} {} {} {} {}".format(
            "sp_chains.py",
            os.path.join(Tracker["constants"]["masterdir"],
                         "class_averages.hdf"),
            os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"),
            os.path.join(Tracker["constants"]["masterdir"],
                         "ordered_class_averages.hdf"),
            "--circular",
            "--radius=%d" % Tracker["constants"]["radius"],
            "--xr=%d" % (target_xr + 1),
            "--yr=%d" % (target_yr + 1),
            "--align",
            ">/dev/null",
        )
        junk = sp_utilities.cmdexecute(cmd)
        cmd = "{} {}".format(
            "rm -rf",
            os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"))
        junk = sp_utilities.cmdexecute(cmd)

    return
Esempio n. 29
0
def helicalshiftali_MPI(stack,
                        maskfile=None,
                        maxit=100,
                        CTF=False,
                        snr=1.0,
                        Fourvar=False,
                        search_rng=-1):

    nproc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
    myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
    main_node = 0

    ftp = file_type(stack)

    if myid == main_node:
        print_begin_msg("helical-shiftali_MPI")

    max_iter = int(maxit)
    if (myid == main_node):
        infils = EMUtil.get_all_attributes(stack, "filament")
        ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord')
        filaments = ordersegments(infils, ptlcoords)
        total_nfils = len(filaments)
        inidl = [0] * total_nfils
        for i in range(total_nfils):
            inidl[i] = len(filaments[i])
        linidl = sum(inidl)
        nima = linidl
        tfilaments = []
        for i in range(total_nfils):
            tfilaments += filaments[i]
        del filaments
    else:
        total_nfils = 0
        linidl = 0
    total_nfils = bcast_number_to_all(total_nfils, source_node=main_node)
    if myid != main_node:
        inidl = [-1] * total_nfils
    inidl = bcast_list_to_all(inidl, myid, source_node=main_node)
    linidl = bcast_number_to_all(linidl, source_node=main_node)
    if myid != main_node:
        tfilaments = [-1] * linidl
    tfilaments = bcast_list_to_all(tfilaments, myid, source_node=main_node)
    filaments = []
    iendi = 0
    for i in range(total_nfils):
        isti = iendi
        iendi = isti + inidl[i]
        filaments.append(tfilaments[isti:iendi])
    del tfilaments, inidl

    if myid == main_node:
        print_msg("total number of filaments: %d" % total_nfils)
    if total_nfils < nproc:
        ERROR(
            'number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'
            % (nproc, total_nfils),
            myid=myid)

    #  balanced load
    temp = chunks_distribution([[len(filaments[i]), i]
                                for i in range(len(filaments))],
                               nproc)[myid:myid + 1][0]
    filaments = [filaments[temp[i][1]] for i in range(len(temp))]
    nfils = len(filaments)

    #filaments = [[0,1]]
    #print "filaments",filaments
    list_of_particles = []
    indcs = []
    k = 0
    for i in range(nfils):
        list_of_particles += filaments[i]
        k1 = k + len(filaments[i])
        indcs.append([k, k1])
        k = k1
    data = EMData.read_images(stack, list_of_particles)
    ldata = len(data)
    sxprint("ldata=", ldata)
    nx = data[0].get_xsize()
    ny = data[0].get_ysize()
    if maskfile == None:
        mrad = min(nx, ny) // 2 - 2
        mask = pad(model_blank(2 * mrad + 1, ny, 1, 1.0), nx, ny, 1, 0.0)
    else:
        mask = get_im(maskfile)

    # apply initial xform.align2d parameters stored in header
    init_params = []
    for im in range(ldata):
        t = data[im].get_attr('xform.align2d')
        init_params.append(t)
        p = t.get_params("2d")
        data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'],
                               p['mirror'], p['scale'])

    if CTF:
        from sp_filter import filt_ctf
        from sp_morphology import ctf_img
        ctf_abs_sum = EMData(nx, ny, 1, False)
        ctf_2_sum = EMData(nx, ny, 1, False)
    else:
        ctf_2_sum = None
        ctf_abs_sum = None

    from sp_utilities import info

    for im in range(ldata):
        data[im].set_attr('ID', list_of_particles[im])
        st = Util.infomask(data[im], mask, False)
        data[im] -= st[0]
        if CTF:
            ctf_params = data[im].get_attr("ctf")
            qctf = data[im].get_attr("ctf_applied")
            if qctf == 0:
                data[im] = filt_ctf(fft(data[im]), ctf_params)
                data[im].set_attr('ctf_applied', 1)
            elif qctf != 1:
                ERROR('Incorrectly set qctf flag', myid=myid)
            ctfimg = ctf_img(nx, ctf_params, ny=ny)
            Util.add_img2(ctf_2_sum, ctfimg)
            Util.add_img_abs(ctf_abs_sum, ctfimg)
        else:
            data[im] = fft(data[im])

    del list_of_particles

    if CTF:
        reduce_EMData_to_root(ctf_2_sum, myid, main_node)
        reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
    if CTF:
        if myid != main_node:
            del ctf_2_sum
            del ctf_abs_sum
        else:
            temp = EMData(nx, ny, 1, False)
            tsnr = 1. / snr
            for i in range(0, nx + 2, 2):
                for j in range(ny):
                    temp.set_value_at(i, j, tsnr)
                    temp.set_value_at(i + 1, j, 0.0)
            #info(ctf_2_sum)
            Util.add_img(ctf_2_sum, temp)
            #info(ctf_2_sum)
            del temp

    total_iter = 0
    shift_x = [0.0] * ldata

    for Iter in range(max_iter):
        if myid == main_node:
            start_time = time()
            print_msg("Iteration #%4d\n" % (total_iter))
        total_iter += 1
        avg = EMData(nx, ny, 1, False)
        for im in range(ldata):
            Util.add_img(avg, fshift(data[im], shift_x[im]))

        reduce_EMData_to_root(avg, myid, main_node)

        if myid == main_node:
            if CTF: tavg = Util.divn_filter(avg, ctf_2_sum)
            else: tavg = Util.mult_scalar(avg, 1.0 / float(nima))
        else:
            tavg = model_blank(nx, ny)

        if Fourvar:
            bcast_EMData_to_all(tavg, myid, main_node)
            vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)

        if myid == main_node:
            if Fourvar:
                tavg = fft(Util.divn_img(fft(tavg), vav))
                vav_r = Util.pack_complex_to_real(vav)
            # normalize and mask tavg in real space
            tavg = fft(tavg)
            stat = Util.infomask(tavg, mask, False)
            tavg -= stat[0]
            Util.mul_img(tavg, mask)
            tavg.write_image("tavg.hdf", Iter)
            # For testing purposes: shift tavg to some random place and see if the centering is still correct
            #tavg = rot_shift3D(tavg,sx=3,sy=-4)

        if Fourvar: del vav
        bcast_EMData_to_all(tavg, myid, main_node)
        tavg = fft(tavg)

        sx_sum = 0.0
        nxc = nx // 2

        for ifil in range(nfils):
            """
			# Calculate filament average
			avg = EMData(nx, ny, 1, False)
			filnima = 0
			for im in xrange(indcs[ifil][0], indcs[ifil][1]):
				Util.add_img(avg, data[im])
				filnima += 1
			tavg = Util.mult_scalar(avg, 1.0/float(filnima))
			"""
            # Calculate 1D ccf between each segment and filament average
            nsegms = indcs[ifil][1] - indcs[ifil][0]
            ctx = [None] * nsegms
            pcoords = [None] * nsegms
            for im in range(indcs[ifil][0], indcs[ifil][1]):
                ctx[im - indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx,
                                                       1)
                pcoords[im - indcs[ifil][0]] = data[im].get_attr(
                    'ptcl_source_coord')
                #ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0])
                #print "  CTX  ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True)
            # search for best x-shift
            cents = nsegms // 2

            dst = sqrt(
                max((pcoords[cents][0] - pcoords[0][0])**2 +
                    (pcoords[cents][1] - pcoords[0][1])**2,
                    (pcoords[cents][0] - pcoords[-1][0])**2 +
                    (pcoords[cents][1] - pcoords[-1][1])**2))
            maxincline = atan2(ny // 2 - 2 - float(search_rng), dst)
            kang = int(dst * tan(maxincline) + 0.5)
            #print  "  settings ",nsegms,cents,dst,search_rng,maxincline,kang

            # ## C code for alignment. @ming
            results = [0.0] * 3
            results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline,
                                         kang, search_rng, nxc)
            sib = int(results[0])
            bang = results[1]
            qm = results[2]
            #print qm, sib, bang

            # qm = -1.e23
            #
            # 			for six in xrange(-search_rng, search_rng+1,1):
            # 				q0 = ctx[cents].get_value_at(six+nxc)
            # 				for incline in xrange(kang+1):
            # 					qt = q0
            # 					qu = q0
            # 					if(kang>0):  tang = tan(maxincline/kang*incline)
            # 					else:        tang = 0.0
            # 					for kim in xrange(cents+1,nsegms):
            # 						dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
            # 						xl = dst*tang+six+nxc
            # 						ixl = int(xl)
            # 						dxl = xl - ixl
            # 						#print "  A  ", ifil,six,incline,kim,xl,ixl,dxl
            # 						qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
            # 						xl = -dst*tang+six+nxc
            # 						ixl = int(xl)
            # 						dxl = xl - ixl
            # 						qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
            # 					for kim in xrange(cents):
            # 						dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
            # 						xl = -dst*tang+six+nxc
            # 						ixl = int(xl)
            # 						dxl = xl - ixl
            # 						qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
            # 						xl =  dst*tang+six+nxc
            # 						ixl = int(xl)
            # 						dxl = xl - ixl
            # 						qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
            # 					if( qt > qm ):
            # 						qm = qt
            # 						sib = six
            # 						bang = tang
            # 					if( qu > qm ):
            # 						qm = qu
            # 						sib = six
            # 						bang = -tang
            #if incline == 0:  print  "incline = 0  ",six,tang,qt,qu
            #print qm,six,sib,bang
            #print " got results   ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True)
            for im in range(indcs[ifil][0], indcs[ifil][1]):
                kim = im - indcs[ifil][0]
                dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 +
                           (pcoords[cents][1] - pcoords[kim][1])**2)
                if (kim < cents): xl = -dst * bang + sib
                else: xl = dst * bang + sib
                shift_x[im] = xl

            # Average shift
            sx_sum += shift_x[indcs[ifil][0] + cents]

        # #print myid,sx_sum,total_nfils
        sx_sum = mpi.mpi_reduce(sx_sum, 1, mpi.MPI_FLOAT, mpi.MPI_SUM,
                                main_node, mpi.MPI_COMM_WORLD)
        if myid == main_node:
            sx_sum = float(sx_sum[0]) / total_nfils
            print_msg("Average shift  %6.2f\n" % (sx_sum))
        else:
            sx_sum = 0.0
        sx_sum = 0.0
        sx_sum = bcast_number_to_all(sx_sum, source_node=main_node)
        for im in range(ldata):
            shift_x[im] -= sx_sum
            #print  "   %3d  %6.3f"%(im,shift_x[im])
        #exit()

    # combine shifts found with the original parameters
    for im in range(ldata):
        t1 = Transform()
        ##import random
        ##shix=random.randint(-10, 10)
        ##t1.set_params({"type":"2D","tx":shix})
        t1.set_params({"type": "2D", "tx": shift_x[im]})
        # combine t0 and t1
        tt = t1 * init_params[im]
        data[im].set_attr("xform.align2d", tt)
    # write out headers and STOP, under MPI writing has to be done sequentially
    mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
    par_str = ["xform.align2d", "ID"]
    if myid == main_node:
        from sp_utilities import file_type
        if (file_type(stack) == "bdb"):
            from sp_utilities import recv_attr_dict_bdb
            recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata,
                               nproc)
        else:
            from sp_utilities import recv_attr_dict
            recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc)
    else:
        send_attr_dict(main_node, data, par_str, 0, ldata)
    if myid == main_node: print_end_msg("helical-shiftali_MPI")
Esempio n. 30
0
def main():
    def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
        # the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D
        if mirror:
            m = 1
            alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
                                                       540.0 - psi, 0, 0, 1.0)
        else:
            m = 0
            alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
                                                       360.0 - psi, 0, 0, 1.0)
        return alpha, sx, sy, m

    progname = os.path.basename(sys.argv[0])
    usage = progname + " prj_stack  --ave2D= --var2D=  --ave3D= --var3D= --img_per_grp= --fl=  --aa=   --sym=symmetry --CTF"
    parser = OptionParser(usage, version=SPARXVERSION)

    parser.add_option("--output_dir",
                      type="string",
                      default="./",
                      help="Output directory")
    parser.add_option("--ave2D",
                      type="string",
                      default=False,
                      help="Write to the disk a stack of 2D averages")
    parser.add_option("--var2D",
                      type="string",
                      default=False,
                      help="Write to the disk a stack of 2D variances")
    parser.add_option("--ave3D",
                      type="string",
                      default=False,
                      help="Write to the disk reconstructed 3D average")
    parser.add_option("--var3D",
                      type="string",
                      default=False,
                      help="Compute 3D variability (time consuming!)")
    parser.add_option(
        "--img_per_grp",
        type="int",
        default=100,
        help="Number of neighbouring projections.(Default is 100)")
    parser.add_option(
        "--no_norm",
        action="store_true",
        default=False,
        help="Do not use normalization.(Default is to apply normalization)")
    #parser.add_option("--radius", 	    type="int"         ,	default=-1   ,				help="radius for 3D variability" )
    parser.add_option(
        "--npad",
        type="int",
        default=2,
        help=
        "Number of time to pad the original images.(Default is 2 times padding)"
    )
    parser.add_option("--sym",
                      type="string",
                      default="c1",
                      help="Symmetry. (Default is no symmetry)")
    parser.add_option(
        "--fl",
        type="float",
        default=0.0,
        help=
        "Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)"
    )
    parser.add_option(
        "--aa",
        type="float",
        default=0.02,
        help=
        "Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)"
    )
    parser.add_option("--CTF",
                      action="store_true",
                      default=False,
                      help="Use CFT correction.(Default is no CTF correction)")
    #parser.add_option("--MPI" , 		action="store_true",	default=False,				help="use MPI version")
    #parser.add_option("--radiuspca", 	type="int"         ,	default=-1   ,				help="radius for PCA" )
    #parser.add_option("--iter", 		type="int"         ,	default=40   ,				help="maximum number of iterations (stop criterion of reconstruction process)" )
    #parser.add_option("--abs", 		type="float"   ,        default=0.0  ,				help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
    #parser.add_option("--squ", 		type="float"   ,	    default=0.0  ,				help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
    parser.add_option(
        "--VAR",
        action="store_true",
        default=False,
        help="Stack of input consists of 2D variances (Default False)")
    parser.add_option(
        "--decimate",
        type="float",
        default=0.25,
        help="Image decimate rate, a number less than 1. (Default is 0.25)")
    parser.add_option(
        "--window",
        type="int",
        default=0,
        help=
        "Target image size relative to original image size. (Default value is zero.)"
    )
    #parser.add_option("--SND",			action="store_true",	default=False,				help="compute squared normalized differences (Default False)")
    #parser.add_option("--nvec",			type="int"         ,	default=0    ,				help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)")
    parser.add_option(
        "--symmetrize",
        action="store_true",
        default=False,
        help="Prepare input stack for handling symmetry (Default False)")
    parser.add_option("--overhead",
                      type="float",
                      default=0.5,
                      help="python overhead per CPU.")

    (options, args) = parser.parse_args()
    #####
    from mpi import mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
    from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
    #from mpi import *
    from sp_applications import MPI_start_end
    from sp_reconstruction import recons3d_em, recons3d_em_MPI
    from sp_reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
    from sp_utilities import print_begin_msg, print_end_msg, print_msg
    from sp_utilities import read_text_row, get_image, get_im, wrap_mpi_send, wrap_mpi_recv
    from sp_utilities import bcast_EMData_to_all, bcast_number_to_all
    from sp_utilities import get_symt

    #  This is code for handling symmetries by the above program.  To be incorporated. PAP 01/27/2015

    from EMAN2db import db_open_dict

    # Set up global variables related to bdb cache
    if sp_global_def.CACHE_DISABLE:
        from sp_utilities import disable_bdb_cache
        disable_bdb_cache()

    # Set up global variables related to ERROR function
    sp_global_def.BATCH = True

    # detect if program is running under MPI
    RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
    if RUNNING_UNDER_MPI: sp_global_def.MPI = True
    if options.output_dir == "./":
        current_output_dir = os.path.abspath(options.output_dir)
    else:
        current_output_dir = options.output_dir
    if options.symmetrize:

        if mpi.mpi_comm_size(MPI_COMM_WORLD) > 1:
            ERROR("Cannot use more than one CPU for symmetry preparation")

        if not os.path.exists(current_output_dir):
            os.makedirs(current_output_dir)
            sp_global_def.write_command(current_output_dir)

        from sp_logger import Logger, BaseLogger_Files
        if os.path.exists(os.path.join(current_output_dir, "log.txt")):
            os.remove(os.path.join(current_output_dir, "log.txt"))
        log_main = Logger(BaseLogger_Files())
        log_main.prefix = os.path.join(current_output_dir, "./")

        instack = args[0]
        sym = options.sym.lower()
        if (sym == "c1"):
            ERROR("There is no need to symmetrize stack for C1 symmetry")

        line = ""
        for a in sys.argv:
            line += " " + a
        log_main.add(line)

        if (instack[:4] != "bdb:"):
            #if output_dir =="./": stack = "bdb:data"
            stack = "bdb:" + current_output_dir + "/data"
            delete_bdb(stack)
            junk = cmdexecute("sp_cpy.py  " + instack + "  " + stack)
        else:
            stack = instack

        qt = EMUtil.get_all_attributes(stack, 'xform.projection')

        na = len(qt)
        ts = get_symt(sym)
        ks = len(ts)
        angsa = [None] * na

        for k in range(ks):
            #Qfile = "Q%1d"%k
            #if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k)
            Qfile = os.path.join(current_output_dir, "Q%1d" % k)
            #delete_bdb("bdb:Q%1d"%k)
            delete_bdb("bdb:" + Qfile)
            #junk = cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
            junk = cmdexecute("e2bdb.py  " + stack + "  --makevstack=bdb:" +
                              Qfile)
            #DB = db_open_dict("bdb:Q%1d"%k)
            DB = db_open_dict("bdb:" + Qfile)
            for i in range(na):
                ut = qt[i] * ts[k]
                DB.set_attr(i, "xform.projection", ut)
                #bt = ut.get_params("spider")
                #angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
            #write_text_row(angsa, 'ptsma%1d.txt'%k)
            #junk = cmdexecute("e2bdb.py  "+stack+"  --makevstack=bdb:Q%1d"%k)
            #junk = cmdexecute("sxheader.py  bdb:Q%1d  --params=xform.projection  --import=ptsma%1d.txt"%(k,k))
            DB.close()
        #if options.output_dir =="./": delete_bdb("bdb:sdata")
        delete_bdb("bdb:" + current_output_dir + "/" + "sdata")
        #junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
        sdata = "bdb:" + current_output_dir + "/" + "sdata"
        sxprint(sdata)
        junk = cmdexecute("e2bdb.py   " + current_output_dir +
                          "  --makevstack=" + sdata + " --filt=Q")
        #junk = cmdexecute("ls  EMAN2DB/sdata*")
        #a = get_im("bdb:sdata")
        a = get_im(sdata)
        a.set_attr("variabilitysymmetry", sym)
        #a.write_image("bdb:sdata")
        a.write_image(sdata)

    else:

        from sp_fundamentals import window2d
        myid = mpi_comm_rank(MPI_COMM_WORLD)
        number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
        main_node = 0
        shared_comm = mpi_comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED,
                                          0, MPI_INFO_NULL)
        myid_on_node = mpi_comm_rank(shared_comm)
        no_of_processes_per_group = mpi_comm_size(shared_comm)
        masters_from_groups_vs_everything_else_comm = mpi_comm_split(
            MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node)
        color, no_of_groups, balanced_processor_load_on_nodes = get_colors_and_subsets(main_node, MPI_COMM_WORLD, myid, \
            shared_comm, myid_on_node, masters_from_groups_vs_everything_else_comm)
        overhead_loading = options.overhead * number_of_proc
        #memory_per_node  = options.memory_per_node
        #if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group
        keepgoing = 1

        current_window = options.window
        current_decimate = options.decimate

        if len(args) == 1: stack = args[0]
        else:
            sxprint("Usage: " + usage)
            sxprint("Please run \'" + progname + " -h\' for detailed options")
            ERROR(
                "Invalid number of parameters used. Please see usage information above."
            )
            return

        t0 = time()
        # obsolete flags
        options.MPI = True
        #options.nvec = 0
        options.radiuspca = -1
        options.iter = 40
        options.abs = 0.0
        options.squ = 0.0

        if options.fl > 0.0 and options.aa == 0.0:
            ERROR("Fall off has to be given for the low-pass filter",
                  myid=myid)

        #if options.VAR and options.SND:
        #	ERROR( "Only one of var and SND can be set!",myid=myid )

        if options.VAR and (options.ave2D or options.ave3D or options.var2D):
            ERROR(
                "When VAR is set, the program cannot output ave2D, ave3D or var2D",
                myid=myid)

        #if options.SND and (options.ave2D or options.ave3D):
        #	ERROR( "When SND is set, the program cannot output ave2D or ave3D", myid=myid )

        #if options.nvec > 0 :
        #	ERROR( "PCA option not implemented", myid=myid )

        #if options.nvec > 0 and options.ave3D == None:
        #	ERROR( "When doing PCA analysis, one must set ave3D", myid=myid )

        if current_decimate > 1.0 or current_decimate < 0.0:
            ERROR("Decimate rate should be a value between 0.0 and 1.0",
                  myid=myid)

        if current_window < 0.0:
            ERROR("Target window size should be always larger than zero",
                  myid=myid)

        if myid == main_node:
            img = get_image(stack, 0)
            nx = img.get_xsize()
            ny = img.get_ysize()
            if (min(nx, ny) < current_window): keepgoing = 0
        keepgoing = bcast_number_to_all(keepgoing, main_node, MPI_COMM_WORLD)
        if keepgoing == 0:
            ERROR(
                "The target window size cannot be larger than the size of decimated image",
                myid=myid)

        import string
        options.sym = options.sym.lower()
        # if global_def.CACHE_DISABLE:
        # 	from utilities import disable_bdb_cache
        # 	disable_bdb_cache()
        # global_def.BATCH = True

        if myid == main_node:
            if not os.path.exists(current_output_dir):
                os.makedirs(current_output_dir
                            )  # Never delete output_dir in the program!

        img_per_grp = options.img_per_grp
        #nvec        = options.nvec
        radiuspca = options.radiuspca
        from sp_logger import Logger, BaseLogger_Files
        #if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
        log_main = Logger(BaseLogger_Files())
        log_main.prefix = os.path.join(current_output_dir, "./")

        if myid == main_node:
            line = ""
            for a in sys.argv:
                line += " " + a
            log_main.add(line)
            log_main.add("-------->>>Settings given by all options<<<-------")
            log_main.add("Symmetry             : %s" % options.sym)
            log_main.add("Input stack          : %s" % stack)
            log_main.add("Output_dir           : %s" % current_output_dir)

            if options.ave3D:
                log_main.add("Ave3d                : %s" % options.ave3D)
            if options.var3D:
                log_main.add("Var3d                : %s" % options.var3D)
            if options.ave2D:
                log_main.add("Ave2D                : %s" % options.ave2D)
            if options.var2D:
                log_main.add("Var2D                : %s" % options.var2D)
            if options.VAR: log_main.add("VAR                  : True")
            else: log_main.add("VAR                  : False")
            if options.CTF: log_main.add("CTF correction       : True  ")
            else: log_main.add("CTF correction       : False ")

            log_main.add("Image per group      : %5d" % options.img_per_grp)
            log_main.add("Image decimate rate  : %4.3f" % current_decimate)
            log_main.add("Low pass filter      : %4.3f" % options.fl)
            current_fl = options.fl
            if current_fl == 0.0: current_fl = 0.5
            log_main.add(
                "Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"
                % round((current_fl * current_decimate), 3))
            log_main.add("Window size          : %5d " % current_window)
            log_main.add("sx3dvariability begins")

        symbaselen = 0
        if myid == main_node:
            nima = EMUtil.get_image_count(stack)
            img = get_image(stack)
            nx = img.get_xsize()
            ny = img.get_ysize()
            nnxo = nx
            nnyo = ny
            if options.sym != "c1":
                imgdata = get_im(stack)
                try:
                    i = imgdata.get_attr("variabilitysymmetry").lower()
                    if (i != options.sym):
                        ERROR(
                            "The symmetry provided does not agree with the symmetry of the input stack",
                            myid=myid)
                except:
                    ERROR(
                        "Input stack is not prepared for symmetry, please follow instructions",
                        myid=myid)
                from sp_utilities import get_symt
                i = len(get_symt(options.sym))
                if ((nima / i) * i != nima):
                    ERROR(
                        "The length of the input stack is incorrect for symmetry processing",
                        myid=myid)
                symbaselen = nima / i
            else:
                symbaselen = nima
        else:
            nima = 0
            nx = 0
            ny = 0
            nnxo = 0
            nnyo = 0
        nima = bcast_number_to_all(nima)
        nx = bcast_number_to_all(nx)
        ny = bcast_number_to_all(ny)
        nnxo = bcast_number_to_all(nnxo)
        nnyo = bcast_number_to_all(nnyo)
        if current_window > max(nx, ny):
            ERROR("Window size is larger than the original image size")

        if current_decimate == 1.:
            if current_window != 0:
                nx = current_window
                ny = current_window
        else:
            if current_window == 0:
                nx = int(nx * current_decimate + 0.5)
                ny = int(ny * current_decimate + 0.5)
            else:
                nx = int(current_window * current_decimate + 0.5)
                ny = nx
        symbaselen = bcast_number_to_all(symbaselen)

        # check FFT prime number
        from sp_fundamentals import smallprime
        is_fft_friendly = (nx == smallprime(nx))

        if not is_fft_friendly:
            if myid == main_node:
                log_main.add(
                    "The target image size is not a product of small prime numbers"
                )
                log_main.add("Program adjusts the input settings!")
            ### two cases
            if current_decimate == 1.:
                nx = smallprime(nx)
                ny = nx
                current_window = nx  # update
                if myid == main_node:
                    log_main.add("The window size is updated to %d." %
                                 current_window)
            else:
                if current_window == 0:
                    nx = smallprime(int(nx * current_decimate + 0.5))
                    current_decimate = float(nx) / nnxo
                    ny = nx
                    if (myid == main_node):
                        log_main.add("The decimate rate is updated to %f." %
                                     current_decimate)
                else:
                    nx = smallprime(
                        int(current_window * current_decimate + 0.5))
                    ny = nx
                    current_window = int(nx / current_decimate + 0.5)
                    if (myid == main_node):
                        log_main.add("The window size is updated to %d." %
                                     current_window)

        if myid == main_node:
            log_main.add("The target image size is %d" % nx)

        if radiuspca == -1: radiuspca = nx / 2 - 2
        if myid == main_node:
            log_main.add("%-70s:  %d\n" % ("Number of projection", nima))
        img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
        """
		if options.SND:
			from sp_projection		import prep_vol, prgs
			from sp_statistics		import im_diff
			from sp_utilities		import get_im, model_circle, get_params_proj, set_params_proj
			from sp_utilities		import get_ctf, generate_ctf
			from sp_filter			import filt_ctf
		
			imgdata = EMData.read_images(stack, range(img_begin, img_end))

			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)

			bcast_EMData_to_all(vol, myid)
			volft, kb = prep_vol(vol)

			mask = model_circle(nx/2-2, nx, ny)
			varList = []
			for i in xrange(img_begin, img_end):
				phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
				ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
				if options.CTF:
					ctf_params = get_ctf(imgdata[i-img_begin])
					ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
				diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
				diff2 = diff*diff
				set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
				varList.append(diff2)
			mpi_barrier(MPI_COMM_WORLD)
		"""

        if options.VAR:  # 2D variance images have no shifts
            #varList   = EMData.read_images(stack, range(img_begin, img_end))
            from EMAN2 import Region
            for index_of_particle in range(img_begin, img_end):
                image = get_im(stack, index_of_proj)
                if current_window > 0:
                    varList.append(
                        fdecimate(
                            window2d(image, current_window, current_window),
                            nx, ny))
                else:
                    varList.append(fdecimate(image, nx, ny))

        else:
            from sp_utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
            from sp_utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
            from sp_utilities import model_blank, nearest_proj, model_circle, write_text_row, wrap_mpi_gatherv
            from sp_applications import pca
            from sp_statistics import avgvar, avgvar_ctf, ccc
            from sp_filter import filt_tanl
            from sp_morphology import threshold, square_root
            from sp_projection import project, prep_vol, prgs
            from sets import Set
            from sp_utilities import wrap_mpi_recv, wrap_mpi_bcast, wrap_mpi_send
            import numpy as np
            if myid == main_node:
                t1 = time()
                proj_angles = []
                aveList = []
                tab = EMUtil.get_all_attributes(stack, 'xform.projection')
                for i in range(nima):
                    t = tab[i].get_params('spider')
                    phi = t['phi']
                    theta = t['theta']
                    psi = t['psi']
                    x = theta
                    if x > 90.0: x = 180.0 - x
                    x = x * 10000 + psi
                    proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
                t2 = time()
                log_main.add(
                    "%-70s:  %d\n" %
                    ("Number of neighboring projections", img_per_grp))
                log_main.add("...... Finding neighboring projections\n")
                log_main.add("Number of images per group: %d" % img_per_grp)
                log_main.add("Now grouping projections")
                proj_angles.sort()
                proj_angles_list = np.full((nima, 4), 0.0, dtype=np.float32)
                for i in range(nima):
                    proj_angles_list[i][0] = proj_angles[i][1]
                    proj_angles_list[i][1] = proj_angles[i][2]
                    proj_angles_list[i][2] = proj_angles[i][3]
                    proj_angles_list[i][3] = proj_angles[i][4]
            else:
                proj_angles_list = 0
            proj_angles_list = wrap_mpi_bcast(proj_angles_list, main_node,
                                              MPI_COMM_WORLD)
            proj_angles = []
            for i in range(nima):
                proj_angles.append([
                    proj_angles_list[i][0], proj_angles_list[i][1],
                    proj_angles_list[i][2],
                    int(proj_angles_list[i][3])
                ])
            del proj_angles_list
            proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp,
                                                  range(img_begin, img_end))
            all_proj = Set()
            for im in proj_list:
                for jm in im:
                    all_proj.add(proj_angles[jm][3])
            all_proj = list(all_proj)
            index = {}
            for i in range(len(all_proj)):
                index[all_proj[i]] = i
            mpi_barrier(MPI_COMM_WORLD)
            if myid == main_node:
                log_main.add("%-70s:  %.2f\n" %
                             ("Finding neighboring projections lasted [s]",
                              time() - t2))
                log_main.add("%-70s:  %d\n" %
                             ("Number of groups processed on the main node",
                              len(proj_list)))
                log_main.add("Grouping projections took:  %12.1f [m]" %
                             ((time() - t2) / 60.))
                log_main.add("Number of groups on main node: ", len(proj_list))
            mpi_barrier(MPI_COMM_WORLD)

            if myid == main_node:
                log_main.add("...... Calculating the stack of 2D variances \n")
            # Memory estimation. There are two memory consumption peaks
            # peak 1. Compute ave, var;
            # peak 2. Var volume reconstruction;
            # proj_params = [0.0]*(nima*5)
            aveList = []
            varList = []
            #if nvec > 0: eigList = [[] for i in range(nvec)]
            dnumber = len(
                all_proj)  # all neighborhood set for assigned to myid
            pnumber = len(proj_list) * 2. + img_per_grp  # aveList and varList
            tnumber = dnumber + pnumber
            vol_size2 = nx**3 * 4. * 8 / 1.e9
            vol_size1 = 2. * nnxo**3 * 4. * 8 / 1.e9
            proj_size = nnxo * nnyo * len(
                proj_list) * 4. * 2. / 1.e9  # both aveList and varList
            orig_data_size = nnxo * nnyo * 4. * tnumber / 1.e9
            reduced_data_size = nx * nx * 4. * tnumber / 1.e9
            full_data = np.full((number_of_proc, 2), -1., dtype=np.float16)
            full_data[myid] = orig_data_size, reduced_data_size
            if myid != main_node:
                wrap_mpi_send(full_data, main_node, MPI_COMM_WORLD)
            if myid == main_node:
                for iproc in range(number_of_proc):
                    if iproc != main_node:
                        dummy = wrap_mpi_recv(iproc, MPI_COMM_WORLD)
                        full_data[np.where(dummy > -1)] = dummy[np.where(
                            dummy > -1)]
                del dummy
            mpi_barrier(MPI_COMM_WORLD)
            full_data = wrap_mpi_bcast(full_data, main_node, MPI_COMM_WORLD)
            # find the CPU with heaviest load
            minindx = np.argsort(full_data, 0)
            heavy_load_myid = minindx[-1][1]
            total_mem = sum(full_data)
            if myid == main_node:
                if current_window == 0:
                    log_main.add(
                        "Nx:   current image size = %d. Decimated by %f from %d"
                        % (nx, current_decimate, nnxo))
                else:
                    log_main.add(
                        "Nx:   current image size = %d. Windowed to %d, and decimated by %f from %d"
                        % (nx, current_window, current_decimate, nnxo))
                log_main.add("Nproj:       number of particle images.")
                log_main.add("Navg:        number of 2D average images.")
                log_main.add("Nvar:        number of 2D variance images.")
                log_main.add(
                    "Img_per_grp: user defined image per group for averaging = %d"
                    % img_per_grp)
                log_main.add(
                    "Overhead:    total python overhead memory consumption   = %f"
                    % overhead_loading)
                log_main.add("Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"%\
                   (total_mem[1] + overhead_loading))
            del full_data
            mpi_barrier(MPI_COMM_WORLD)
            if myid == heavy_load_myid:
                log_main.add(
                    "Begin reading and preprocessing images on processor. Wait... "
                )
                ttt = time()
            #imgdata = EMData.read_images(stack, all_proj)
            imgdata = [None for im in range(len(all_proj))]
            for index_of_proj in range(len(all_proj)):
                #image = get_im(stack, all_proj[index_of_proj])
                if (current_window > 0):
                    imgdata[index_of_proj] = fdecimate(
                        window2d(get_im(stack, all_proj[index_of_proj]),
                                 current_window, current_window), nx, ny)
                else:
                    imgdata[index_of_proj] = fdecimate(
                        get_im(stack, all_proj[index_of_proj]), nx, ny)

                if (current_decimate > 0.0 and options.CTF):
                    ctf = imgdata[index_of_proj].get_attr("ctf")
                    ctf.apix = ctf.apix / current_decimate
                    imgdata[index_of_proj].set_attr("ctf", ctf)

                if myid == heavy_load_myid and index_of_proj % 100 == 0:
                    log_main.add(" ...... %6.2f%% " %
                                 (index_of_proj / float(len(all_proj)) * 100.))
            mpi_barrier(MPI_COMM_WORLD)
            if myid == heavy_load_myid:
                log_main.add("All_proj preprocessing cost %7.2f m" %
                             ((time() - ttt) / 60.))
                log_main.add("Wait untill reading on all CPUs done...")
            '''	
			imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
			if options.fl > 0.0:
				for k in xrange(len(imgdata2)):
					imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
			if options.CTF:
				vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			else:
				vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
			if myid == main_node:
				vol.write_image("vol_ctf.hdf")
				print_msg("Writing to the disk volume reconstructed from averages as		:  %s\n"%("vol_ctf.hdf"))
			del vol, imgdata2
			mpi_barrier(MPI_COMM_WORLD)
			'''
            from sp_applications import prepare_2d_forPCA
            from sp_utilities import model_blank
            from EMAN2 import Transform
            if not options.no_norm:
                mask = model_circle(nx / 2 - 2, nx, nx)
            if options.CTF:
                from sp_utilities import pad
                from sp_filter import filt_ctf
            from sp_filter import filt_tanl
            if myid == heavy_load_myid:
                log_main.add("Start computing 2D aveList and varList. Wait...")
                ttt = time()
            inner = nx // 2 - 4
            outer = inner + 2
            xform_proj_for_2D = [None for i in range(len(proj_list))]
            for i in range(len(proj_list)):
                ki = proj_angles[proj_list[i][0]][3]
                if ki >= symbaselen: continue
                mi = index[ki]
                dpar = Util.get_transform_params(imgdata[mi],
                                                 "xform.projection", "spider")
                phiM, thetaM, psiM, s2xM, s2yM = dpar["phi"], dpar[
                    "theta"], dpar[
                        "psi"], -dpar["tx"] * current_decimate, -dpar[
                            "ty"] * current_decimate
                grp_imgdata = []
                for j in range(img_per_grp):
                    mj = index[proj_angles[proj_list[i][j]][3]]
                    cpar = Util.get_transform_params(imgdata[mj],
                                                     "xform.projection",
                                                     "spider")
                    alpha, sx, sy, mirror = params_3D_2D_NEW(
                        cpar["phi"], cpar["theta"], cpar["psi"],
                        -cpar["tx"] * current_decimate,
                        -cpar["ty"] * current_decimate, mirror_list[i][j])
                    if thetaM <= 90:
                        if mirror == 0:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0,
                                0.0, 1.0)
                        else:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0, 180 - (phiM - cpar["phi"]),
                                0.0, 0.0, 1.0)
                    else:
                        if mirror == 0:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0, -(phiM - cpar["phi"]), 0.0,
                                0.0, 1.0)
                        else:
                            alpha, sx, sy, scale = compose_transform2(
                                alpha, sx, sy, 1.0,
                                -(180 - (phiM - cpar["phi"])), 0.0, 0.0, 1.0)
                    imgdata[mj].set_attr(
                        "xform.align2d",
                        Transform({
                            "type": "2D",
                            "alpha": alpha,
                            "tx": sx,
                            "ty": sy,
                            "mirror": mirror,
                            "scale": 1.0
                        }))
                    grp_imgdata.append(imgdata[mj])
                if not options.no_norm:
                    for k in range(img_per_grp):
                        ave, std, minn, maxx = Util.infomask(
                            grp_imgdata[k], mask, False)
                        grp_imgdata[k] -= ave
                        grp_imgdata[k] /= std
                if options.fl > 0.0:
                    for k in range(img_per_grp):
                        grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl,
                                                   options.aa)

                #  Because of background issues, only linear option works.
                if options.CTF:
                    ave, var = aves_wiener(grp_imgdata,
                                           SNR=1.0e5,
                                           interpolation_method="linear")
                else:
                    ave, var = ave_var(grp_imgdata)
                # Switch to std dev
                # threshold is not really needed,it is just in case due to numerical accuracy something turns out negative.
                var = square_root(threshold(var))

                set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
                set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])

                aveList.append(ave)
                varList.append(var)
                xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0]
                '''
				if nvec > 0:
					eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
					for k in range(nvec):
						set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
						eigList[k].append(eig[k])
					"""
					if myid == 0 and i == 0:
						for k in xrange(nvec):
							eig[k].write_image("eig.hdf", k)
					"""
				'''
                if (myid == heavy_load_myid) and (i % 100 == 0):
                    log_main.add(" ......%6.2f%%  " %
                                 (i / float(len(proj_list)) * 100.))
            del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index
            if not options.no_norm: del mask
            if myid == main_node: del tab
            #  At this point, all averages and variances are computed
            mpi_barrier(MPI_COMM_WORLD)

            if (myid == heavy_load_myid):
                log_main.add("Computing aveList and varList took %12.1f [m]" %
                             ((time() - ttt) / 60.))

            xform_proj_for_2D = wrap_mpi_gatherv(xform_proj_for_2D, main_node,
                                                 MPI_COMM_WORLD)
            if (myid == main_node):
                write_text_row([str(entry) for entry in xform_proj_for_2D],
                               os.path.join(current_output_dir, "params.txt"))
            del xform_proj_for_2D
            mpi_barrier(MPI_COMM_WORLD)
            if options.ave2D:
                from sp_fundamentals import fpol
                from sp_applications import header
                if myid == main_node:
                    log_main.add("Compute ave2D ... ")
                    km = 0
                    for i in range(number_of_proc):
                        if i == main_node:
                            for im in range(len(aveList)):
                                aveList[im].write_image(
                                    os.path.join(current_output_dir,
                                                 options.ave2D), km)
                                km += 1
                        else:
                            nl = mpi_recv(1, MPI_INT, i,
                                          SPARX_MPI_TAG_UNIVERSAL,
                                          MPI_COMM_WORLD)
                            nl = int(nl[0])
                            for im in range(nl):
                                ave = recv_EMData(i, im + i + 70000)
                                """
								nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								nm = int(nm[0])
								members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('members', map(int, members))
								members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('pix_err', map(float, members))
								members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
								ave.set_attr('refprojdir', map(float, members))
								"""
                                tmpvol = fpol(ave, nx, nx, 1)
                                tmpvol.write_image(
                                    os.path.join(current_output_dir,
                                                 options.ave2D), km)
                                km += 1
                else:
                    mpi_send(len(aveList), 1, MPI_INT, main_node,
                             SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
                    for im in range(len(aveList)):
                        send_EMData(aveList[im], main_node, im + myid + 70000)
                        """
						members = aveList[im].get_attr('members')
						mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						members = aveList[im].get_attr('pix_err')
						mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						try:
							members = aveList[im].get_attr('refprojdir')
							mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						except:
							mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
						"""
                if myid == main_node:
                    header(os.path.join(current_output_dir, options.ave2D),
                           params='xform.projection',
                           fimport=os.path.join(current_output_dir,
                                                "params.txt"))
                mpi_barrier(MPI_COMM_WORLD)
            if options.ave3D:
                from sp_fundamentals import fpol
                t5 = time()
                if myid == main_node: log_main.add("Reconstruct ave3D ... ")
                ave3D = recons3d_4nn_MPI(myid,
                                         aveList,
                                         symmetry=options.sym,
                                         npad=options.npad)
                bcast_EMData_to_all(ave3D, myid)
                if myid == main_node:
                    if current_decimate != 1.0:
                        ave3D = resample(ave3D, 1. / current_decimate)
                    ave3D = fpol(ave3D, nnxo, nnxo,
                                 nnxo)  # always to the orignal image size
                    set_pixel_size(ave3D, 1.0)
                    ave3D.write_image(
                        os.path.join(current_output_dir, options.ave3D))
                    log_main.add("Ave3D reconstruction took %12.1f [m]" %
                                 ((time() - t5) / 60.0))
                    log_main.add("%-70s:  %s\n" %
                                 ("The reconstructed ave3D is saved as ",
                                  options.ave3D))

            mpi_barrier(MPI_COMM_WORLD)
            del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList
            '''
			if nvec > 0:
				for k in range(nvec):
					if myid == main_node:log_main.add("Reconstruction eigenvolumes", k)
					cont = True
					ITER = 0
					mask2d = model_circle(radiuspca, nx, nx)
					while cont:
						#print "On node %d, iteration %d"%(myid, ITER)
						eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
						bcast_EMData_to_all(eig3D, myid, main_node)
						if options.fl > 0.0:
							eig3D = filt_tanl(eig3D, options.fl, options.aa)
						if myid == main_node:
							eig3D.write_image(os.path.join(options.outpout_dir, "eig3d_%03d.hdf"%(k, ITER)))
						Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
						eig3Df, kb = prep_vol(eig3D)
						del eig3D
						cont = False
						icont = 0
						for l in range(len(eigList[k])):
							phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
							proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
							cl = ccc(proj, eigList[k][l], mask2d)
							if cl < 0.0:
								icont += 1
								cont = True
								eigList[k][l] *= -1.0
						u = int(cont)
						u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
						icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)

						if myid == main_node:
							u = int(u[0])
							log_main.add(" Eigenvector: ",k," number changed ",int(icont[0]))
						else: u = 0
						u = bcast_number_to_all(u, main_node)
						cont = bool(u)
						ITER += 1

					del eig3Df, kb
					mpi_barrier(MPI_COMM_WORLD)
				del eigList, mask2d
			'''
            if options.ave3D: del ave3D
            if options.var2D:
                from sp_fundamentals import fpol
                from sp_applications import header
                if myid == main_node:
                    log_main.add("Compute var2D...")
                    km = 0
                    for i in range(number_of_proc):
                        if i == main_node:
                            for im in range(len(varList)):
                                tmpvol = fpol(varList[im], nx, nx, 1)
                                tmpvol.write_image(
                                    os.path.join(current_output_dir,
                                                 options.var2D), km)
                                km += 1
                        else:
                            nl = mpi_recv(1, MPI_INT, i,
                                          SPARX_MPI_TAG_UNIVERSAL,
                                          MPI_COMM_WORLD)
                            nl = int(nl[0])
                            for im in range(nl):
                                ave = recv_EMData(i, im + i + 70000)
                                tmpvol = fpol(ave, nx, nx, 1)
                                tmpvol.write_image(
                                    os.path.join(current_output_dir,
                                                 options.var2D), km)
                                km += 1
                else:
                    mpi_send(len(varList), 1, MPI_INT, main_node,
                             SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
                    for im in range(len(varList)):
                        send_EMData(varList[im], main_node, im + myid +
                                    70000)  #  What with the attributes??
                mpi_barrier(MPI_COMM_WORLD)
                if myid == main_node:
                    from sp_applications import header
                    header(os.path.join(current_output_dir, options.var2D),
                           params='xform.projection',
                           fimport=os.path.join(current_output_dir,
                                                "params.txt"))
                mpi_barrier(MPI_COMM_WORLD)
        if options.var3D:
            if myid == main_node: log_main.add("Reconstruct var3D ...")
            t6 = time()
            # radiusvar = options.radius
            # if( radiusvar < 0 ):  radiusvar = nx//2 -3
            res = recons3d_4nn_MPI(myid,
                                   varList,
                                   symmetry=options.sym,
                                   npad=options.npad)
            #res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
            if myid == main_node:
                from sp_fundamentals import fpol
                if current_decimate != 1.0:
                    res = resample(res, 1. / current_decimate)
                res = fpol(res, nnxo, nnxo, nnxo)
                set_pixel_size(res, 1.0)
                res.write_image(os.path.join(current_output_dir,
                                             options.var3D))
                log_main.add(
                    "%-70s:  %s\n" %
                    ("The reconstructed var3D is saved as ", options.var3D))
                log_main.add("Var3D reconstruction took %f12.1 [m]" %
                             ((time() - t6) / 60.0))
                log_main.add("Total computation time %f12.1 [m]" %
                             ((time() - t0) / 60.0))
                log_main.add("sx3dvariability finishes")

        if RUNNING_UNDER_MPI:
            sp_global_def.MPI = False

        sp_global_def.BATCH = False