def test_combinemaps_Halfset(self):
testargs_new = [
path.join(ABSOLUTE_BIN_PATH, "sp_process.py"), "--combinemaps",
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"11_MERIDIEN", "vol_0_unfil_028.hdf"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"11_MERIDIEN", "vol_1_unfil_028.hdf"),
"--output_dir=" + self.new_output_folder, "--pixel_size=1.14",
"--do_adaptive_mask", "--threshold=0.02", "--mtf=" +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"FalconIImtf.txt")
]
testargs_old = [
path.join(ABSOLUTE_OLDBIN_PATH, "sp_process.py"), "--combinemaps",
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"11_MERIDIEN", "vol_0_unfil_028.hdf"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"11_MERIDIEN", "vol_1_unfil_028.hdf"),
"--output_dir=" + self.old_output_folder, "--pixel_size=1.14",
"--do_adaptive_mask", "--threshold=0.02", "--mtf=" +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"FalconIImtf.txt")
]
with patch.object(sys, 'argv', testargs_new):
fu.main()
with patch.object(sys, 'argv', testargs_old):
oldfu.main()
return_new = get_im(path.join(self.new_output_folder, self.fname))
return_old = get_im(path.join(self.old_output_folder, self.fname))
self.assertTrue(
array_equal(return_new.get_3dview(), return_old.get_3dview()))
def test_calls(self):
old_final = "old_final.hdf"
new_final = "new_final.hdf"
testargs_new = [
str(MPI_PATH) + " -np " + str(NUM_PROC) + " " +
path.join(ABSOLUTE_BIN_PATH, "sp_filterlocal.py") + " " +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"12_POSTREFINER", "vol_combined.hdf") + " " +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"17_LOCAL_RES", "localres.hdf") + " " +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"12_POSTREFINER", "vol_adaptive_mask.hdf") + " " +
new_final + " --radius=145 --MPI"
]
testargs_old = [
str(MPI_PATH) + " -np " + str(NUM_PROC) + " " +
path.join(ABSOLUTE_OLDBIN_PATH, "sp_filterlocal.py") + " " +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"12_POSTREFINER", "vol_combined.hdf") + " " +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"17_LOCAL_RES", "localres.hdf") + " " +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"12_POSTREFINER", "vol_adaptive_mask.hdf") + " " +
old_final + " --radius=145 --MPI"
]
a = subprocess.run(args=testargs_new,
shell=True,
stderr=subprocess.STDOUT)
b = subprocess.run(args=testargs_old,
shell=True,
stderr=subprocess.STDOUT)
return_old = get_im(old_final)
return_new = get_im(new_final)
self.assertTrue(
array_equal(return_old.get_3dview(), return_new.get_3dview()))
self.assertTrue(
allclose(
return_new.get_3dview().flatten().tolist()[4034427:4034527], [
3.201385334250517e-05, -0.0033583296462893486,
-0.0033261957578361034, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0
],
atol=0.01))
remove_list_of_file([old_final, new_final])
def test_(self):
out_dir_old = "oldviper"
out_dir_new = "newviper"
filename_vol = "volf.hdf"
filename_refvol = "refvol2.hdf"
testcommand_new = (
MPI_PATH + " -np " + str(NUM_PROC) + " " +
path.join(ABSOLUTE_OLDBIN_PATH, "sp_viper.py") + " " +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"04_ISAC", "class_averages.hdf") + " " + out_dir_old +
" --sym=c5")
testcommand_old = (
MPI_PATH + " -np " + str(NUM_PROC) + " " +
path.join(ABSOLUTE_BIN_PATH, "sp_viper.py") + " " +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"04_ISAC", "class_averages.hdf") + " " + out_dir_new +
" --sym=c5")
subprocess.run(args=[testcommand_new],
shell=True,
stderr=subprocess.STDOUT)
subprocess.run(args=[testcommand_old],
shell=True,
stderr=subprocess.STDOUT)
return_new_avg = get_im(
path.join(ABSOLUTE_PATH, out_dir_new, filename_vol))
return_new_var = get_im(
path.join(ABSOLUTE_PATH, out_dir_new, filename_refvol))
return_old_avg = get_im(
path.join(ABSOLUTE_PATH, out_dir_old, filename_vol))
return_old_var = get_im(
path.join(ABSOLUTE_PATH, out_dir_old, filename_refvol))
self.assertTrue(
allclose(return_new_avg.get_3dview(),
return_old_avg.get_3dview(),
atol=TOLERANCE))
self.assertTrue(
allclose(return_old_var.get_3dview(),
return_new_var.get_3dview(),
atol=TOLERANCE))
remove_dir(out_dir_new)
remove_dir(out_dir_old)
def read_volume(path_vol_1,
path_vol_2,
path_mask=None,
resolution=None,
pixel_size=None):
"""
Reads the 3D density map
:param path_vol_1: Path to the density map
:param path_mask: If provided, the volume is masked.
:return: Volume for density map
"""
print("Read volumes, masking them and prepare them for reprojections")
mask_vol = None
vol2 = None
if path_mask is not None:
mask_vol = sp_utilities.get_im(path_mask)
vol1 = prepare_volume(path_vol_1, mask_vol, resolution, pixel_size)
if path_vol_2:
vol2 = prepare_volume(path_vol_2, mask_vol, resolution, pixel_size)
if mask_vol:
mask_vol = sp_projection.prep_vol(mask_vol, interpolation_method=1)
return vol1, vol2, mask_vol
def resample_prepare(prjfile, nvol, snr, CTF, npad):
from sp_utilities import get_im
nx = get_im(prjfile, 0).get_xsize()
fftvols = [None] * nvol
wgtvols = [None] * nvol
rectors = [None] * nvol
for i in range(nvol):
fftvols[i] = EMData()
wgtvols[i] = EMData()
if CTF:
params = {
"size": nx,
"npad": npad,
"snr": snr,
"weight": wgtvols[i],
"fftvol": fftvols[i]
}
rectors[i] = Reconstructors.get("nn4_ctf", params)
else:
params = {
"size": nx,
"npad": npad,
"snr": snr,
"weight": wgtvols[i],
"fftvol": fftvols[i]
}
rectors[i] = Reconstructors.get("nn4", params)
rectors[i].setup()
return rectors, fftvols, wgtvols
def test_(self):
testargs_new = [
path.join(ABSOLUTE_BIN_PATH, "sp_mask.py"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"09_ADJUSTMENT", "vol3d_ref_moon_eliminated.hdf"),
"--pixel_size=1.14", self.new_output_folder, "--mol_mass=1400",
"--edge_width=10"
]
testargs_old = [
path.join(ABSOLUTE_OLDBIN_PATH, "sp_mask.py"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"09_ADJUSTMENT", "vol3d_ref_moon_eliminated.hdf"),
"--pixel_size=1.14", self.old_output_folder, "--mol_mass=1400",
"--edge_width=10"
]
with patch.object(sys, 'argv', testargs_new):
fu.main()
with patch.object(sys, 'argv', testargs_old):
oldfu.main()
old_value = get_im(path.join(self.old_output_folder, self.filename))
new_value = get_im(path.join(self.new_output_folder, self.filename))
self.assertTrue(
array_equal(old_value.get_3dview(), new_value.get_3dview()))
self.assertTrue(
allclose(
new_value.get_3dview().flatten().tolist()[3022799:3022899], [
0.00024921807926148176, 0.0005635463166981936,
0.0005635463166981936, 0.00024921807926148176, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
]))
def genbuf(prjfile, bufprefix, beg, end, CTF, npad, verbose=0):
if (verbose == 1):
finfo = open(os.path.join(outdir, "progress.txt"), "w")
else:
finfo = None
start_time = time()
istore = newfile_store(bufprefix, npad, CTF)
for i in range(beg, end):
prj = get_im(prjfile, i)
istore.add_image(prj, prj.get_attr("xform.projection"))
if (not (finfo is None) and ((i % 100 == 99 or i == end - 1))):
finfo.write("%6d buffered, time: %10.3f\n" %
(i + 1, time() - start_time))
finfo.flush()
def cml_export_struc(stack, outdir, irun, Ori):
from sp_projection import plot_angles
from sp_utilities import set_params_proj, get_im
global g_n_prj
pagls = []
for i in range(g_n_prj):
data = get_im(stack, i)
p = [Ori[4 * i], Ori[4 * i + 1], Ori[4 * i + 2], 0.0, 0.0]
set_params_proj(data, p)
data.write_image(outdir + '/structure_%03i.hdf' % irun, i)
# prepare angles to plot
pagls.append([Ori[4 * i], Ori[4 * i + 1], Ori[4 * i + 2]])
# plot angles
im = plot_angles(pagls)
im.write_image(outdir + '/plot_agls_%03i.hdf' % irun)
def do_volume_mask(ref_data):
"""
1. - volume
2. - Tracker, see meridien
3. - current iteration number
"""
# Retrieve the function specific input arguments from ref_data
vol = ref_data[0]
Tracker = ref_data[1]
mainiteration = ref_data[2]
if Tracker["constants"]["mask3D"] is None:
vol = sp_morphology.cosinemask(vol,
radius=Tracker["constants"]["radius"])
else:
EMAN2_cppwrap.Util.mul_img(
vol, sp_utilities.get_im(Tracker["constants"]["mask3D"]))
return vol
def prepare_volume(volume_path, mask=None, resolution=None, pixel_size=None):
"""
Prepares the volume for reprojections
:param volume_path: Path to volume file
:param mask: Particle mask
:param resolution: Resolution of the current reconstruction.
:return:
"""
vol = sp_utilities.get_im(volume_path)
if resolution:
vol = vol.process(
"filter.lowpass.gauss",
{
"cutoff_freq": old_div(1.0, resolution),
"apix": pixel_size
},
)
if mask:
vol = vol * mask
vol = sp_projection.prep_vol(vol, interpolation_method=1)
return vol
def generate_helimic(refvol,
outdir,
pixel,
CTF=False,
Cs=2.0,
voltage=200.0,
ampcont=10.0,
nonoise=False,
rand_seed=14567):
from sp_utilities import model_blank, model_gauss, model_gauss_noise, pad, get_im
from random import random
from sp_projection import prgs, prep_vol
from sp_filter import filt_gaussl, filt_ctf
from EMAN2 import EMAN2Ctf
if os.path.exists(outdir):
ERROR(
"Output directory exists, please change the name and restart the program"
)
return
os.mkdir(outdir)
seed(rand_seed)
Util.set_randnum_seed(rand_seed)
angles = []
for i in range(3):
angles.append([0.0 + 60.0 * i, 90.0 - i * 5, 0.0, 0.0, 0.0])
nangle = len(angles)
volfts = get_im(refvol)
nx = volfts.get_xsize()
ny = volfts.get_ysize()
nz = volfts.get_zsize()
volfts, kbx, kby, kbz = prep_vol(volfts)
iprj = 0
width = 500
xstart = 0
ystart = 0
for idef in range(3, 6):
mic = model_blank(2048, 2048)
#defocus = idef*0.2
defocus = idef * 0.6 ##@ming
if CTF:
#ctf = EMAN2Ctf()
#ctf.from_dict( {"defocus":defocus, "cs":Cs, "voltage":voltage, "apix":pixel, "ampcont":ampcont, "bfactor":0.0} )
from sp_utilities import generate_ctf
ctf = generate_ctf(
[defocus, 2, 200, 1.84, 0.0, ampcont, defocus * 0.2, 80]
) ##@ming the range of astigmatism amplitude is between 10 percent and 22 percent. 20 percent is a good choice.
i = idef - 4
for k in range(1):
psi = 90 + 10 * i
proj = prgs(
volfts, kbz,
[angles[idef - 3][0], angles[idef - 3][1], psi, 0.0, 0.0], kbx,
kby)
proj = Util.window(proj, 320, nz)
mic += pad(proj, 2048, 2048, 1, 0.0, 750 * i, 20 * i, 0)
if not nonoise: mic += model_gauss_noise(30.0, 2048, 2048)
if CTF:
#apply CTF
mic = filt_ctf(mic, ctf)
if not nonoise:
mic += filt_gaussl(model_gauss_noise(17.5, 2048, 2048), 0.3)
mic.write_image("%s/mic%1d.hdf" % (outdir, idef - 3), 0)
def mode_meridien(reconfile,
classavgstack,
classdocs,
partangles,
selectdoc,
maxshift,
outerrad,
outanglesdoc,
outaligndoc,
interpolation_method=1,
outliers=None,
goodclassparttemplate=None,
alignopt='apsh',
ringstep=1,
log=None,
verbose=False):
# Resample reference
recondata = EMAN2.EMData(reconfile)
idim = recondata['nx']
reconprep = prep_vol(recondata,
npad=2,
interpolation_method=interpolation_method)
# Initialize output angles
outangleslist = []
outalignlist = []
# Read class lists
classdoclist = glob.glob(classdocs)
partangleslist = read_text_row(partangles)
# Loop through class lists
for classdoc in classdoclist: # [classdoclist[32]]: #
# Strip out three-digit filenumber
classexample = os.path.splitext(classdoc)
classnum = int(classexample[0][-3:])
# Initial average
[avg_phi_init, avg_theta_init] = average_angles(partangleslist,
classdoc,
selectdoc=selectdoc)
# Look for outliers
if outliers:
[avg_phi_final, avg_theta_final] = average_angles(
partangleslist,
classdoc,
selectdoc=selectdoc,
init_angles=[avg_phi_init, avg_theta_init],
threshold=outliers,
goodpartdoc=goodclassparttemplate.format(classnum),
log=log,
verbose=verbose)
else:
[avg_phi_final, avg_theta_final] = [avg_phi_init, avg_theta_init]
# Compute re-projection
refprjreal = prgl(reconprep, [avg_phi_final, avg_theta_final, 0, 0, 0],
interpolation_method=1,
return_real=True)
# Align to class average
classavg = get_im(classavgstack, classnum)
# Alignment using self-correlation function
if alignopt == 'scf':
ang_align2d, sxs, sys, mirrorflag, peak = align2d_scf(classavg,
refprjreal,
maxshift,
maxshift,
ou=outerrad)
# Weird results
elif alignopt == 'align2d':
# Set search range
currshift = 0
txrng = tyrng = search_range(idim, outerrad, currshift, maxshift)
# Perform alignment
ang_align2d, sxs, sys, mirrorflag, peak = align2d(
classavg, refprjreal, txrng, tyrng, last_ring=outerrad)
# Direct3 (angles seemed to be quantized)
elif alignopt == 'direct3':
[[ang_align2d, sxs, sys, mirrorflag,
peak]] = align2d_direct3([classavg],
refprjreal,
maxshift,
maxshift,
ou=outerrad)
# APSH-like alignment (default)
else:
[[ang_align2d, sxs, sys, mirrorflag,
scale]] = apsh(refprjreal,
classavg,
outerradius=outerrad,
maxshift=maxshift,
ringstep=ringstep)
outalignlist.append([ang_align2d, sxs, sys, mirrorflag, 1])
msg = "Particle list %s: ang_align2d=%s sx=%s sy=%s mirror=%s\n" % (
classdoc, ang_align2d, sxs, sys, mirrorflag)
print_log_msg(msg, log, verbose)
# Check for mirroring
if mirrorflag == 1:
tempeulers = list(
compose_transform3(avg_phi_final, avg_theta_final, 0, 0, 0, 0,
1, 0, 180, 0, 0, 0, 0, 1))
combinedparams = list(
compose_transform3(tempeulers[0], tempeulers[1], tempeulers[2],
tempeulers[3], tempeulers[4], 0, 1, 0, 0,
-ang_align2d, 0, 0, 0, 1))
else:
combinedparams = list(
compose_transform3(avg_phi_final, avg_theta_final, 0, 0, 0, 0,
1, 0, 0, -ang_align2d, 0, 0, 0, 1))
# compose_transform3: returns phi,theta,psi, tx,ty,tz, scale
outangleslist.append(combinedparams)
# End class-loop
write_text_row(outangleslist, outanglesdoc)
write_text_row(outalignlist, outaligndoc)
print_log_msg(
'Wrote alignment parameters to %s and %s\n' %
(outanglesdoc, outaligndoc), log, verbose)
del recondata # Clean up
def compare_projs(reconfile,
classavgstack,
inputanglesdoc,
outdir,
interpolation_method=1,
log=None,
verbose=False):
"""
Make comparison stack between class averages (even-numbered (starts from 0)) and re-projections (odd-numbered).
Arguments:
reconfile : Input volume from which to generate re-projections
classavgstack ; Input image stack
inputanglesdoc : Input Euler angles doc
outdir ; Output directory
interpolation_method : Interpolation method: nearest neighbor (nn, 0), trilinear (1, default), gridding (-1)
log : Logger object
verbose : (boolean) Whether to write additional information to screen
Returns:
compstack : Stack of comparisons between input image stack (even-numbered (starts from 0)) and input volume (odd-numbered)
"""
recondata = EMAN2.EMData(reconfile)
nx = recondata.get_xsize()
# Resample reference
reconprep = prep_vol(recondata,
npad=2,
interpolation_method=interpolation_method)
ccclist = []
# Here you need actual radius to compute proper ccc's, but if you do, you have to deal with translations, PAP
mask = model_circle(nx // 2 - 2, nx, nx)
mask.write_image(os.path.join(outdir, 'maskalign.hdf'))
compstack = os.path.join(outdir, 'comp-proj-reproj.hdf')
# Number of images may have changed
nimg1 = EMAN2.EMUtil.get_image_count(classavgstack)
angleslist = read_text_row(inputanglesdoc)
for imgnum in range(nimg1):
# Get class average
classimg = get_im(classavgstack, imgnum)
# Compute re-projection
prjimg = prgl(reconprep,
angleslist[imgnum],
interpolation_method=1,
return_real=False)
# Calculate 1D power spectra
rops_dst = rops_table(classimg * mask)
rops_src = rops_table(prjimg)
# Set power spectrum of reprojection to the data.
# Since data has an envelope, it would make more sense to set data to reconstruction,
# but to do it one would have to know the actual resolution of the data.
# you can check sxprocess.py --adjpw to see how this is done properly PAP
table = [0.0] * len(rops_dst) # initialize table
for j in range(len(rops_dst)):
table[j] = sqrt(rops_dst[j] / rops_src[j])
prjimg = fft(filt_table(
prjimg,
table)) # match FFT amplitudes of re-projection and class average
cccoeff = ccc(prjimg, classimg, mask)
#print imgnum, cccoeff
classimg.set_attr_dict({'cross-corr': cccoeff})
prjimg.set_attr_dict({'cross-corr': cccoeff})
montagestack = []
montagestack.append(prjimg)
montagestack.append(classimg)
comparison_pair = montage2(montagestack, ncol=2, marginwidth=1)
comparison_pair.write_image(compstack, imgnum)
ccclist.append(cccoeff)
del angleslist
meanccc = sum(ccclist) / nimg1
print_log_msg("Average CCC is %s\n" % meanccc, log, verbose)
nimg2 = EMAN2.EMUtil.get_image_count(compstack)
for imgnum in range(nimg2): # xrange will be deprecated in Python3
prjimg = get_im(compstack, imgnum)
meanccc1 = prjimg.get_attr_default('mean-cross-corr', -1.0)
prjimg.set_attr_dict({'mean-cross-corr': meanccc})
write_header(compstack, prjimg, imgnum)
return compstack
def test_(self):
testargs_new = [
path.join(ABSOLUTE_BIN_PATH, "localRes.py"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"11_MERIDIEN", "vol_0_unfil_028.hdf"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"11_MERIDIEN", "vol_1_unfil_028.hdf"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"12_POSTREFINER", "vol_adaptive_mask.hdf"),
self.out_dir_new, "--out_ang_res", "--apix=1.14"
]
testargs_old = [
path.join(ABSOLUTE_OLDBIN_PATH, "sp_process.py"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"11_MERIDIEN", "vol_0_unfil_028.hdf"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"11_MERIDIEN", "vol_1_unfil_028.hdf"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"12_POSTREFINER", "vol_adaptive_mask.hdf"),
self.out_dir_old, "--out_ang_res", "--apix=1.14"
]
with patch.object(sys, 'argv', testargs_new):
fu.main()
with patch.object(sys, 'argv', testargs_old):
oldfu.main()
old_localRes_ang = get_im(path.join(self.out_dir_old, self.fname_ang))
old_localRes = get_im(path.join(self.out_dir_old, self.fname))
new_localRes_ang = get_im(path.join(self.out_dir_new, self.fname_ang))
new_localRes = get_im(path.join(self.out_dir_new, self.fname))
self.assertTrue(
allclose(old_localRes_ang.get_3dview(),
new_localRes_ang.get_3dview(),
atol=0.05))
self.assertTrue(
allclose(old_localRes.get_3dview(),
new_localRes.get_3dview(),
atol=0.05))
self.assertTrue(
allclose(
new_localRes_ang.get_3dview().flatten().tolist()
[4034427:4034527], [
6.859487056732178, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
],
atol=0.05))
self.assertTrue(
allclose(
new_localRes.get_3dview().flatten().tolist()[4034427:4034527],
[
0.1661931872367859, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
],
atol=0.05))
def test_symmetrize(self):
old_output_folder = "3DvariabilitySimmetry_sym_Old"
new_output_folder = "3DvariabilitySimmetry_sym_New"
remove_dir(new_output_folder)
remove_dir(old_output_folder)
filename = "sdata"
testargs_new = (path.join(ABSOLUTE_BIN_PATH, "sp_3dvariability.py") +
" " + "--symmetrize" + " " + "'bdb:" + path.join(
ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"06_SUBSTACK_ANO#isac_substack'") + " " +
"--output_dir=" + new_output_folder + " " + "--sym=c5")
testargs_old = (
path.join(ABSOLUTE_OLDBIN_PATH, "sp_3dvariability.py") + " " +
"--symmetrize" + " " + "'bdb:" +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"06_SUBSTACK_ANO#isac_substack'") + " " +
"--output_dir=" + old_output_folder + " " + "--sym=c5")
subprocess.run(args=[testargs_new], shell=True, capture_output=True)
subprocess.run(args=[testargs_old], shell=True, capture_output=True)
return_old = get_im('bdb:{0}#{1}'.format(old_output_folder, filename))
return_new = get_im('bdb:{0}#{1}'.format(new_output_folder, filename))
self.assertTrue(
allclose(return_new.get_2dview(),
return_old.get_2dview(),
atol=0.1))
self.assertTrue(
allclose(return_new.get_2dview().flatten()[0:100], [
-0.38441434502601624, -0.4573197066783905,
-0.39960816502571106, -0.10431570559740067,
0.018583765253424644, 0.37694355845451355, 0.4353419840335846,
0.06498070061206818, -0.38467326760292053, -0.7100721001625061,
-0.7571740746498108, -0.9328497648239136, -0.8550546169281006,
-0.21433517336845398, 0.34148913621902466, 0.30916473269462585,
0.39394184947013855, 0.4447155296802521, -0.032833874225616455,
0.4121330976486206, 0.638403594493866, 0.34945985674858093,
0.2751978039741516, -0.14872148633003235, -0.5307353138923645,
-0.6574574112892151, -0.8945914506912231, -1.107301950454712,
-1.3676011562347412, -1.2225056886672974, -1.2273787260055542,
-0.7121877074241638, 0.06733409315347672, 0.29879996180534363,
0.6046097874641418, 0.496036559343338, 0.7214235663414001,
0.7652679681777954, 0.9319247603416443, 0.11259084939956665,
-0.3054805397987366, -0.8183746933937073, -1.4462037086486816,
-1.4775571823120117, -1.2116808891296387, -0.911469042301178,
-0.9934141039848328, -0.9891195297241211, -1.0073580741882324,
-1.234678864479065, -1.4292954206466675, -1.5580313205718994,
-1.446936845779419, -1.67134428024292, -2.0199873447418213,
-2.0543317794799805, -2.2354137897491455, -1.99867844581604,
-1.4405670166015625, -1.7088592052459717, -2.060110569000244,
-1.6585056781768799, -1.0386717319488525, -0.4783007502555847,
-0.12102964520454407, 0.16874085366725922,
-0.12371158599853516, 0.01841016113758087, -0.6442297101020813,
-0.6972493529319763, -0.1051267459988594, 0.016293810680508614,
-0.10689342767000198, -0.472159206867218, -0.4886413514614105,
-0.09828135371208191, -0.7409976720809937, -0.8893253207206726,
-1.1477444171905518, -0.35942375659942627,
0.030692437663674355, 0.5380961298942566, 1.06252121925354,
0.6961002349853516, 0.7563707232475281, 1.0197871923446655,
0.9021824598312378, 0.34891727566719055, 0.402149498462677,
0.634938657283783, 0.6044892072677612, 0.24482420086860657,
-0.4132026731967926, -0.9945327043533325, -1.260424256324768,
-1.6913681030273438, -1.7999876737594604, -1.5891680717468262,
-1.3196191787719727, -0.9893324375152588
],
atol=0.1))
remove_dir(new_output_folder)
remove_dir(old_output_folder)
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
# the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D
if mirror:
m = 1
alpha, sx, sy, scalen = sp_utilities.compose_transform2(
0, s2x, s2y, 1.0, 540.0 - psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = sp_utilities.compose_transform2(
0, s2x, s2y, 1.0, 360.0 - psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = optparse.os.path.basename(sys.argv[0])
usage = (
progname +
" prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl= --aa= --sym=symmetry --CTF"
)
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option("--output_dir",
type="string",
default="./",
help="Output directory")
parser.add_option(
"--ave2D",
type="string",
default=False,
help="Write to the disk a stack of 2D averages",
)
parser.add_option(
"--var2D",
type="string",
default=False,
help="Write to the disk a stack of 2D variances",
)
parser.add_option(
"--ave3D",
type="string",
default=False,
help="Write to the disk reconstructed 3D average",
)
parser.add_option(
"--var3D",
type="string",
default=False,
help="Compute 3D variability (time consuming!)",
)
parser.add_option(
"--img_per_grp",
type="int",
default=100,
help="Number of neighbouring projections.(Default is 100)",
)
parser.add_option(
"--no_norm",
action="store_true",
default=False,
help="Do not use normalization.(Default is to apply normalization)",
)
# parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option(
"--npad",
type="int",
default=2,
help=
"Number of time to pad the original images.(Default is 2 times padding)",
)
parser.add_option("--sym",
type="string",
default="c1",
help="Symmetry. (Default is no symmetry)")
parser.add_option(
"--fl",
type="float",
default=0.0,
help=
"Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)",
)
parser.add_option(
"--aa",
type="float",
default=0.02,
help=
"Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)",
)
parser.add_option(
"--CTF",
action="store_true",
default=False,
help="Use CFT correction.(Default is no CTF correction)",
)
# parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
# parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
# parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
# parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
# parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option(
"--VAR",
action="store_true",
default=False,
help="Stack of input consists of 2D variances (Default False)",
)
parser.add_option(
"--decimate",
type="float",
default=0.25,
help="Image decimate rate, a number less than 1. (Default is 0.25)",
)
parser.add_option(
"--window",
type="int",
default=0,
help=
"Target image size relative to original image size. (Default value is zero.)",
)
# parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
# parser.add_option("--nvec", type="int" , default=0 , help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)")
parser.add_option(
"--symmetrize",
action="store_true",
default=False,
help="Prepare input stack for handling symmetry (Default False)",
)
parser.add_option("--overhead",
type="float",
default=0.5,
help="python overhead per CPU.")
(options, args) = parser.parse_args()
#####
# from mpi import *
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
# Set up global variables related to bdb cache
if sp_global_def.CACHE_DISABLE:
sp_utilities.disable_bdb_cache()
# Set up global variables related to ERROR function
sp_global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in optparse.os.environ
if RUNNING_UNDER_MPI:
sp_global_def.MPI = True
if options.output_dir == "./":
current_output_dir = optparse.os.path.abspath(options.output_dir)
else:
current_output_dir = options.output_dir
if options.symmetrize:
if mpi.mpi_comm_size(mpi.MPI_COMM_WORLD) > 1:
sp_global_def.ERROR(
"Cannot use more than one CPU for symmetry preparation")
if not optparse.os.path.exists(current_output_dir):
optparse.os.makedirs(current_output_dir)
sp_global_def.write_command(current_output_dir)
if optparse.os.path.exists(
optparse.os.path.join(current_output_dir, "log.txt")):
optparse.os.remove(
optparse.os.path.join(current_output_dir, "log.txt"))
log_main = sp_logger.Logger(sp_logger.BaseLogger_Files())
log_main.prefix = optparse.os.path.join(current_output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if sym == "c1":
sp_global_def.ERROR(
"There is no need to symmetrize stack for C1 symmetry")
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
if instack[:4] != "bdb:":
# if output_dir =="./": stack = "bdb:data"
stack = "bdb:" + current_output_dir + "/data"
sp_utilities.delete_bdb(stack)
junk = sp_utilities.cmdexecute("sp_cpy.py " + instack + " " +
stack)
else:
stack = instack
qt = EMAN2_cppwrap.EMUtil.get_all_attributes(stack, "xform.projection")
na = len(qt)
ts = sp_utilities.get_symt(sym)
ks = len(ts)
angsa = [None] * na
for k in range(ks):
# Qfile = "Q%1d"%k
# if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k)
Qfile = optparse.os.path.join(current_output_dir, "Q%1d" % k)
# delete_bdb("bdb:Q%1d"%k)
sp_utilities.delete_bdb("bdb:" + Qfile)
# junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = sp_utilities.cmdexecute("e2bdb.py " + stack +
" --makevstack=bdb:" + Qfile)
# DB = db_open_dict("bdb:Q%1d"%k)
DB = EMAN2db.db_open_dict("bdb:" + Qfile)
for i in range(na):
ut = qt[i] * ts[k]
DB.set_attr(i, "xform.projection", ut)
# bt = ut.get_params("spider")
# angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
# write_text_row(angsa, 'ptsma%1d.txt'%k)
# junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
# junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
# if options.output_dir =="./": delete_bdb("bdb:sdata")
sp_utilities.delete_bdb("bdb:" + current_output_dir + "/" + "sdata")
# junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:" + current_output_dir + "/" + "sdata"
sp_global_def.sxprint(sdata)
junk = sp_utilities.cmdexecute("e2bdb.py " + current_output_dir +
" --makevstack=" + sdata + " --filt=Q")
# junk = cmdexecute("ls EMAN2DB/sdata*")
# a = get_im("bdb:sdata")
a = sp_utilities.get_im(sdata)
a.set_attr("variabilitysymmetry", sym)
# a.write_image("bdb:sdata")
a.write_image(sdata)
else:
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
main_node = 0
shared_comm = mpi.mpi_comm_split_type(mpi.MPI_COMM_WORLD,
mpi.MPI_COMM_TYPE_SHARED, 0,
mpi.MPI_INFO_NULL)
myid_on_node = mpi.mpi_comm_rank(shared_comm)
no_of_processes_per_group = mpi.mpi_comm_size(shared_comm)
masters_from_groups_vs_everything_else_comm = mpi.mpi_comm_split(
mpi.MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node)
color, no_of_groups, balanced_processor_load_on_nodes = sp_utilities.get_colors_and_subsets(
main_node,
mpi.MPI_COMM_WORLD,
myid,
shared_comm,
myid_on_node,
masters_from_groups_vs_everything_else_comm,
)
overhead_loading = options.overhead * number_of_proc
# memory_per_node = options.memory_per_node
# if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group
keepgoing = 1
current_window = options.window
current_decimate = options.decimate
if len(args) == 1:
stack = args[0]
else:
sp_global_def.sxprint("Usage: " + usage)
sp_global_def.sxprint("Please run '" + progname +
" -h' for detailed options")
sp_global_def.ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
t0 = time.time()
# obsolete flags
options.MPI = True
# options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
sp_global_def.ERROR(
"Fall off has to be given for the low-pass filter", myid=myid)
# if options.VAR and options.SND:
# ERROR( "Only one of var and SND can be set!",myid=myid )
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
sp_global_def.ERROR(
"When VAR is set, the program cannot output ave2D, ave3D or var2D",
myid=myid,
)
# if options.SND and (options.ave2D or options.ave3D):
# ERROR( "When SND is set, the program cannot output ave2D or ave3D", myid=myid )
# if options.nvec > 0 :
# ERROR( "PCA option not implemented", myid=myid )
# if options.nvec > 0 and options.ave3D == None:
# ERROR( "When doing PCA analysis, one must set ave3D", myid=myid )
if current_decimate > 1.0 or current_decimate < 0.0:
sp_global_def.ERROR(
"Decimate rate should be a value between 0.0 and 1.0",
myid=myid)
if current_window < 0.0:
sp_global_def.ERROR(
"Target window size should be always larger than zero",
myid=myid)
if myid == main_node:
img = sp_utilities.get_image(stack, 0)
nx = img.get_xsize()
ny = img.get_ysize()
if min(nx, ny) < current_window:
keepgoing = 0
keepgoing = sp_utilities.bcast_number_to_all(keepgoing, main_node,
mpi.MPI_COMM_WORLD)
if keepgoing == 0:
sp_global_def.ERROR(
"The target window size cannot be larger than the size of decimated image",
myid=myid,
)
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if not optparse.os.path.exists(current_output_dir):
optparse.os.makedirs(
current_output_dir
) # Never delete output_dir in the program!
img_per_grp = options.img_per_grp
# nvec = options.nvec
radiuspca = options.radiuspca
# if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = sp_logger.Logger(sp_logger.BaseLogger_Files())
log_main.prefix = optparse.os.path.join(current_output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("Symmetry : %s" % options.sym)
log_main.add("Input stack : %s" % stack)
log_main.add("Output_dir : %s" % current_output_dir)
if options.ave3D:
log_main.add("Ave3d : %s" % options.ave3D)
if options.var3D:
log_main.add("Var3d : %s" % options.var3D)
if options.ave2D:
log_main.add("Ave2D : %s" % options.ave2D)
if options.var2D:
log_main.add("Var2D : %s" % options.var2D)
if options.VAR:
log_main.add("VAR : True")
else:
log_main.add("VAR : False")
if options.CTF:
log_main.add("CTF correction : True ")
else:
log_main.add("CTF correction : False ")
log_main.add("Image per group : %5d" % options.img_per_grp)
log_main.add("Image decimate rate : %4.3f" % current_decimate)
log_main.add("Low pass filter : %4.3f" % options.fl)
current_fl = options.fl
if current_fl == 0.0:
current_fl = 0.5
log_main.add(
"Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"
% round((current_fl * current_decimate), 3))
log_main.add("Window size : %5d " % current_window)
log_main.add("sx3dvariability begins")
symbaselen = 0
if myid == main_node:
nima = EMAN2_cppwrap.EMUtil.get_image_count(stack)
img = sp_utilities.get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
nnxo = nx
nnyo = ny
if options.sym != "c1":
imgdata = sp_utilities.get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if i != options.sym:
sp_global_def.ERROR(
"The symmetry provided does not agree with the symmetry of the input stack",
myid=myid,
)
except:
sp_global_def.ERROR(
"Input stack is not prepared for symmetry, please follow instructions",
myid=myid,
)
i = len(sp_utilities.get_symt(options.sym))
if (old_div(nima, i)) * i != nima:
sp_global_def.ERROR(
"The length of the input stack is incorrect for symmetry processing",
myid=myid,
)
symbaselen = old_div(nima, i)
else:
symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nnxo = 0
nnyo = 0
nima = sp_utilities.bcast_number_to_all(nima)
nx = sp_utilities.bcast_number_to_all(nx)
ny = sp_utilities.bcast_number_to_all(ny)
nnxo = sp_utilities.bcast_number_to_all(nnxo)
nnyo = sp_utilities.bcast_number_to_all(nnyo)
if current_window > max(nx, ny):
sp_global_def.ERROR(
"Window size is larger than the original image size")
if current_decimate == 1.0:
if current_window != 0:
nx = current_window
ny = current_window
else:
if current_window == 0:
nx = int(nx * current_decimate + 0.5)
ny = int(ny * current_decimate + 0.5)
else:
nx = int(current_window * current_decimate + 0.5)
ny = nx
symbaselen = sp_utilities.bcast_number_to_all(symbaselen)
# check FFT prime number
is_fft_friendly = nx == sp_fundamentals.smallprime(nx)
if not is_fft_friendly:
if myid == main_node:
log_main.add(
"The target image size is not a product of small prime numbers"
)
log_main.add("Program adjusts the input settings!")
### two cases
if current_decimate == 1.0:
nx = sp_fundamentals.smallprime(nx)
ny = nx
current_window = nx # update
if myid == main_node:
log_main.add("The window size is updated to %d." %
current_window)
else:
if current_window == 0:
nx = sp_fundamentals.smallprime(
int(nx * current_decimate + 0.5))
current_decimate = old_div(float(nx), nnxo)
ny = nx
if myid == main_node:
log_main.add("The decimate rate is updated to %f." %
current_decimate)
else:
nx = sp_fundamentals.smallprime(
int(current_window * current_decimate + 0.5))
ny = nx
current_window = int(old_div(nx, current_decimate) + 0.5)
if myid == main_node:
log_main.add("The window size is updated to %d." %
current_window)
if myid == main_node:
log_main.add("The target image size is %d" % nx)
if radiuspca == -1:
radiuspca = old_div(nx, 2) - 2
if myid == main_node:
log_main.add("%-70s: %d\n" % ("Number of projection", nima))
img_begin, img_end = sp_applications.MPI_start_end(
nima, number_of_proc, myid)
"""Multiline Comment0"""
"""
Comments from adnan, replace index_of_proj to index_of_particle, index_of_proj was not defined
also varList is not defined not made an empty list there
"""
if options.VAR: # 2D variance images have no shifts
varList = []
# varList = EMData.read_images(stack, range(img_begin, img_end))
for index_of_particle in range(img_begin, img_end):
image = sp_utilities.get_im(stack, index_of_particle)
if current_window > 0:
varList.append(
sp_fundamentals.fdecimate(
sp_fundamentals.window2d(image, current_window,
current_window),
nx,
ny,
))
else:
varList.append(sp_fundamentals.fdecimate(image, nx, ny))
else:
if myid == main_node:
t1 = time.time()
proj_angles = []
aveList = []
tab = EMAN2_cppwrap.EMUtil.get_all_attributes(
stack, "xform.projection")
for i in range(nima):
t = tab[i].get_params("spider")
phi = t["phi"]
theta = t["theta"]
psi = t["psi"]
x = theta
if x > 90.0:
x = 180.0 - x
x = x * 10000 + psi
proj_angles.append([x, t["phi"], t["theta"], t["psi"], i])
t2 = time.time()
log_main.add(
"%-70s: %d\n" %
("Number of neighboring projections", img_per_grp))
log_main.add("...... Finding neighboring projections\n")
log_main.add("Number of images per group: %d" % img_per_grp)
log_main.add("Now grouping projections")
proj_angles.sort()
proj_angles_list = numpy.full((nima, 4),
0.0,
dtype=numpy.float32)
for i in range(nima):
proj_angles_list[i][0] = proj_angles[i][1]
proj_angles_list[i][1] = proj_angles[i][2]
proj_angles_list[i][2] = proj_angles[i][3]
proj_angles_list[i][3] = proj_angles[i][4]
else:
proj_angles_list = 0
proj_angles_list = sp_utilities.wrap_mpi_bcast(
proj_angles_list, main_node, mpi.MPI_COMM_WORLD)
proj_angles = []
for i in range(nima):
proj_angles.append([
proj_angles_list[i][0],
proj_angles_list[i][1],
proj_angles_list[i][2],
int(proj_angles_list[i][3]),
])
del proj_angles_list
proj_list, mirror_list = sp_utilities.nearest_proj(
proj_angles, img_per_grp, range(img_begin, img_end))
all_proj = []
for im in proj_list:
for jm in im:
all_proj.append(proj_angles[jm][3])
all_proj = list(set(all_proj))
index = {}
for i in range(len(all_proj)):
index[all_proj[i]] = i
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == main_node:
log_main.add("%-70s: %.2f\n" %
("Finding neighboring projections lasted [s]",
time.time() - t2))
log_main.add("%-70s: %d\n" %
("Number of groups processed on the main node",
len(proj_list)))
log_main.add("Grouping projections took: %12.1f [m]" %
(old_div((time.time() - t2), 60.0)))
log_main.add("Number of groups on main node: ", len(proj_list))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == main_node:
log_main.add("...... Calculating the stack of 2D variances \n")
# Memory estimation. There are two memory consumption peaks
# peak 1. Compute ave, var;
# peak 2. Var volume reconstruction;
# proj_params = [0.0]*(nima*5)
aveList = []
varList = []
# if nvec > 0: eigList = [[] for i in range(nvec)]
dnumber = len(
all_proj) # all neighborhood set for assigned to myid
pnumber = len(proj_list) * 2.0 + img_per_grp # aveList and varList
tnumber = dnumber + pnumber
vol_size2 = old_div(nx**3 * 4.0 * 8, 1.0e9)
vol_size1 = old_div(2.0 * nnxo**3 * 4.0 * 8, 1.0e9)
proj_size = old_div(nnxo * nnyo * len(proj_list) * 4.0 * 2.0,
1.0e9) # both aveList and varList
orig_data_size = old_div(nnxo * nnyo * 4.0 * tnumber, 1.0e9)
reduced_data_size = old_div(nx * nx * 4.0 * tnumber, 1.0e9)
full_data = numpy.full((number_of_proc, 2),
-1.0,
dtype=numpy.float16)
full_data[myid] = orig_data_size, reduced_data_size
if myid != main_node:
sp_utilities.wrap_mpi_send(full_data, main_node,
mpi.MPI_COMM_WORLD)
if myid == main_node:
for iproc in range(number_of_proc):
if iproc != main_node:
dummy = sp_utilities.wrap_mpi_recv(
iproc, mpi.MPI_COMM_WORLD)
full_data[numpy.where(dummy > -1)] = dummy[numpy.where(
dummy > -1)]
del dummy
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
full_data = sp_utilities.wrap_mpi_bcast(full_data, main_node,
mpi.MPI_COMM_WORLD)
# find the CPU with heaviest load
minindx = numpy.argsort(full_data, 0)
heavy_load_myid = minindx[-1][1]
total_mem = sum(full_data)
if myid == main_node:
if current_window == 0:
log_main.add(
"Nx: current image size = %d. Decimated by %f from %d"
% (nx, current_decimate, nnxo))
else:
log_main.add(
"Nx: current image size = %d. Windowed to %d, and decimated by %f from %d"
% (nx, current_window, current_decimate, nnxo))
log_main.add("Nproj: number of particle images.")
log_main.add("Navg: number of 2D average images.")
log_main.add("Nvar: number of 2D variance images.")
log_main.add(
"Img_per_grp: user defined image per group for averaging = %d"
% img_per_grp)
log_main.add(
"Overhead: total python overhead memory consumption = %f"
% overhead_loading)
log_main.add(
"Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"
% (total_mem[1] + overhead_loading))
del full_data
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add(
"Begin reading and preprocessing images on processor. Wait... "
)
ttt = time.time()
# imgdata = EMData.read_images(stack, all_proj)
imgdata = [None for im in range(len(all_proj))]
for index_of_proj in range(len(all_proj)):
# image = get_im(stack, all_proj[index_of_proj])
if current_window > 0:
imgdata[index_of_proj] = sp_fundamentals.fdecimate(
sp_fundamentals.window2d(
sp_utilities.get_im(stack,
all_proj[index_of_proj]),
current_window,
current_window,
),
nx,
ny,
)
else:
imgdata[index_of_proj] = sp_fundamentals.fdecimate(
sp_utilities.get_im(stack, all_proj[index_of_proj]),
nx, ny)
if current_decimate > 0.0 and options.CTF:
ctf = imgdata[index_of_proj].get_attr("ctf")
ctf.apix = old_div(ctf.apix, current_decimate)
imgdata[index_of_proj].set_attr("ctf", ctf)
if myid == heavy_load_myid and index_of_proj % 100 == 0:
log_main.add(
" ...... %6.2f%% " %
(old_div(index_of_proj, float(len(all_proj))) * 100.0))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("All_proj preprocessing cost %7.2f m" % (old_div(
(time.time() - ttt), 60.0)))
log_main.add("Wait untill reading on all CPUs done...")
"""Multiline Comment1"""
if not options.no_norm:
mask = sp_utilities.model_circle(old_div(nx, 2) - 2, nx, nx)
if myid == heavy_load_myid:
log_main.add("Start computing 2D aveList and varList. Wait...")
ttt = time.time()
inner = old_div(nx, 2) - 4
outer = inner + 2
xform_proj_for_2D = [None for i in range(len(proj_list))]
for i in range(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen:
continue
mi = index[ki]
dpar = EMAN2_cppwrap.Util.get_transform_params(
imgdata[mi], "xform.projection", "spider")
phiM, thetaM, psiM, s2xM, s2yM = (
dpar["phi"],
dpar["theta"],
dpar["psi"],
-dpar["tx"] * current_decimate,
-dpar["ty"] * current_decimate,
)
grp_imgdata = []
for j in range(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
cpar = EMAN2_cppwrap.Util.get_transform_params(
imgdata[mj], "xform.projection", "spider")
alpha, sx, sy, mirror = params_3D_2D_NEW(
cpar["phi"],
cpar["theta"],
cpar["psi"],
-cpar["tx"] * current_decimate,
-cpar["ty"] * current_decimate,
mirror_list[i][j],
)
if thetaM <= 90:
if mirror == 0:
alpha, sx, sy, scale = sp_utilities.compose_transform2(
alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0,
0.0, 1.0)
else:
alpha, sx, sy, scale = sp_utilities.compose_transform2(
alpha,
sx,
sy,
1.0,
180 - (phiM - cpar["phi"]),
0.0,
0.0,
1.0,
)
else:
if mirror == 0:
alpha, sx, sy, scale = sp_utilities.compose_transform2(
alpha, sx, sy, 1.0, -(phiM - cpar["phi"]), 0.0,
0.0, 1.0)
else:
alpha, sx, sy, scale = sp_utilities.compose_transform2(
alpha,
sx,
sy,
1.0,
-(180 - (phiM - cpar["phi"])),
0.0,
0.0,
1.0,
)
imgdata[mj].set_attr(
"xform.align2d",
EMAN2_cppwrap.Transform({
"type": "2D",
"alpha": alpha,
"tx": sx,
"ty": sy,
"mirror": mirror,
"scale": 1.0,
}),
)
grp_imgdata.append(imgdata[mj])
if not options.no_norm:
for k in range(img_per_grp):
ave, std, minn, maxx = EMAN2_cppwrap.Util.infomask(
grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] = old_div(grp_imgdata[k], std)
if options.fl > 0.0:
for k in range(img_per_grp):
grp_imgdata[k] = sp_filter.filt_tanl(
grp_imgdata[k], options.fl, options.aa)
# Because of background issues, only linear option works.
if options.CTF:
ave, var = sp_statistics.aves_wiener(
grp_imgdata, SNR=1.0e5, interpolation_method="linear")
else:
ave, var = sp_statistics.ave_var(grp_imgdata)
# Switch to std dev
# threshold is not really needed,it is just in case due to numerical accuracy something turns out negative.
var = sp_morphology.square_root(sp_morphology.threshold(var))
sp_utilities.set_params_proj(ave,
[phiM, thetaM, 0.0, 0.0, 0.0])
sp_utilities.set_params_proj(var,
[phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0]
"""Multiline Comment2"""
if (myid == heavy_load_myid) and (i % 100 == 0):
log_main.add(" ......%6.2f%% " %
(old_div(i, float(len(proj_list))) * 100.0))
del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index
if not options.no_norm:
del mask
if myid == main_node:
del tab
# At this point, all averages and variances are computed
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("Computing aveList and varList took %12.1f [m]" %
(old_div((time.time() - ttt), 60.0)))
xform_proj_for_2D = sp_utilities.wrap_mpi_gatherv(
xform_proj_for_2D, main_node, mpi.MPI_COMM_WORLD)
if myid == main_node:
sp_utilities.write_text_row(
[str(entry) for entry in xform_proj_for_2D],
optparse.os.path.join(current_output_dir, "params.txt"),
)
del xform_proj_for_2D
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if options.ave2D:
if myid == main_node:
log_main.add("Compute ave2D ... ")
km = 0
for i in range(number_of_proc):
if i == main_node:
for im in range(len(aveList)):
aveList[im].write_image(
optparse.os.path.join(
current_output_dir, options.ave2D),
km,
)
km += 1
else:
nl = mpi.mpi_recv(
1,
mpi.MPI_INT,
i,
sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
mpi.MPI_COMM_WORLD,
)
nl = int(nl[0])
for im in range(nl):
ave = sp_utilities.recv_EMData(
i, im + i + 70000)
"""Multiline Comment3"""
tmpvol = sp_fundamentals.fpol(ave, nx, nx, 1)
tmpvol.write_image(
optparse.os.path.join(
current_output_dir, options.ave2D),
km,
)
km += 1
else:
mpi.mpi_send(
len(aveList),
1,
mpi.MPI_INT,
main_node,
sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
mpi.MPI_COMM_WORLD,
)
for im in range(len(aveList)):
sp_utilities.send_EMData(aveList[im], main_node,
im + myid + 70000)
"""Multiline Comment4"""
if myid == main_node:
sp_applications.header(
optparse.os.path.join(current_output_dir,
options.ave2D),
params="xform.projection",
fimport=optparse.os.path.join(current_output_dir,
"params.txt"),
)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if options.ave3D:
t5 = time.time()
if myid == main_node:
log_main.add("Reconstruct ave3D ... ")
ave3D = sp_reconstruction.recons3d_4nn_MPI(
myid, aveList, symmetry=options.sym, npad=options.npad)
sp_utilities.bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
if current_decimate != 1.0:
ave3D = sp_fundamentals.resample(
ave3D, old_div(1.0, current_decimate))
ave3D = sp_fundamentals.fpol(
ave3D, nnxo, nnxo,
nnxo) # always to the orignal image size
sp_utilities.set_pixel_size(ave3D, 1.0)
ave3D.write_image(
optparse.os.path.join(current_output_dir,
options.ave3D))
log_main.add("Ave3D reconstruction took %12.1f [m]" %
(old_div((time.time() - t5), 60.0)))
log_main.add("%-70s: %s\n" %
("The reconstructed ave3D is saved as ",
options.ave3D))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList
"""Multiline Comment5"""
if options.ave3D:
del ave3D
if options.var2D:
if myid == main_node:
log_main.add("Compute var2D...")
km = 0
for i in range(number_of_proc):
if i == main_node:
for im in range(len(varList)):
tmpvol = sp_fundamentals.fpol(
varList[im], nx, nx, 1)
tmpvol.write_image(
optparse.os.path.join(
current_output_dir, options.var2D),
km,
)
km += 1
else:
nl = mpi.mpi_recv(
1,
mpi.MPI_INT,
i,
sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
mpi.MPI_COMM_WORLD,
)
nl = int(nl[0])
for im in range(nl):
ave = sp_utilities.recv_EMData(
i, im + i + 70000)
tmpvol = sp_fundamentals.fpol(ave, nx, nx, 1)
tmpvol.write_image(
optparse.os.path.join(
current_output_dir, options.var2D),
km,
)
km += 1
else:
mpi.mpi_send(
len(varList),
1,
mpi.MPI_INT,
main_node,
sp_global_def.SPARX_MPI_TAG_UNIVERSAL,
mpi.MPI_COMM_WORLD,
)
for im in range(len(varList)):
sp_utilities.send_EMData(
varList[im], main_node,
im + myid + 70000) # What with the attributes??
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == main_node:
sp_applications.header(
optparse.os.path.join(current_output_dir,
options.var2D),
params="xform.projection",
fimport=optparse.os.path.join(current_output_dir,
"params.txt"),
)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if options.var3D:
if myid == main_node:
log_main.add("Reconstruct var3D ...")
t6 = time.time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = sp_reconstruction.recons3d_4nn_MPI(myid,
varList,
symmetry=options.sym,
npad=options.npad)
# res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
if current_decimate != 1.0:
res = sp_fundamentals.resample(
res, old_div(1.0, current_decimate))
res = sp_fundamentals.fpol(res, nnxo, nnxo, nnxo)
sp_utilities.set_pixel_size(res, 1.0)
res.write_image(os.path.join(current_output_dir,
options.var3D))
log_main.add(
"%-70s: %s\n" %
("The reconstructed var3D is saved as ", options.var3D))
log_main.add("Var3D reconstruction took %f12.1 [m]" % (old_div(
(time.time() - t6), 60.0)))
log_main.add("Total computation time %f12.1 [m]" % (old_div(
(time.time() - t0), 60.0)))
log_main.add("sx3dvariability finishes")
if RUNNING_UNDER_MPI:
sp_global_def.MPI = False
sp_global_def.BATCH = False
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = optparse.os.path.basename(arglist[0])
usage = progname + """ inputvolume locresvolume maskfile outputfile --radius --falloff --MPI
Locally filer a volume based on local resolution volume (sxlocres.py) within area outlined by the maskfile
"""
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--radius",
type="int",
default=-1,
help=
"if there is no maskfile, sphere with r=radius will be used, by default the radius is nx/2-1"
)
parser.add_option("--falloff",
type="float",
default=0.1,
help="falloff of tanl filter (default 0.1)")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 3 or len(args) > 4:
sp_global_def.sxprint("See usage " + usage)
sp_global_def.ERROR(
"Wrong number of parameters. Please see usage information above.")
return
if sp_global_def.CACHE_DISABLE:
pass #IMPORTIMPORTIMPORT from sp_utilities import disable_bdb_cache
sp_utilities.disable_bdb_cache()
if options.MPI:
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
if (myid == main_node):
#print sys.argv
vi = sp_utilities.get_im(sys.argv[1])
ui = sp_utilities.get_im(sys.argv[2])
#print Util.infomask(ui, None, True)
radius = options.radius
nx = vi.get_xsize()
ny = vi.get_ysize()
nz = vi.get_zsize()
dis = [nx, ny, nz]
else:
falloff = 0.0
radius = 0
dis = [0, 0, 0]
vi = None
ui = None
dis = sp_utilities.bcast_list_to_all(dis, myid, source_node=main_node)
if (myid != main_node):
nx = int(dis[0])
ny = int(dis[1])
nz = int(dis[2])
radius = sp_utilities.bcast_number_to_all(radius, main_node)
if len(args) == 3:
if (radius == -1):
radius = min(nx, ny, nz) // 2 - 1
m = sp_utilities.model_circle(radius, nx, ny, nz)
outvol = args[2]
elif len(args) == 4:
if (myid == main_node):
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
else:
m = sp_utilities.model_blank(nx, ny, nz)
outvol = args[3]
sp_utilities.bcast_EMData_to_all(m, myid, main_node)
pass #IMPORTIMPORTIMPORT from sp_filter import filterlocal
filteredvol = sp_filter.filterlocal(ui, vi, m, options.falloff, myid,
main_node, number_of_proc)
if (myid == 0):
filteredvol.write_image(outvol)
else:
vi = sp_utilities.get_im(args[0])
ui = sp_utilities.get_im(
args[1]
) # resolution volume, values are assumed to be from 0 to 0.5
nn = vi.get_xsize()
falloff = options.falloff
if len(args) == 3:
radius = options.radius
if (radius == -1):
radius = nn // 2 - 1
m = sp_utilities.model_circle(radius, nn, nn, nn)
outvol = args[2]
elif len(args) == 4:
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
outvol = args[3]
sp_fundamentals.fftip(vi) # this is the volume to be filtered
# Round all resolution numbers to two digits
for x in range(nn):
for y in range(nn):
for z in range(nn):
ui.set_value_at_fast(x, y, z,
round(ui.get_value_at(x, y, z), 2))
st = EMAN2_cppwrap.Util.infomask(ui, m, True)
filteredvol = sp_utilities.model_blank(nn, nn, nn)
cutoff = max(st[2] - 0.01, 0.0)
while (cutoff < st[3]):
cutoff = round(cutoff + 0.01, 2)
pt = EMAN2_cppwrap.Util.infomask(
sp_morphology.threshold_outside(ui, cutoff - 0.00501,
cutoff + 0.005), m, True)
if (pt[0] != 0.0):
vovo = sp_fundamentals.fft(
sp_filter.filt_tanl(vi, cutoff, falloff))
for x in range(nn):
for y in range(nn):
for z in range(nn):
if (m.get_value_at(x, y, z) > 0.5):
if (round(ui.get_value_at(x, y, z),
2) == cutoff):
filteredvol.set_value_at_fast(
x, y, z, vovo.get_value_at(x, y, z))
sp_global_def.write_command(optparse.os.path.dirname(outvol))
filteredvol.write_image(outvol)
def test_(self):
testargs_new = [
path.join(ABSOLUTE_BIN_PATH, "sp_ctf_refine.py"), "manual",
"bdb:" + path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"06_SUBSTACK_ANO#isac_substack"),
self.new_output_folder,
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"12_POSTREFINER", "vol_combined.hdf"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"11_MERIDIEN", "final_params_028.txt")
]
print("2nd round")
testargs_old = [
path.join(ABSOLUTE_OLDBIN_PATH, "sp_ctf_refine.py"), "manual",
"bdb:" + path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"06_SUBSTACK_ANO#isac_substack"),
self.old_output_folder,
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"12_POSTREFINER", "vol_combined.hdf"),
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"11_MERIDIEN", "final_params_028.txt")
]
with patch.object(sys, 'argv', testargs_new):
fu._main_()
with patch.object(sys, 'argv', testargs_old):
oldfu._main_()
fpath_new = 'bdb:{0}#{1}'.format(self.new_output_folder, self.bdb_name)
fpath_old = 'bdb:{0}#{1}'.format(self.old_output_folder, self.bdb_name)
return_old = get_im(fpath_old)
return_new = get_im(fpath_new)
self.assertTrue(
array_equal(return_new.get_3dview(), return_old.get_3dview()))
self.assertTrue(
array_equal(return_new.get_3dview().flatten().tolist()[0:100], [
-0.38441434502601624, -0.4573197066783905,
-0.39960816502571106, -0.10431570559740067,
0.018583765253424644, 0.37694355845451355, 0.4353419840335846,
0.06498070061206818, -0.38467326760292053, -0.7100721001625061,
-0.7571740746498108, -0.9328497648239136, -0.8550546169281006,
-0.21433517336845398, 0.34148913621902466, 0.30916473269462585,
0.39394184947013855, 0.4447155296802521, -0.032833874225616455,
0.4121330976486206, 0.638403594493866, 0.34945985674858093,
0.2751978039741516, -0.14872148633003235, -0.5307353138923645,
-0.6574574112892151, -0.8945914506912231, -1.107301950454712,
-1.3676011562347412, -1.2225056886672974, -1.2273787260055542,
-0.7121877074241638, 0.06733409315347672, 0.29879996180534363,
0.6046097874641418, 0.496036559343338, 0.7214235663414001,
0.7652679681777954, 0.9319247603416443, 0.11259084939956665,
-0.3054805397987366, -0.8183746933937073, -1.4462037086486816,
-1.4775571823120117, -1.2116808891296387, -0.911469042301178,
-0.9934141039848328, -0.9891195297241211, -1.0073580741882324,
-1.234678864479065, -1.4292954206466675, -1.5580313205718994,
-1.446936845779419, -1.67134428024292, -2.0199873447418213,
-2.0543317794799805, -2.2354137897491455, -1.99867844581604,
-1.4405670166015625, -1.7088592052459717, -2.060110569000244,
-1.6585056781768799, -1.0386717319488525, -0.4783007502555847,
-0.12102964520454407, 0.16874085366725922,
-0.12371158599853516, 0.01841016113758087, -0.6442297101020813,
-0.6972493529319763, -0.1051267459988594, 0.016293810680508614,
-0.10689342767000198, -0.472159206867218, -0.4886413514614105,
-0.09828135371208191, -0.7409976720809937, -0.8893253207206726,
-1.1477444171905518, -0.35942375659942627,
0.030692437663674355, 0.5380961298942566, 1.06252121925354,
0.6961002349853516, 0.7563707232475281, 1.0197871923446655,
0.9021824598312378, 0.34891727566719055, 0.402149498462677,
0.634938657283783, 0.6044892072677612, 0.24482420086860657,
-0.4132026731967926, -0.9945327043533325, -1.260424256324768,
-1.6913681030273438, -1.7999876737594604, -1.5891680717468262,
-1.3196191787719727, -0.9893324375152588
]))
def combine_isac_params(isac_dir,
classavgstack,
chains_params_file,
old_combined_parts,
classdoc,
combined_params_file,
log=False,
verbose=False):
"""
Combines initial and all_params from ISAC.
Arguments:
isac_dir : ISAC directory
classavgstack : Input image stack
chains_params_file : Input alignment parameters applied to averages in sp_chains
old_combined_parts
classdoc
combined_params_file : Output combined alignment parameters
log : instance of Logger class
verbose : (boolean) Whether to write to screen
"""
from sp_utilities import combine_params2, read_text_row
# File-handling
init_params_file = os.path.join(isac_dir, "2dalignment",
"initial2Dparams.txt")
all_params_file = os.path.join(isac_dir, "all_parameters.txt")
init_params_list = read_text_row(init_params_file)
all_params = read_text_row(all_params_file)
isac_shrink_path = os.path.join(isac_dir, "README_shrink_ratio.txt")
isac_shrink_file = open(isac_shrink_path, "r")
isac_shrink_lines = isac_shrink_file.readlines()
isac_shrink_ratio = float(isac_shrink_lines[5])
"""
Three cases:
1) Using class_averages.hdf
2) Using ordered_class_averages.hdf, but chains_params.txt doesn't exist
3) Using ordered_class_averages.hdf and chains_params.txt
"""
msg = "Combining alignment parameters from %s and %s, dividing by %s, and writing to %s" % \
(init_params_file, all_params_file, isac_shrink_ratio, combined_params_file)
# Check if using ordered class averages and whether chains_params exists
if os.path.basename(classavgstack) == 'ordered_class_averages.hdf':
if not os.path.exists(chains_params_file):
msg += "WARNING: '%s' does not exist. " % chains_params_file
msg += " Using '%s' but alignment parameters correspond to 'class_averages.hdf'.\n" % classavgstack
else:
msg = "Combining alignment parameters from %s, %s, and %s, dividing by %s, and writing to %s" % \
(init_params_file, all_params_file, chains_params_file, isac_shrink_ratio, combined_params_file)
print_log_msg(msg, log, verbose)
if os.path.basename(
classavgstack) == 'ordered_class_averages.hdf' and os.path.exists(
chains_params_file):
chains_params_list = read_text_row(chains_params_file)
old_combined_list = read_text_row(old_combined_parts)
num_classes = EMUtil.get_image_count(classavgstack)
tmp_combined = []
# Loop through classes
for class_num in range(num_classes):
# Extract members
image = get_im(classavgstack, class_num)
members = sorted(image.get_attr("members"))
old_class_list = read_text_row(classdoc.format(class_num))
new_class_list = []
# Loop through particles
for idx, im in enumerate(members):
tmp_par = combine_params2(
init_params_list[im][0], init_params_list[im][1],
init_params_list[im][2], init_params_list[im][3],
all_params[im][0], all_params[im][1] / isac_shrink_ratio,
all_params[im][2] / isac_shrink_ratio, all_params[im][3])
# Combine with class-average parameters
P = combine_params2(tmp_par[0], tmp_par[1], tmp_par[2],
tmp_par[3],
chains_params_list[class_num][2],
chains_params_list[class_num][3],
chains_params_list[class_num][4],
chains_params_list[class_num][5])
tmp_combined.append([im, P[0], P[1], P[2], P[3]])
# Need to update class number in class docs
old_part_num = old_class_list[idx]
try:
new_part_num = old_combined_list.index(old_part_num)
except ValueError:
print(
"Couldn't find particle: class_num %s, old_part_num %s, new_part_num %s"
% (class_num, old_part_num[0], new_part_num))
new_class_list.append(new_part_num)
# End particle-loop
# Overwrite pre-existing class doc
write_text_row(new_class_list, classdoc.format(class_num))
# End class-loop
# Sort by particle number
combined_params = sorted(tmp_combined, key=itemgetter(0))
# Remove first column
for row in combined_params:
del row[0]
# Not applying alignments of ordered_class_averages
else:
combined_params = []
# Loop through images
for im in range(len(all_params)):
P = combine_params2(
init_params_list[im][0], init_params_list[im][1],
init_params_list[im][2], init_params_list[im][3],
all_params[im][0], all_params[im][1] / isac_shrink_ratio,
all_params[im][2] / isac_shrink_ratio, all_params[im][3])
combined_params.append([P[0], P[1], P[2], P[3], 1.0])
write_text_row(combined_params, combined_params_file)
print_log_msg(
'Wrote %s entries to %s\n' %
(len(combined_params), combined_params_file), log, verbose)
return combined_params
def shiftali_MPI(stack,
maskfile=None,
maxit=100,
CTF=False,
snr=1.0,
Fourvar=False,
search_rng=-1,
oneDx=False,
search_rng_y=-1):
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("shiftali_MPI")
max_iter = int(maxit)
if myid == main_node:
if ftp == "bdb":
from EMAN2db import db_open_dict
dummy = db_open_dict(stack, True)
nima = EMUtil.get_image_count(stack)
else:
nima = 0
nima = bcast_number_to_all(nima, source_node=main_node)
list_of_particles = list(range(nima))
image_start, image_end = MPI_start_end(nima, number_of_proc, myid)
list_of_particles = list_of_particles[image_start:image_end]
# read nx and ctf_app (if CTF) and broadcast to all nodes
if myid == main_node:
ima = EMData()
ima.read_image(stack, list_of_particles[0], True)
nx = ima.get_xsize()
ny = ima.get_ysize()
if CTF: ctf_app = ima.get_attr_default('ctf_applied', 2)
del ima
else:
nx = 0
ny = 0
if CTF: ctf_app = 0
nx = bcast_number_to_all(nx, source_node=main_node)
ny = bcast_number_to_all(ny, source_node=main_node)
if CTF:
ctf_app = bcast_number_to_all(ctf_app, source_node=main_node)
if ctf_app > 0:
ERROR("data cannot be ctf-applied", myid=myid)
if maskfile == None:
mrad = min(nx, ny)
mask = model_circle(mrad // 2 - 2, nx, ny)
else:
mask = get_im(maskfile)
if CTF:
from sp_filter import filt_ctf
from sp_morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
from sp_global_def import CACHE_DISABLE
if CACHE_DISABLE:
data = EMData.read_images(stack, list_of_particles)
else:
for i in range(number_of_proc):
if myid == i:
data = EMData.read_images(stack, list_of_particles)
if ftp == "bdb": mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
for im in range(len(data)):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
else:
ctf_2_sum = None
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
for i in range(0, nx, 2):
for j in range(ny):
temp.set_value_at(i, j, snr)
Util.add_img(ctf_2_sum, temp)
del temp
total_iter = 0
# apply initial xform.align2d parameters stored in header
init_params = []
for im in range(len(data)):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im],
p['alpha'],
sx=p['tx'],
sy=p['ty'],
mirror=p['mirror'],
scale=p['scale'])
# fourier transform all images, and apply ctf if CTF
for im in range(len(data)):
if CTF:
ctf_params = data[im].get_attr("ctf")
data[im] = filt_ctf(fft(data[im]), ctf_params)
else:
data[im] = fft(data[im])
sx_sum = 0
sy_sum = 0
sx_sum_total = 0
sy_sum_total = 0
shift_x = [0.0] * len(data)
shift_y = [0.0] * len(data)
ishift_x = [0.0] * len(data)
ishift_y = [0.0] * len(data)
for Iter in range(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n" % (total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in data:
Util.add_img(avg, im)
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF:
tavg = Util.divn_filter(avg, ctf_2_sum)
else:
tavg = Util.mult_scalar(avg, 1.0 / float(nima))
else:
tavg = EMData(nx, ny, 1, False)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask(tavg, mask, False)
tavg -= stat[0]
Util.mul_img(tavg, mask)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
tavg = fft(tavg)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
sx_sum = 0
sy_sum = 0
if search_rng > 0: nwx = 2 * search_rng + 1
else: nwx = nx
if search_rng_y > 0: nwy = 2 * search_rng_y + 1
else: nwy = ny
not_zero = 0
for im in range(len(data)):
if oneDx:
ctx = Util.window(ccf(data[im], tavg), nwx, 1)
p1 = peak_search(ctx)
p1_x = -int(p1[0][3])
ishift_x[im] = p1_x
sx_sum += p1_x
else:
p1 = peak_search(Util.window(ccf(data[im], tavg), nwx, nwy))
p1_x = -int(p1[0][4])
p1_y = -int(p1[0][5])
ishift_x[im] = p1_x
ishift_y[im] = p1_y
sx_sum += p1_x
sy_sum += p1_y
if not_zero == 0:
if (not (ishift_x[im] == 0.0)) or (not (ishift_y[im] == 0.0)):
not_zero = 1
sx_sum = mpi.mpi_reduce(sx_sum, 1, mpi.MPI_INT, mpi.MPI_SUM, main_node,
mpi.MPI_COMM_WORLD)
if not oneDx:
sy_sum = mpi.mpi_reduce(sy_sum, 1, mpi.MPI_INT, mpi.MPI_SUM,
main_node, mpi.MPI_COMM_WORLD)
if myid == main_node:
sx_sum_total = int(sx_sum[0])
if not oneDx:
sy_sum_total = int(sy_sum[0])
else:
sx_sum_total = 0
sy_sum_total = 0
sx_sum_total = bcast_number_to_all(sx_sum_total, source_node=main_node)
if not oneDx:
sy_sum_total = bcast_number_to_all(sy_sum_total,
source_node=main_node)
sx_ave = round(float(sx_sum_total) / nima)
sy_ave = round(float(sy_sum_total) / nima)
for im in range(len(data)):
p1_x = ishift_x[im] - sx_ave
p1_y = ishift_y[im] - sy_ave
params2 = {
"filter_type": Processor.fourier_filter_types.SHIFT,
"x_shift": p1_x,
"y_shift": p1_y,
"z_shift": 0.0
}
data[im] = Processor.EMFourierFilter(data[im], params2)
shift_x[im] += p1_x
shift_y[im] += p1_y
# stop if all shifts are zero
not_zero = mpi.mpi_reduce(not_zero, 1, mpi.MPI_INT, mpi.MPI_SUM,
main_node, mpi.MPI_COMM_WORLD)
if myid == main_node:
not_zero_all = int(not_zero[0])
else:
not_zero_all = 0
not_zero_all = bcast_number_to_all(not_zero_all, source_node=main_node)
if myid == main_node:
print_msg("Time of iteration = %12.2f\n" % (time() - start_time))
start_time = time()
if not_zero_all == 0: break
#for im in xrange(len(data)): data[im] = fft(data[im]) This should not be required as only header information is used
# combine shifts found with the original parameters
for im in range(len(data)):
t0 = init_params[im]
t1 = Transform()
t1.set_params({
"type": "2D",
"alpha": 0,
"scale": t0.get_scale(),
"mirror": 0,
"tx": shift_x[im],
"ty": shift_y[im]
})
# combine t0 and t1
tt = t1 * t0
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from sp_utilities import file_type
if (file_type(stack) == "bdb"):
from sp_utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, image_start,
image_end, number_of_proc)
else:
from sp_utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, image_start,
image_end, number_of_proc)
else:
send_attr_dict(main_node, data, par_str, image_start, image_end)
if myid == main_node: print_end_msg("shiftali_MPI")
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack .. average eigvol output_factcoords --rad=radius --neigvol=number_of_eigvol --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--rad", type="int", default=-1, help="radius of mask")
parser.add_option("--neigvol",
type="int",
default=-1,
help="number of eigvenvectors to use (default all)")
parser.add_option(
"--fl",
type="float",
default=0.0,
help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
parser.add_option(
"--aa",
type="float",
default=0.0,
help="fall-off of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--CTF",
action="store_true",
default=False,
help="Use CTF")
parser.add_option("--MPI", action="store_true", help="use MPI")
(options, args) = parser.parse_args()
if (len(args) < 4):
sxprint("Usage: " + usage)
sxprint("Please run \'" + progname + " -h\' for detailed options")
ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
else:
stacks = args[0:-3]
avgvol = args[-3]
eigvol = args[-2]
output = args[-1]
if options.rad < 0:
ERROR("Mask radius is not given")
return
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
from sp_utilities import get_im
sp_global_def.BATCH = True
if (get_im(stacks[0]).get_zsize() == 1
and get_im(eigvol).get_zsize() > 1):
from sp_applications import factcoords_prj
factcoords_prj(stacks, avgvol, eigvol, output, options.rad,
options.neigvol, options.fl, options.aa,
options.CTF, options.MPI)
else:
from sp_applications import factcoords_vol
factcoords_vol(stacks, avgvol, eigvol, output, options.rad,
options.neigvol, options.fl, options.aa,
options.MPI)
sp_global_def.BATCH = False
def main():
from sp_logger import Logger, BaseLogger_Files
arglist = []
i = 0
while (i < len(sys.argv)):
if sys.argv[i] == '-p4pg':
i = i + 2
elif sys.argv[i] == '-p4wd':
i = i + 2
else:
arglist.append(sys.argv[i])
i = i + 1
progname = os.path.basename(arglist[0])
usage = progname + " stack outdir <mask> --focus=3Dmask --radius=outer_radius --delta=angular_step" +\
"--an=angular_neighborhood --maxit=max_iter --CTF --sym=c1 --function=user_function --independent=indenpendent_runs --number_of_images_per_group=number_of_images_per_group --low_pass_filter=.25 --seed=random_seed"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--focus",
type="string",
default='',
help="bineary 3D mask for focused clustering ")
parser.add_option(
"--ir",
type="int",
default=1,
help="inner radius for rotational correlation > 0 (set to 1)")
parser.add_option(
"--radius",
type="int",
default=-1,
help=
"particle radius in pixel for rotational correlation <nx-1 (set to the radius of the particle)"
)
parser.add_option("--maxit",
type="int",
default=25,
help="maximum number of iteration")
parser.add_option(
"--rs",
type="int",
default=1,
help="step between rings in rotational correlation >0 (set to 1)")
parser.add_option(
"--xr",
type="string",
default='1',
help="range for translation search in x direction, search is +/-xr ")
parser.add_option(
"--yr",
type="string",
default='-1',
help=
"range for translation search in y direction, search is +/-yr (default = same as xr)"
)
parser.add_option(
"--ts",
type="string",
default='0.25',
help=
"step size of the translation search in both directions direction, search is -xr, -xr+ts, 0, xr-ts, xr "
)
parser.add_option("--delta",
type="string",
default='2',
help="angular step of reference projections")
parser.add_option("--an",
type="string",
default='-1',
help="angular neighborhood for local searches")
parser.add_option(
"--center",
type="int",
default=0,
help=
"0 - if you do not want the volume to be centered, 1 - center the volume using cog (default=0)"
)
parser.add_option(
"--nassign",
type="int",
default=1,
help=
"number of reassignment iterations performed for each angular step (set to 3) "
)
parser.add_option(
"--nrefine",
type="int",
default=0,
help=
"number of alignment iterations performed for each angular step (set to 0)"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="do CTF correction during clustring")
parser.add_option(
"--stoprnct",
type="float",
default=3.0,
help="Minimum percentage of assignment change to stop the program")
parser.add_option("--sym",
type="string",
default='c1',
help="symmetry of the structure ")
parser.add_option("--function",
type="string",
default='do_volume_mrk05',
help="name of the reference preparation function")
parser.add_option("--independent",
type="int",
default=3,
help="number of independent run")
parser.add_option("--number_of_images_per_group",
type="int",
default=1000,
help="number of groups")
parser.add_option(
"--low_pass_filter",
type="float",
default=-1.0,
help=
"absolute frequency of low-pass filter for 3d sorting on the original image size"
)
parser.add_option("--nxinit",
type="int",
default=64,
help="initial image size for sorting")
parser.add_option("--unaccounted",
action="store_true",
default=False,
help="reconstruct the unaccounted images")
parser.add_option(
"--seed",
type="int",
default=-1,
help="random seed for create initial random assignment for EQ Kmeans")
parser.add_option("--smallest_group",
type="int",
default=500,
help="minimum members for identified group")
parser.add_option("--sausage",
action="store_true",
default=False,
help="way of filter volume")
parser.add_option("--chunk0",
type="string",
default='',
help="chunk0 for computing margin of error")
parser.add_option("--chunk1",
type="string",
default='',
help="chunk1 for computing margin of error")
parser.add_option(
"--PWadjustment",
type="string",
default='',
help=
"1-D power spectrum of PDB file used for EM volume power spectrum correction"
)
parser.add_option(
"--protein_shape",
type="string",
default='g',
help=
"protein shape. It defines protein preferred orientation angles. Currently it has g and f two types "
)
parser.add_option(
"--upscale",
type="float",
default=0.5,
help=" scaling parameter to adjust the power spectrum of EM volumes")
parser.add_option("--wn",
type="int",
default=0,
help="optimal window size for data processing")
parser.add_option(
"--interpolation",
type="string",
default="4nn",
help="3-d reconstruction interpolation method, two options trl and 4nn"
)
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 1 or len(args) > 4:
sxprint("Usage: " + usage)
sxprint("Please run \'" + progname + " -h\' for detailed options")
ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
else:
if len(args) > 2:
mask_file = args[2]
else:
mask_file = None
orgstack = args[0]
masterdir = args[1]
sp_global_def.BATCH = True
#---initialize MPI related variables
nproc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
mpi_comm = mpi.MPI_COMM_WORLD
main_node = 0
# import some utilities
from sp_utilities import get_im, bcast_number_to_all, cmdexecute, write_text_file, read_text_file, wrap_mpi_bcast, get_params_proj, write_text_row
from sp_applications import recons3d_n_MPI, mref_ali3d_MPI, Kmref_ali3d_MPI
from sp_statistics import k_means_match_clusters_asg_new, k_means_stab_bbenum
from sp_applications import mref_ali3d_EQ_Kmeans, ali3d_mref_Kmeans_MPI
# Create the main log file
from sp_logger import Logger, BaseLogger_Files
if myid == main_node:
log_main = Logger(BaseLogger_Files())
log_main.prefix = masterdir + "/"
else:
log_main = None
#--- fill input parameters into dictionary named after Constants
Constants = {}
Constants["stack"] = args[0]
Constants["masterdir"] = masterdir
Constants["mask3D"] = mask_file
Constants["focus3Dmask"] = options.focus
Constants["indep_runs"] = options.independent
Constants["stoprnct"] = options.stoprnct
Constants[
"number_of_images_per_group"] = options.number_of_images_per_group
Constants["CTF"] = options.CTF
Constants["maxit"] = options.maxit
Constants["ir"] = options.ir
Constants["radius"] = options.radius
Constants["nassign"] = options.nassign
Constants["rs"] = options.rs
Constants["xr"] = options.xr
Constants["yr"] = options.yr
Constants["ts"] = options.ts
Constants["delta"] = options.delta
Constants["an"] = options.an
Constants["sym"] = options.sym
Constants["center"] = options.center
Constants["nrefine"] = options.nrefine
#Constants["fourvar"] = options.fourvar
Constants["user_func"] = options.function
Constants[
"low_pass_filter"] = options.low_pass_filter # enforced low_pass_filter
#Constants["debug"] = options.debug
Constants["main_log_prefix"] = args[1]
#Constants["importali3d"] = options.importali3d
Constants["myid"] = myid
Constants["main_node"] = main_node
Constants["nproc"] = nproc
Constants["log_main"] = log_main
Constants["nxinit"] = options.nxinit
Constants["unaccounted"] = options.unaccounted
Constants["seed"] = options.seed
Constants["smallest_group"] = options.smallest_group
Constants["sausage"] = options.sausage
Constants["chunk0"] = options.chunk0
Constants["chunk1"] = options.chunk1
Constants["PWadjustment"] = options.PWadjustment
Constants["upscale"] = options.upscale
Constants["wn"] = options.wn
Constants["3d-interpolation"] = options.interpolation
Constants["protein_shape"] = options.protein_shape
# -----------------------------------------------------
#
# Create and initialize Tracker dictionary with input options
Tracker = {}
Tracker["constants"] = Constants
Tracker["maxit"] = Tracker["constants"]["maxit"]
Tracker["radius"] = Tracker["constants"]["radius"]
#Tracker["xr"] = ""
#Tracker["yr"] = "-1" # Do not change!
#Tracker["ts"] = 1
#Tracker["an"] = "-1"
#Tracker["delta"] = "2.0"
#Tracker["zoom"] = True
#Tracker["nsoft"] = 0
#Tracker["local"] = False
#Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
Tracker["upscale"] = Tracker["constants"]["upscale"]
#Tracker["upscale"] = 0.5
Tracker[
"applyctf"] = False # Should the data be premultiplied by the CTF. Set to False for local continuous.
#Tracker["refvol"] = None
Tracker["nxinit"] = Tracker["constants"]["nxinit"]
#Tracker["nxstep"] = 32
Tracker["icurrentres"] = -1
#Tracker["ireachedres"] = -1
#Tracker["lowpass"] = 0.4
#Tracker["falloff"] = 0.2
#Tracker["inires"] = options.inires # Now in A, convert to absolute before using
Tracker["fuse_freq"] = 50 # Now in A, convert to absolute before using
#Tracker["delpreviousmax"] = False
#Tracker["anger"] = -1.0
#Tracker["shifter"] = -1.0
#Tracker["saturatecrit"] = 0.95
#Tracker["pixercutoff"] = 2.0
#Tracker["directory"] = ""
#Tracker["previousoutputdir"] = ""
#Tracker["eliminated-outliers"] = False
#Tracker["mainiteration"] = 0
#Tracker["movedback"] = False
#Tracker["state"] = Tracker["constants"]["states"][0]
#Tracker["global_resolution"] =0.0
Tracker["orgstack"] = orgstack
#--------------------------------------------------------------------
# import from utilities
from sp_utilities import sample_down_1D_curve, get_initial_ID, remove_small_groups, print_upper_triangular_matrix, print_a_line_with_timestamp
from sp_utilities import print_dict, get_resolution_mrk01, partition_to_groups, partition_independent_runs, get_outliers
from sp_utilities import merge_groups, save_alist, margin_of_error, get_margin_of_error, do_two_way_comparison, select_two_runs, get_ali3d_params
from sp_utilities import counting_projections, unload_dict, load_dict, get_stat_proj, create_random_list, get_number_of_groups, recons_mref
from sp_utilities import apply_low_pass_filter, get_groups_from_partition, get_number_of_groups, get_complementary_elements_total, update_full_dict
from sp_utilities import count_chunk_members, set_filter_parameters_from_adjusted_fsc, get_two_chunks_from_stack
####------------------------------------------------------------------
#
# Get the pixel size; if none, set to 1.0, and the original image size
from sp_utilities import get_shrink_data_huang
if (myid == main_node):
line = strftime("%Y-%m-%d_%H:%M:%S", localtime()) + " =>"
sxprint((line + "Initialization of 3-D sorting"))
a = get_im(orgstack)
nnxo = a.get_xsize()
if (Tracker["nxinit"] > nnxo):
sp_global_def.ERROR(
"Image size less than minimum permitted $d" %
Tracker["nxinit"])
nnxo = -1
else:
if Tracker["constants"]["CTF"]:
i = a.get_attr('ctf')
pixel_size = i.apix
fq = pixel_size / Tracker["fuse_freq"]
else:
pixel_size = 1.0
# No pixel size, fusing computed as 5 Fourier pixels
fq = 5.0 / nnxo
del a
else:
nnxo = 0
fq = 0.0
pixel_size = 1.0
nnxo = bcast_number_to_all(nnxo, source_node=main_node)
if (nnxo < 0):
return
pixel_size = bcast_number_to_all(pixel_size, source_node=main_node)
fq = bcast_number_to_all(fq, source_node=main_node)
if Tracker["constants"]["wn"] == 0:
Tracker["constants"]["nnxo"] = nnxo
else:
Tracker["constants"]["nnxo"] = Tracker["constants"]["wn"]
nnxo = Tracker["constants"]["nnxo"]
Tracker["constants"]["pixel_size"] = pixel_size
Tracker["fuse_freq"] = fq
del fq, nnxo, pixel_size
if (Tracker["constants"]["radius"] < 1):
Tracker["constants"][
"radius"] = Tracker["constants"]["nnxo"] // 2 - 2
elif ((2 * Tracker["constants"]["radius"] + 2) >
Tracker["constants"]["nnxo"]):
sp_global_def.ERROR("Particle radius set too large!", myid=myid)
####-----------------------------------------------------------------------------------------
# Master directory
if myid == main_node:
if masterdir == "":
timestring = strftime("_%d_%b_%Y_%H_%M_%S", localtime())
masterdir = "master_sort3d" + timestring
li = len(masterdir)
cmd = "{} {}".format("mkdir -p", masterdir)
os.system(cmd)
else:
li = 0
li = mpi.mpi_bcast(li, 1, mpi.MPI_INT, main_node,
mpi.MPI_COMM_WORLD)[0]
if li > 0:
masterdir = mpi.mpi_bcast(masterdir, li, mpi.MPI_CHAR, main_node,
mpi.MPI_COMM_WORLD)
import string
masterdir = string.join(masterdir, "")
if myid == main_node:
print_dict(Tracker["constants"],
"Permanent settings of 3-D sorting program")
######### create a vstack from input stack to the local stack in masterdir
# stack name set to default
Tracker["constants"]["stack"] = "bdb:" + masterdir + "/rdata"
Tracker["constants"]["ali3d"] = os.path.join(masterdir,
"ali3d_init.txt")
Tracker["constants"]["ctf_params"] = os.path.join(
masterdir, "ctf_params.txt")
Tracker["constants"]["partstack"] = Tracker["constants"][
"ali3d"] # also serves for refinement
if myid == main_node:
total_stack = EMUtil.get_image_count(Tracker["orgstack"])
else:
total_stack = 0
total_stack = bcast_number_to_all(total_stack, source_node=main_node)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
from time import sleep
while not os.path.exists(masterdir):
sxprint("Node ", myid, " waiting...")
sleep(5)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == main_node:
log_main.add("Sphire sort3d ")
log_main.add("the sort3d master directory is " + masterdir)
#####
###----------------------------------------------------------------------------------
# Initial data analysis and handle two chunk files
from random import shuffle
# Compute the resolution
#### make chunkdir dictionary for computing margin of error
import sp_user_functions
user_func = sp_user_functions.factory[Tracker["constants"]
["user_func"]]
chunk_dict = {}
chunk_list = []
if myid == main_node:
chunk_one = read_text_file(Tracker["constants"]["chunk0"])
chunk_two = read_text_file(Tracker["constants"]["chunk1"])
else:
chunk_one = 0
chunk_two = 0
chunk_one = wrap_mpi_bcast(chunk_one, main_node)
chunk_two = wrap_mpi_bcast(chunk_two, main_node)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
######################## Read/write bdb: data on main node ############################
if myid == main_node:
if (orgstack[:4] == "bdb:"):
cmd = "{} {} {}".format(
"e2bdb.py", orgstack,
"--makevstack=" + Tracker["constants"]["stack"])
else:
cmd = "{} {} {}".format("sp_cpy.py", orgstack,
Tracker["constants"]["stack"])
junk = cmdexecute(cmd)
cmd = "{} {} {}".format(
"sp_header.py --params=xform.projection",
"--export=" + Tracker["constants"]["ali3d"], orgstack)
junk = cmdexecute(cmd)
cmd = "{} {} {}".format(
"sp_header.py --params=ctf",
"--export=" + Tracker["constants"]["ctf_params"], orgstack)
junk = cmdexecute(cmd)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
########-----------------------------------------------------------------------------
Tracker["total_stack"] = total_stack
Tracker["constants"]["total_stack"] = total_stack
Tracker["shrinkage"] = float(
Tracker["nxinit"]) / Tracker["constants"]["nnxo"]
Tracker[
"radius"] = Tracker["constants"]["radius"] * Tracker["shrinkage"]
if Tracker["constants"]["mask3D"]:
Tracker["mask3D"] = os.path.join(masterdir, "smask.hdf")
else:
Tracker["mask3D"] = None
if Tracker["constants"]["focus3Dmask"]:
Tracker["focus3D"] = os.path.join(masterdir, "sfocus.hdf")
else:
Tracker["focus3D"] = None
if myid == main_node:
if Tracker["constants"]["mask3D"]:
mask_3D = get_shrink_3dmask(Tracker["nxinit"],
Tracker["constants"]["mask3D"])
mask_3D.write_image(Tracker["mask3D"])
if Tracker["constants"]["focus3Dmask"]:
mask_3D = get_shrink_3dmask(
Tracker["nxinit"], Tracker["constants"]["focus3Dmask"])
st = Util.infomask(mask_3D, None, True)
if (st[0] == 0.0):
ERROR(
"Incorrect focused mask, after binarize all values zero"
)
mask_3D.write_image(Tracker["focus3D"])
del mask_3D
if Tracker["constants"]["PWadjustment"] != '':
PW_dict = {}
nxinit_pwsp = sample_down_1D_curve(
Tracker["constants"]["nxinit"], Tracker["constants"]["nnxo"],
Tracker["constants"]["PWadjustment"])
Tracker["nxinit_PW"] = os.path.join(masterdir, "spwp.txt")
if myid == main_node:
write_text_file(nxinit_pwsp, Tracker["nxinit_PW"])
PW_dict[Tracker["constants"]
["nnxo"]] = Tracker["constants"]["PWadjustment"]
PW_dict[Tracker["constants"]["nxinit"]] = Tracker["nxinit_PW"]
Tracker["PW_dict"] = PW_dict
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
#-----------------------From two chunks to FSC, and low pass filter-----------------------------------------###
for element in chunk_one:
chunk_dict[element] = 0
for element in chunk_two:
chunk_dict[element] = 1
chunk_list = [chunk_one, chunk_two]
Tracker["chunk_dict"] = chunk_dict
Tracker["P_chunk0"] = len(chunk_one) / float(total_stack)
Tracker["P_chunk1"] = len(chunk_two) / float(total_stack)
### create two volumes to estimate resolution
if myid == main_node:
for index in range(2):
write_text_file(
chunk_list[index],
os.path.join(masterdir, "chunk%01d.txt" % index))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
vols = []
for index in range(2):
data, old_shifts = get_shrink_data_huang(
Tracker,
Tracker["constants"]["nxinit"],
os.path.join(masterdir, "chunk%01d.txt" % index),
Tracker["constants"]["partstack"],
myid,
main_node,
nproc,
preshift=True)
vol = recons3d_4nn_ctf_MPI(myid=myid,
prjlist=data,
symmetry=Tracker["constants"]["sym"],
finfo=None)
if myid == main_node:
vol.write_image(os.path.join(masterdir, "vol%d.hdf" % index))
vols.append(vol)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if myid == main_node:
low_pass, falloff, currentres = get_resolution_mrk01(
vols, Tracker["constants"]["radius"],
Tracker["constants"]["nxinit"], masterdir, Tracker["mask3D"])
if low_pass > Tracker["constants"]["low_pass_filter"]:
low_pass = Tracker["constants"]["low_pass_filter"]
else:
low_pass = 0.0
falloff = 0.0
currentres = 0.0
bcast_number_to_all(currentres, source_node=main_node)
bcast_number_to_all(low_pass, source_node=main_node)
bcast_number_to_all(falloff, source_node=main_node)
Tracker["currentres"] = currentres
Tracker["falloff"] = falloff
if Tracker["constants"]["low_pass_filter"] == -1.0:
Tracker["low_pass_filter"] = min(
.45, low_pass / Tracker["shrinkage"]) # no better than .45
else:
Tracker["low_pass_filter"] = min(
.45,
Tracker["constants"]["low_pass_filter"] / Tracker["shrinkage"])
Tracker["lowpass"] = Tracker["low_pass_filter"]
Tracker["falloff"] = .1
Tracker["global_fsc"] = os.path.join(masterdir, "fsc.txt")
############################################################################################
if myid == main_node:
log_main.add("The command-line inputs are as following:")
log_main.add(
"**********************************************************")
for a in sys.argv:
if myid == main_node: log_main.add(a)
if myid == main_node:
log_main.add("number of cpus used in this run is %d" %
Tracker["constants"]["nproc"])
log_main.add(
"**********************************************************")
from sp_filter import filt_tanl
### START 3-D sorting
if myid == main_node:
log_main.add("----------3-D sorting program------- ")
log_main.add(
"current resolution %6.3f for images of original size in terms of absolute frequency"
% Tracker["currentres"])
log_main.add("equivalent to %f Angstrom resolution" %
(Tracker["constants"]["pixel_size"] /
Tracker["currentres"] / Tracker["shrinkage"]))
log_main.add("the user provided enforced low_pass_filter is %f" %
Tracker["constants"]["low_pass_filter"])
#log_main.add("equivalent to %f Angstrom resolution"%(Tracker["constants"]["pixel_size"]/Tracker["constants"]["low_pass_filter"]))
for index in range(2):
filt_tanl(
get_im(os.path.join(masterdir, "vol%01d.hdf" % index)),
Tracker["low_pass_filter"],
Tracker["falloff"]).write_image(
os.path.join(masterdir, "volf%01d.hdf" % index))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
from sp_utilities import get_input_from_string
delta = get_input_from_string(Tracker["constants"]["delta"])
delta = delta[0]
from sp_utilities import even_angles
n_angles = even_angles(delta, 0, 180)
this_ali3d = Tracker["constants"]["ali3d"]
sampled = get_stat_proj(Tracker, delta, this_ali3d)
if myid == main_node:
nc = 0
for a in sampled:
if len(sampled[a]) > 0:
nc += 1
log_main.add("total sampled direction %10d at angle step %6.3f" %
(len(n_angles), delta))
log_main.add(
"captured sampled directions %10d percentage covered by data %6.3f"
% (nc, float(nc) / len(n_angles) * 100))
number_of_images_per_group = Tracker["constants"][
"number_of_images_per_group"]
if myid == main_node:
log_main.add("user provided number_of_images_per_group %d" %
number_of_images_per_group)
Tracker["number_of_images_per_group"] = number_of_images_per_group
number_of_groups = get_number_of_groups(total_stack,
number_of_images_per_group)
Tracker["number_of_groups"] = number_of_groups
generation = 0
partition_dict = {}
full_dict = {}
workdir = os.path.join(masterdir, "generation%03d" % generation)
Tracker["this_dir"] = workdir
if myid == main_node:
log_main.add("---- generation %5d" % generation)
log_main.add("number of images per group is set as %d" %
number_of_images_per_group)
log_main.add("the initial number of groups is %10d " %
number_of_groups)
cmd = "{} {}".format("mkdir", workdir)
os.system(cmd)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
list_to_be_processed = list(range(Tracker["constants"]["total_stack"]))
Tracker["this_data_list"] = list_to_be_processed
create_random_list(Tracker)
#################################
full_dict = {}
for iptl in range(Tracker["constants"]["total_stack"]):
full_dict[iptl] = iptl
Tracker["full_ID_dict"] = full_dict
#################################
for indep_run in range(Tracker["constants"]["indep_runs"]):
Tracker["this_particle_list"] = Tracker["this_indep_list"][
indep_run]
ref_vol = recons_mref(Tracker)
if myid == main_node:
log_main.add("independent run %10d" % indep_run)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
Tracker["this_data_list"] = list_to_be_processed
Tracker["total_stack"] = len(Tracker["this_data_list"])
Tracker["this_particle_text_file"] = os.path.join(
workdir,
"independent_list_%03d.txt" % indep_run) # for get_shrink_data
if myid == main_node:
write_text_file(Tracker["this_data_list"],
Tracker["this_particle_text_file"])
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
outdir = os.path.join(workdir, "EQ_Kmeans%03d" % indep_run)
ref_vol = apply_low_pass_filter(ref_vol, Tracker)
mref_ali3d_EQ_Kmeans(ref_vol, outdir,
Tracker["this_particle_text_file"], Tracker)
partition_dict[indep_run] = Tracker["this_partition"]
Tracker["partition_dict"] = partition_dict
Tracker["total_stack"] = len(Tracker["this_data_list"])
Tracker["this_total_stack"] = Tracker["total_stack"]
###############################
do_two_way_comparison(Tracker)
###############################
ref_vol_list = []
from time import sleep
number_of_ref_class = []
for igrp in range(len(Tracker["two_way_stable_member"])):
Tracker["this_data_list"] = Tracker["two_way_stable_member"][igrp]
Tracker["this_data_list_file"] = os.path.join(
workdir, "stable_class%d.txt" % igrp)
if myid == main_node:
write_text_file(Tracker["this_data_list"],
Tracker["this_data_list_file"])
data, old_shifts = get_shrink_data_huang(
Tracker,
Tracker["nxinit"],
Tracker["this_data_list_file"],
Tracker["constants"]["partstack"],
myid,
main_node,
nproc,
preshift=True)
volref = recons3d_4nn_ctf_MPI(myid=myid,
prjlist=data,
symmetry=Tracker["constants"]["sym"],
finfo=None)
ref_vol_list.append(volref)
number_of_ref_class.append(len(Tracker["this_data_list"]))
if myid == main_node:
log_main.add("group %d members %d " %
(igrp, len(Tracker["this_data_list"])))
Tracker["number_of_ref_class"] = number_of_ref_class
nx_of_image = ref_vol_list[0].get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"] = Tracker["constants"][
"PWadjustment"] # no PW adjustment
if myid == main_node:
for iref in range(len(ref_vol_list)):
refdata = [None] * 4
refdata[0] = ref_vol_list[iref]
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
volref.write_image(os.path.join(workdir, "volf_stable.hdf"),
iref)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
Tracker["this_data_list"] = Tracker["this_accounted_list"]
outdir = os.path.join(workdir, "Kmref")
empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI(
ref_vol_list, outdir, Tracker["this_accounted_text"], Tracker)
Tracker["this_unaccounted_list"] = get_complementary_elements(
list_to_be_processed, final_list)
if myid == main_node:
log_main.add("the number of particles not processed is %d" %
len(Tracker["this_unaccounted_list"]))
write_text_file(Tracker["this_unaccounted_list"],
Tracker["this_unaccounted_text"])
update_full_dict(Tracker["this_unaccounted_list"], Tracker)
#######################################
number_of_groups = len(res_groups)
vol_list = []
number_of_ref_class = []
for igrp in range(number_of_groups):
data, old_shifts = get_shrink_data_huang(
Tracker,
Tracker["constants"]["nnxo"],
os.path.join(outdir, "Class%d.txt" % igrp),
Tracker["constants"]["partstack"],
myid,
main_node,
nproc,
preshift=True)
volref = recons3d_4nn_ctf_MPI(myid=myid,
prjlist=data,
symmetry=Tracker["constants"]["sym"],
finfo=None)
vol_list.append(volref)
if (myid == main_node):
npergroup = len(
read_text_file(os.path.join(outdir, "Class%d.txt" % igrp)))
else:
npergroup = 0
npergroup = bcast_number_to_all(npergroup, main_node)
number_of_ref_class.append(npergroup)
Tracker["number_of_ref_class"] = number_of_ref_class
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
nx_of_image = vol_list[0].get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
if myid == main_node:
for ivol in range(len(vol_list)):
refdata = [None] * 4
refdata[0] = vol_list[ivol]
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
volref.write_image(
os.path.join(workdir, "volf_of_Classes.hdf"), ivol)
log_main.add("number of unaccounted particles %10d" %
len(Tracker["this_unaccounted_list"]))
log_main.add("number of accounted particles %10d" %
len(Tracker["this_accounted_list"]))
Tracker["this_data_list"] = Tracker[
"this_unaccounted_list"] # reset parameters for the next round calculation
Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
Tracker["this_total_stack"] = Tracker["total_stack"]
number_of_groups = get_number_of_groups(
len(Tracker["this_unaccounted_list"]), number_of_images_per_group)
Tracker["number_of_groups"] = number_of_groups
while number_of_groups >= 2:
generation += 1
partition_dict = {}
workdir = os.path.join(masterdir, "generation%03d" % generation)
Tracker["this_dir"] = workdir
if myid == main_node:
log_main.add("*********************************************")
log_main.add("----- generation %5d " %
generation)
log_main.add("number of images per group is set as %10d " %
number_of_images_per_group)
log_main.add("the number of groups is %10d " %
number_of_groups)
log_main.add(" number of particles for clustering is %10d" %
Tracker["total_stack"])
cmd = "{} {}".format("mkdir", workdir)
os.system(cmd)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
create_random_list(Tracker)
for indep_run in range(Tracker["constants"]["indep_runs"]):
Tracker["this_particle_list"] = Tracker["this_indep_list"][
indep_run]
ref_vol = recons_mref(Tracker)
if myid == main_node:
log_main.add("independent run %10d" % indep_run)
outdir = os.path.join(workdir, "EQ_Kmeans%03d" % indep_run)
Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
#ref_vol=apply_low_pass_filter(ref_vol,Tracker)
mref_ali3d_EQ_Kmeans(ref_vol, outdir,
Tracker["this_unaccounted_text"], Tracker)
partition_dict[indep_run] = Tracker["this_partition"]
Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
Tracker["partition_dict"] = partition_dict
Tracker["this_total_stack"] = Tracker["total_stack"]
total_list_of_this_run = Tracker["this_unaccounted_list"]
###############################
do_two_way_comparison(Tracker)
###############################
ref_vol_list = []
number_of_ref_class = []
for igrp in range(len(Tracker["two_way_stable_member"])):
Tracker["this_data_list"] = Tracker["two_way_stable_member"][
igrp]
Tracker["this_data_list_file"] = os.path.join(
workdir, "stable_class%d.txt" % igrp)
if myid == main_node:
write_text_file(Tracker["this_data_list"],
Tracker["this_data_list_file"])
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
data, old_shifts = get_shrink_data_huang(
Tracker,
Tracker["constants"]["nxinit"],
Tracker["this_data_list_file"],
Tracker["constants"]["partstack"],
myid,
main_node,
nproc,
preshift=True)
volref = recons3d_4nn_ctf_MPI(
myid=myid,
prjlist=data,
symmetry=Tracker["constants"]["sym"],
finfo=None)
#volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1)
if myid == main_node:
volref.write_image(os.path.join(workdir, "vol_stable.hdf"),
iref)
#volref = resample(volref,Tracker["shrinkage"])
ref_vol_list.append(volref)
number_of_ref_class.append(len(Tracker["this_data_list"]))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
Tracker["number_of_ref_class"] = number_of_ref_class
Tracker["this_data_list"] = Tracker["this_accounted_list"]
outdir = os.path.join(workdir, "Kmref")
empty_group, res_groups, final_list = ali3d_mref_Kmeans_MPI(
ref_vol_list, outdir, Tracker["this_accounted_text"], Tracker)
# calculate the 3-D structure of original image size for each group
number_of_groups = len(res_groups)
Tracker["this_unaccounted_list"] = get_complementary_elements(
total_list_of_this_run, final_list)
if myid == main_node:
log_main.add("the number of particles not processed is %d" %
len(Tracker["this_unaccounted_list"]))
write_text_file(Tracker["this_unaccounted_list"],
Tracker["this_unaccounted_text"])
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
update_full_dict(Tracker["this_unaccounted_list"], Tracker)
vol_list = []
for igrp in range(number_of_groups):
data, old_shifts = get_shrink_data_huang(
Tracker,
Tracker["constants"]["nnxo"],
os.path.join(outdir, "Class%d.txt" % igrp),
Tracker["constants"]["partstack"],
myid,
main_node,
nproc,
preshift=True)
volref = recons3d_4nn_ctf_MPI(
myid=myid,
prjlist=data,
symmetry=Tracker["constants"]["sym"],
finfo=None)
vol_list.append(volref)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
nx_of_image = ref_vol_list[0].get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
if myid == main_node:
for ivol in range(len(vol_list)):
refdata = [None] * 4
refdata[0] = vol_list[ivol]
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
volref.write_image(
os.path.join(workdir, "volf_of_Classes.hdf"), ivol)
log_main.add("number of unaccounted particles %10d" %
len(Tracker["this_unaccounted_list"]))
log_main.add("number of accounted particles %10d" %
len(Tracker["this_accounted_list"]))
del vol_list
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
number_of_groups = get_number_of_groups(
len(Tracker["this_unaccounted_list"]),
number_of_images_per_group)
Tracker["number_of_groups"] = number_of_groups
Tracker["this_data_list"] = Tracker["this_unaccounted_list"]
Tracker["total_stack"] = len(Tracker["this_unaccounted_list"])
if Tracker["constants"]["unaccounted"]:
data, old_shifts = get_shrink_data_huang(
Tracker,
Tracker["constants"]["nnxo"],
Tracker["this_unaccounted_text"],
Tracker["constants"]["partstack"],
myid,
main_node,
nproc,
preshift=True)
volref = recons3d_4nn_ctf_MPI(myid=myid,
prjlist=data,
symmetry=Tracker["constants"]["sym"],
finfo=None)
nx_of_image = volref.get_xsize()
if Tracker["constants"]["PWadjustment"]:
Tracker["PWadjustment"] = Tracker["PW_dict"][nx_of_image]
else:
Tracker["PWadjustment"] = Tracker["constants"]["PWadjustment"]
if (myid == main_node):
refdata = [None] * 4
refdata[0] = volref
refdata[1] = Tracker
refdata[2] = Tracker["constants"]["myid"]
refdata[3] = Tracker["constants"]["nproc"]
volref = user_func(refdata)
#volref = filt_tanl(volref, Tracker["constants"]["low_pass_filter"],.1)
volref.write_image(
os.path.join(workdir, "volf_unaccounted.hdf"))
# Finish program
if myid == main_node: log_main.add("sxsort3d finishes")
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
return
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ Input Output [options]
Generate three micrographs, each micrograph contains one projection of a long filament.
Input: Reference Volume, output directory
Output: Three micrographs stored in output directory
sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84
Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
Generate run through example script for helicon
sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
"""
parser = OptionParser(usage, version=SPARXVERSION)
# helicise the Atom coordinates
# generate micrographs of helical filament
parser.add_option(
"--generate_micrograph",
action="store_true",
default=False,
help=
"Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="Use CTF correction")
parser.add_option("--apix",
type="float",
default=-1,
help="pixel size in Angstroms")
parser.add_option(
"--rand_seed",
type="int",
default=14567,
help=
"the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs."
)
parser.add_option("--Cs",
type="float",
default=2.0,
help="Microscope Cs (spherical aberation)")
parser.add_option("--voltage",
type="float",
default=200.0,
help="Microscope voltage in KV")
parser.add_option("--ac",
type="float",
default=10.0,
help="Amplitude contrast (percentage, default=10)")
parser.add_option("--nonoise",
action="store_true",
default=False,
help="Do not add noise to the micrograph.")
# generate initial volume
parser.add_option("--generate_noisycyl",
action="store_true",
default=False,
help="Generate initial volume of noisy cylinder.")
parser.add_option(
"--boxsize",
type="string",
default="100,100,200",
help=
"String containing x , y, z dimensions (separated by comma) in pixels")
parser.add_option("--rad",
type="int",
default=35,
help="Radius of initial volume in pixels")
# generate 2D mask
parser.add_option("--generate_mask",
action="store_true",
default=False,
help="Generate 2D rectangular mask.")
parser.add_option(
"--masksize",
type="string",
default="200,200",
help=
"String containing x and y dimensions (separated by comma) in pixels")
parser.add_option("--maskwidth",
type="int",
default=60,
help="Width of rectangular mask")
# Apply 2D alignment parameters to input stack and output new images to output stack
parser.add_option(
"--applyparams",
action="store_true",
default=False,
help=
"Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack"
)
# Generate run script
parser.add_option("--generate_script",
action="store_true",
default=False,
help="Generate script for helicon run through example")
parser.add_option("--filename",
type="string",
default="runhelicon",
help="Name of run script to generate")
parser.add_option("--seg_ny",
type="int",
default=180,
help="y-dimension of segment used for refinement")
parser.add_option(
"--ptcl_dist",
type="int",
default=15,
help=
"Distance in pixels between adjacent segments windowed from same filament"
)
parser.add_option(
"--fract",
type="float",
default=0.35,
help="Fraction of the volume used for applying helical symmetry.")
(options, args) = parser.parse_args()
if len(args) > 3:
sxprint("usage: " + usage)
sxprint("Please run '" + progname + " -h' for detailed options")
ERROR(
"Invalid number of parameters. Please see usage information above."
)
return
else:
if options.generate_script:
generate_runscript(options.filename, options.seg_ny,
options.ptcl_dist, options.fract)
if options.generate_micrograph:
if options.apix <= 0:
ERROR("Please enter pixel size.")
return
generate_helimic(args[0], args[1], options.apix, options.CTF,
options.Cs, options.voltage, options.ac,
options.nonoise, options.rand_seed)
if options.generate_noisycyl:
from sp_utilities import model_cylinder, model_gauss_noise
outvol = args[0]
boxdims = options.boxsize.split(',')
if len(boxdims) < 1 or len(boxdims) > 3:
ERROR(
"Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize=\'100,100,200\'"
)
return
nx = int(boxdims[0])
if len(boxdims) == 1:
ny = nx
nz = nx
else:
ny = int(boxdims[1])
if len(boxdims) == 3:
nz = int(boxdims[2])
(model_cylinder(options.rad, nx, ny, nz) *
model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol)
if options.generate_mask:
from sp_utilities import model_blank, pad
outvol = args[0]
maskdims = options.masksize.split(',')
if len(maskdims) < 1 or len(maskdims) > 2:
ERROR(
"Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize=\'200,200\'"
)
return
nx = int(maskdims[0])
if len(maskdims) == 1:
ny = nx
else:
ny = int(maskdims[1])
mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1,
0.0)
mask.write_image(outvol)
if options.applyparams:
from sp_utilities import get_im, get_params2D, set_params2D
from sp_fundamentals import cyclic_shift
stack = args[0]
newstack = args[1]
mask = get_im(args[2])
nima = EMUtil.get_image_count(stack)
for im in range(nima):
prj = get_im(stack, im)
alpha, sx, sy, mirror, scale = get_params2D(prj)
prj = cyclic_shift(prj, int(sx))
set_params2D(prj, [0.0, 0., 0.0, 0, 1])
stat = Util.infomask(prj, mask, False)
prj = (prj - stat[0]) / stat[1]
ctf_params = prj.get_attr("ctf")
prj.set_attr('ctf_applied', 0)
prj.write_image(newstack, im)
def cml_open_proj(stack, ir, ou, lf, hf, dpsi=1):
from sp_projection import cml_sinogram
from sp_utilities import model_circle, get_params_proj, model_blank, get_im
from sp_fundamentals import fftip
from sp_filter import filt_tanh
# number of projections
if type(stack) == type(""): nprj = EMUtil.get_image_count(stack)
else: nprj = len(stack)
Prj = [] # list of projections
Ori = [
-1
] * 4 * nprj # orientation intial (phi, theta, psi, index) for each projection
for i in range(nprj):
image = get_im(stack, i)
# read initial angles if given
try:
Ori[4 * i], Ori[4 * i +
1], Ori[4 * i +
2], s2x, s2y = get_params_proj(image)
except:
pass
if (i == 0):
nx = image.get_xsize()
if (ou < 1): ou = nx // 2 - 1
diameter = int(2 * ou)
mask2D = model_circle(ou, nx, nx)
if ir > 0: mask2D -= model_circle(ir, nx, nx)
# normalize under the mask
[mean_a, sigma, imin, imax] = Util.infomask(image, mask2D, True)
image -= mean_a
Util.mul_scalar(image, 1.0 / sigma)
Util.mul_img(image, mask2D)
# sinogram
sino = cml_sinogram(image, diameter, dpsi)
# prepare the cut positions in order to filter (lf: low freq; hf: high freq)
ihf = min(int(2 * hf * diameter), diameter + (diameter + 1) % 2)
ihf = ihf + (ihf + 1) % 2 # index ihf must be odd to take the img part
ilf = max(int(2 * lf * diameter), 0)
ilf = ilf + ilf % 2 # index ilf must be even to fall in the real part
bdf = ihf - ilf + 1
# process lines
nxe = sino.get_xsize()
nye = sino.get_ysize()
prj = model_blank(bdf, 2 * nye)
pp = model_blank(nxe, 2 * nye)
for li in range(nye):
# get the line li
line = Util.window(sino, nxe, 1, 1, 0, li - nye // 2, 0)
# u2 (not improve the results)
#line = filt_tanh(line, ou / float(nx), ou / float(nx))
# normalize this line
[mean_l, sigma_l, imin, imax] = Util.infomask(line, None, True)
line = (line - mean_l) / sigma_l
# fft
fftip(line)
# filter (cut part of coef) and create mirror line
Util.cml_prepare_line(prj, line, ilf, ihf, li, nye)
# store the projection
Prj.append(prj)
return Prj, Ori
def main():
"""
Main function.
Arguments:
None
Returns:
None
"""
command_args = parse_command_line()
# Import volume
sp_global_def.sxprint("Import volume.")
input_vol = sp_utilities.get_im(command_args.input_volume)
# Sanity checks
sanity_checks(command_args, input_vol)
try:
os.makedirs(command_args.output_dir)
except OSError:
sp_global_def.sxprint(
"Output directory already exists. No need to create it.")
else:
sp_global_def.sxprint("Created output directory.")
sp_global_def.write_command(command_args.output_dir)
output_prefix = os.path.join(command_args.output_dir, command_args.prefix)
# Filter volume if specified
if command_args.low_pass_filter_resolution is not None:
sp_global_def.sxprint("Filter volume to {0}A.".format(
command_args.low_pass_filter_resolution))
input_vol = sp_filter.filt_tanl(
input_vol,
old_div(command_args.pixel_size,
command_args.low_pass_filter_resolution),
command_args.low_pass_filter_falloff,
)
input_vol.write_image(output_prefix + "_filtered_volume.hdf")
else:
sp_global_def.sxprint("Skip filter volume.")
# Create a mask based on the filtered volume
sp_global_def.sxprint("Create mask")
density_threshold = -9999.0
nsigma = 1.0
if command_args.mol_mass:
density_threshold = input_vol.find_3d_threshold(
command_args.mol_mass, command_args.pixel_size)
sp_global_def.sxprint(
"Mask molecular mass translated into binary threshold: ",
density_threshold)
elif command_args.threshold:
density_threshold = command_args.threshold
elif command_args.nsigma:
nsigma = command_args.nsigma
else:
assert False
if command_args.edge_type == "cosine":
mode = "C"
elif command_args.edge_type == "gaussian":
mode = "G"
else:
assert False
mask_first = sp_morphology.adaptive_mask_scipy(
input_vol,
nsigma=nsigma,
threshold=density_threshold,
ndilation=command_args.ndilation,
nerosion=command_args.nerosion,
edge_width=command_args.edge_width,
allow_disconnected=command_args.allow_disconnected,
mode=mode,
do_approx=command_args.do_old,
do_fill=command_args.fill_mask,
do_print=True,
)
# Create a second mask based on the filtered volume
s_mask = None
s_density_threshold = 1
s_nsigma = 1.0
if command_args.second_mask is not None:
sp_global_def.sxprint("Prepare second mask")
s_mask = sp_utilities.get_im(command_args.second_mask)
s_density_threshold = -9999.0
s_nsigma = 1.0
if command_args.s_mol_mass:
s_density_threshold = input_vol.find_3d_threshold(
command_args.s_mol_mass, command_args.s_pixel_size)
sp_global_def.sxprint(
"Second mask molecular mass translated into binary threshold: ",
s_density_threshold,
)
elif command_args.s_threshold:
s_density_threshold = command_args.s_threshold
elif command_args.s_nsigma:
s_nsigma = command_args.s_nsigma
else:
assert False
elif command_args.second_mask_shape is not None:
sp_global_def.sxprint("Prepare second mask")
nx = mask_first.get_xsize()
ny = mask_first.get_ysize()
nz = mask_first.get_zsize()
if command_args.second_mask_shape == "cube":
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
s_mask = sp_utilities.model_blank(s_nx, s_ny, s_nz, 1)
elif command_args.second_mask_shape == "cylinder":
s_radius = command_args.s_radius
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
try:
s_mask = sp_utilities.model_cylinder(s_radius, s_nx, s_ny,
s_nz)
except RuntimeError as e:
sp_global_def.sxprint(
"An error occured! Please check the error log")
raise
elif command_args.second_mask_shape == "sphere":
s_radius = command_args.s_radius
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
try:
s_mask = sp_utilities.model_circle(s_radius, s_nx, s_ny, s_nz)
except RuntimeError as e:
sp_global_def.sxprint(
"An error occured! Please check the error log")
raise
else:
assert False
s_mask = sp_utilities.pad(s_mask, nx, ny, nz, 0)
if s_mask is not None:
sp_global_def.sxprint("Create second mask")
if command_args.s_edge_type == "cosine":
mode = "C"
elif command_args.s_edge_type == "gaussian":
mode = "G"
else:
assert False
s_mask = sp_morphology.adaptive_mask_scipy(
s_mask,
nsigma=s_nsigma,
threshold=s_density_threshold,
ndilation=command_args.s_ndilation,
nerosion=command_args.s_nerosion,
edge_width=command_args.s_edge_width,
allow_disconnected=command_args.s_allow_disconnected,
mode=mode,
do_approx=command_args.s_do_old,
do_fill=command_args.s_fill_mask,
do_print=True,
)
if command_args.s_invert:
s_mask = 1 - s_mask
sp_global_def.sxprint("Write outputs.")
mask_first.write_image(output_prefix + "_mask_first.hdf")
s_mask.write_image(output_prefix + "_mask_second.hdf")
masked_combined = mask_first * s_mask
masked_combined.write_image(output_prefix + "_mask.hdf")
else:
sp_global_def.sxprint("Write outputs.")
mask_first.write_image(output_prefix + "_mask.hdf")
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = progname + """ firstvolume secondvolume maskfile directory --prefix --wn --step --cutoff --radius --fsc --res_overall --out_ang_res --apix --MPI
Compute local resolution in real space within area outlined by the maskfile and within regions wn x wn x wn
"""
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option("--prefix",
type="str",
default='localres',
help="Prefix for the output files. (default localres)")
parser.add_option(
"--wn",
type="int",
default=7,
help=
"Size of window within which local real-space FSC is computed. (default 7)"
)
parser.add_option(
"--step",
type="float",
default=1.0,
help="Shell step in Fourier size in pixels. (default 1.0)")
parser.add_option("--cutoff",
type="float",
default=0.143,
help="Resolution cut-off for FSC. (default 0.143)")
parser.add_option(
"--radius",
type="int",
default=-1,
help=
"If there is no maskfile, sphere with r=radius will be used. By default, the radius is nx/2-wn (default -1)"
)
parser.add_option(
"--fsc",
type="string",
default=None,
help=
"Save overall FSC curve (might be truncated). By default, the program does not save the FSC curve. (default none)"
)
parser.add_option(
"--res_overall",
type="float",
default=-1.0,
help=
"Overall resolution at the cutoff level estimated by the user [abs units]. (default None)"
)
parser.add_option(
"--out_ang_res",
action="store_true",
default=False,
help=
"Additionally creates a local resolution file in Angstroms. (default False)"
)
parser.add_option(
"--apix",
type="float",
default=1.0,
help=
"Pixel size in Angstrom. Effective only with --out_ang_res options. (default 1.0)"
)
parser.add_option("--MPI",
action="store_true",
default=False,
help="Use MPI version.")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 3 or len(args) > 4:
sxprint("Usage: " + usage)
ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
if sp_global_def.CACHE_DISABLE:
sp_utilities.disable_bdb_cache()
res_overall = options.res_overall
if options.MPI:
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
sp_global_def.MPI = True
cutoff = options.cutoff
nk = int(options.wn)
if (myid == main_node):
#print sys.argv
vi = sp_utilities.get_im(sys.argv[1])
ui = sp_utilities.get_im(sys.argv[2])
nx = vi.get_xsize()
ny = vi.get_ysize()
nz = vi.get_zsize()
dis = [nx, ny, nz]
else:
dis = [0, 0, 0, 0]
sp_global_def.BATCH = True
dis = sp_utilities.bcast_list_to_all(dis, myid, source_node=main_node)
if (myid != main_node):
nx = int(dis[0])
ny = int(dis[1])
nz = int(dis[2])
vi = sp_utilities.model_blank(nx, ny, nz)
ui = sp_utilities.model_blank(nx, ny, nz)
if len(args) == 3:
m = sp_utilities.model_circle((min(nx, ny, nz) - nk) // 2, nx, ny,
nz)
outdir = args[2]
elif len(args) == 4:
if (myid == main_node):
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
else:
m = sp_utilities.model_blank(nx, ny, nz)
outdir = args[3]
if os.path.exists(outdir) and myid == 0:
sp_global_def.ERROR('Output directory already exists!')
elif myid == 0:
os.makedirs(outdir)
sp_global_def.write_command(outdir)
sp_utilities.bcast_EMData_to_all(m, myid, main_node)
"""Multiline Comment0"""
freqvol, resolut = sp_statistics.locres(vi, ui, m, nk, cutoff,
options.step, myid, main_node,
number_of_proc)
if (myid == 0):
# Remove outliers based on the Interquartile range
output_volume(freqvol, resolut, options.apix, outdir,
options.prefix, options.fsc, options.out_ang_res, nx,
ny, nz, res_overall)
else:
cutoff = options.cutoff
vi = sp_utilities.get_im(args[0])
ui = sp_utilities.get_im(args[1])
nn = vi.get_xsize()
nx = nn
ny = nn
nz = nn
nk = int(options.wn)
if len(args) == 3:
m = sp_utilities.model_circle((nn - nk) // 2, nn, nn, nn)
outdir = args[2]
elif len(args) == 4:
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
outdir = args[3]
if os.path.exists(outdir):
sp_global_def.ERROR('Output directory already exists!')
else:
os.makedirs(outdir)
sp_global_def.write_command(outdir)
mc = sp_utilities.model_blank(nn, nn, nn, 1.0) - m
vf = sp_fundamentals.fft(vi)
uf = sp_fundamentals.fft(ui)
"""Multiline Comment1"""
lp = int(nn / 2 / options.step + 0.5)
step = 0.5 / lp
freqvol = sp_utilities.model_blank(nn, nn, nn)
resolut = []
for i in range(1, lp):
fl = step * i
fh = fl + step
#print(lp,i,step,fl,fh)
v = sp_fundamentals.fft(sp_filter.filt_tophatb(vf, fl, fh))
u = sp_fundamentals.fft(sp_filter.filt_tophatb(uf, fl, fh))
tmp1 = EMAN2_cppwrap.Util.muln_img(v, v)
tmp2 = EMAN2_cppwrap.Util.muln_img(u, u)
do = EMAN2_cppwrap.Util.infomask(
sp_morphology.square_root(
sp_morphology.threshold(
EMAN2_cppwrap.Util.muln_img(tmp1, tmp2))), m, True)[0]
tmp3 = EMAN2_cppwrap.Util.muln_img(u, v)
dp = EMAN2_cppwrap.Util.infomask(tmp3, m, True)[0]
resolut.append([i, (fl + fh) / 2.0, dp / do])
tmp1 = EMAN2_cppwrap.Util.box_convolution(tmp1, nk)
tmp2 = EMAN2_cppwrap.Util.box_convolution(tmp2, nk)
tmp3 = EMAN2_cppwrap.Util.box_convolution(tmp3, nk)
EMAN2_cppwrap.Util.mul_img(tmp1, tmp2)
tmp1 = sp_morphology.square_root(sp_morphology.threshold(tmp1))
EMAN2_cppwrap.Util.mul_img(tmp1, m)
EMAN2_cppwrap.Util.add_img(tmp1, mc)
EMAN2_cppwrap.Util.mul_img(tmp3, m)
EMAN2_cppwrap.Util.add_img(tmp3, mc)
EMAN2_cppwrap.Util.div_img(tmp3, tmp1)
EMAN2_cppwrap.Util.mul_img(tmp3, m)
freq = (fl + fh) / 2.0
bailout = True
for x in range(nn):
for y in range(nn):
for z in range(nn):
if (m.get_value_at(x, y, z) > 0.5):
if (freqvol.get_value_at(x, y, z) == 0.0):
if (tmp3.get_value_at(x, y, z) < cutoff):
freqvol.set_value_at(x, y, z, freq)
bailout = False
else:
bailout = False
if (bailout): break
#print(len(resolut))
# remove outliers
output_volume(freqvol, resolut, options.apix, outdir, options.prefix,
options.fsc, options.out_ang_res, nx, ny, nz,
res_overall)
def test_(self):
out_dir_old = "oldmeridien"
out_dir_new = "newmeridien"
mkdir(out_dir_old)
mkdir(out_dir_new)
testcommand_old = (
MPI_PATH + " -np " + str(NUM_PROC) + " " +
path.join(ABSOLUTE_OLDBIN_PATH, "sp_meridien.py") + " " +
" 'bdb:" +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"06_SUBSTACK_ANO#isac_substack'") + " " +
self.out_dir_old + " " +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"09_ADJUSTMENT", "vol3d_ref_moon_eliminated_mask.hdf") +
" --initialshifts" + " --radius=145" + " --mask3D=" +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"10_MASK", "sp_mask_mask.hdf") + " --symmetry=c5")
testcommand_new = (
MPI_PATH + " -np " + str(NUM_PROC) + " " +
path.join(ABSOLUTE_BIN_PATH, "sp_meridien.py") + " 'bdb:" +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"06_SUBSTACK_ANO#isac_substack'") + " " +
self.out_dir_new + " " +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"09_ADJUSTMENT", "vol3d_ref_moon_eliminated_mask.hdf") +
" --initialshifts" + " --radius=145" + " --mask3D=" +
path.join(ABSOLUTE_PATH_TO_SPHIRE_DEMO_RESULTS_FOLDER_NEW,
"10_MASK", "sp_mask_mask.hdf") + " --symmetry=c5")
# as they are ordered alphanumerically the first is always vol0
subprocess.run(args=[testcommand_old],
shell=True,
stderr=subprocess.STDOUT)
subprocess.run(args=[testcommand_new],
shell=True,
stderr=subprocess.STDOUT)
hdfFilesOld = [
f for f in listdir(path.join(ABSOLUTE_PATH, self.out_dir_old))
if path.isfile(path.join(ABSOLUTE_PATH, self.out_dir_old, f))
and f.split(".")[1] == "hdf"
]
hdfFilesNew = [
f for f in listdir(path.join(ABSOLUTE_PATH, self.out_dir_old))
if path.isfile(path.join(ABSOLUTE_PATH, self.out_dir_new, f))
and f.split(".")[1] == "hdf"
]
old_v0 = get_im(
path.join(ABSOLUTE_PATH, self.out_dir_old, hdfFilesOld[0]))
old_v1 = get_im(
path.join(ABSOLUTE_PATH, self.out_dir_old, hdfFilesOld[1]))
new_v0 = get_im(
path.join(ABSOLUTE_PATH, self.out_dir_new, hdfFilesNew[0]))
new_v1 = get_im(
path.join(ABSOLUTE_PATH, self.out_dir_new, hdfFilesNew[1]))
self.assertTrue(
allclose(old_v0.get_3dview(), new_v0.get_3dview(), atol=TOLERANCE))
self.assertTrue(
allclose(old_v1.get_3dview(), new_v1.get_3dview(), atol=TOLERANCE))
self.assertTrue(
allclose(new_v0.get_3dview().flatten().tolist()[0:100], [
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539,
0.000977565417997539, 0.000977565417997539
],
atol=0.005))
self.assertTrue(
allclose(new_v1.get_3dview().flatten().tolist()[0:100], [
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855,
0.0011777321342378855, 0.0011777321342378855
],
atol=0.005))
def main():
global Tracker, Blockdata
progname = os.path.basename(sys.argv[0])
usage = progname + " --output_dir=output_dir --isac_dir=output_dir_of_isac "
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--pw_adjustment",
type="string",
default="analytical_model",
help=
"adjust power spectrum of 2-D averages to an analytic model. Other opions: no_adjustment; bfactor; a text file of 1D rotationally averaged PW",
)
#### Four options for --pw_adjustment:
# 1> analytical_model(default);
# 2> no_adjustment;
# 3> bfactor;
# 4> adjust_to_given_pw2(user has to provide a text file that contains 1D rotationally averaged PW)
# options in common
parser.add_option(
"--isac_dir",
type="string",
default="",
help="ISAC run output directory, input directory for this command",
)
parser.add_option(
"--output_dir",
type="string",
default="",
help="output directory where computed averages are saved",
)
parser.add_option(
"--pixel_size",
type="float",
default=-1.0,
help=
"pixel_size of raw images. one can put 1.0 in case of negative stain data",
)
parser.add_option(
"--fl",
type="float",
default=-1.0,
help=
"low pass filter, = -1.0, not applied; =0.0, using FH1 (initial resolution), = 1.0 using FH2 (resolution after local alignment), or user provided value in absolute freqency [0.0:0.5]",
)
parser.add_option("--stack",
type="string",
default="",
help="data stack used in ISAC")
parser.add_option("--radius", type="int", default=-1, help="radius")
parser.add_option("--xr",
type="float",
default=-1.0,
help="local alignment search range")
# parser.add_option("--ts", type ="float", default =1.0, help= "local alignment search step")
parser.add_option(
"--fh",
type="float",
default=-1.0,
help="local alignment high frequencies limit",
)
# parser.add_option("--maxit", type ="int", default =5, help= "local alignment iterations")
parser.add_option("--navg",
type="int",
default=1000000,
help="number of aveages")
parser.add_option(
"--local_alignment",
action="store_true",
default=False,
help="do local alignment",
)
parser.add_option(
"--noctf",
action="store_true",
default=False,
help=
"no ctf correction, useful for negative stained data. always ctf for cryo data",
)
parser.add_option(
"--B_start",
type="float",
default=45.0,
help=
"start frequency (Angstrom) of power spectrum for B_factor estimation",
)
parser.add_option(
"--Bfactor",
type="float",
default=-1.0,
help=
"User defined bactors (e.g. 25.0[A^2]). By default, the program automatically estimates B-factor. ",
)
(options, args) = parser.parse_args(sys.argv[1:])
adjust_to_analytic_model = (True if options.pw_adjustment
== "analytical_model" else False)
no_adjustment = True if options.pw_adjustment == "no_adjustment" else False
B_enhance = True if options.pw_adjustment == "bfactor" else False
adjust_to_given_pw2 = (
True if not (adjust_to_analytic_model or no_adjustment or B_enhance)
else False)
# mpi
nproc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
Blockdata = {}
Blockdata["nproc"] = nproc
Blockdata["myid"] = myid
Blockdata["main_node"] = 0
Blockdata["shared_comm"] = mpi.mpi_comm_split_type(
mpi.MPI_COMM_WORLD, mpi.MPI_COMM_TYPE_SHARED, 0, mpi.MPI_INFO_NULL)
Blockdata["myid_on_node"] = mpi.mpi_comm_rank(Blockdata["shared_comm"])
Blockdata["no_of_processes_per_group"] = mpi.mpi_comm_size(
Blockdata["shared_comm"])
masters_from_groups_vs_everything_else_comm = mpi.mpi_comm_split(
mpi.MPI_COMM_WORLD,
Blockdata["main_node"] == Blockdata["myid_on_node"],
Blockdata["myid_on_node"],
)
Blockdata["color"], Blockdata[
"no_of_groups"], balanced_processor_load_on_nodes = sp_utilities.get_colors_and_subsets(
Blockdata["main_node"],
mpi.MPI_COMM_WORLD,
Blockdata["myid"],
Blockdata["shared_comm"],
Blockdata["myid_on_node"],
masters_from_groups_vs_everything_else_comm,
)
# We need two nodes for processing of volumes
Blockdata["node_volume"] = [
Blockdata["no_of_groups"] - 3,
Blockdata["no_of_groups"] - 2,
Blockdata["no_of_groups"] - 1,
] # For 3D stuff take three last nodes
# We need two CPUs for processing of volumes, they are taken to be main CPUs on each volume
# We have to send the two myids to all nodes so we can identify main nodes on two selected groups.
Blockdata["nodes"] = [
Blockdata["node_volume"][0] * Blockdata["no_of_processes_per_group"],
Blockdata["node_volume"][1] * Blockdata["no_of_processes_per_group"],
Blockdata["node_volume"][2] * Blockdata["no_of_processes_per_group"],
]
# End of Blockdata: sorting requires at least three nodes, and the used number of nodes be integer times of three
sp_global_def.BATCH = True
sp_global_def.MPI = True
if adjust_to_given_pw2:
checking_flag = 0
if Blockdata["myid"] == Blockdata["main_node"]:
if not os.path.exists(options.pw_adjustment):
checking_flag = 1
checking_flag = sp_utilities.bcast_number_to_all(
checking_flag, Blockdata["main_node"], mpi.MPI_COMM_WORLD)
if checking_flag == 1:
sp_global_def.ERROR("User provided power spectrum does not exist",
myid=Blockdata["myid"])
Tracker = {}
Constants = {}
Constants["isac_dir"] = options.isac_dir
Constants["masterdir"] = options.output_dir
Constants["pixel_size"] = options.pixel_size
Constants["orgstack"] = options.stack
Constants["radius"] = options.radius
Constants["xrange"] = options.xr
Constants["FH"] = options.fh
Constants["low_pass_filter"] = options.fl
# Constants["maxit"] = options.maxit
Constants["navg"] = options.navg
Constants["B_start"] = options.B_start
Constants["Bfactor"] = options.Bfactor
if adjust_to_given_pw2:
Constants["modelpw"] = options.pw_adjustment
Tracker["constants"] = Constants
# -------------------------------------------------------------
#
# Create and initialize Tracker dictionary with input options # State Variables
# <<<---------------------->>>imported functions<<<---------------------------------------------
# x_range = max(Tracker["constants"]["xrange"], int(1./Tracker["ini_shrink"])+1)
# y_range = x_range
####-----------------------------------------------------------
# Create Master directory and associated subdirectories
line = time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()) + " =>"
if Tracker["constants"]["masterdir"] == Tracker["constants"]["isac_dir"]:
masterdir = os.path.join(Tracker["constants"]["isac_dir"], "sharpen")
else:
masterdir = Tracker["constants"]["masterdir"]
if Blockdata["myid"] == Blockdata["main_node"]:
msg = "Postprocessing ISAC 2D averages starts"
sp_global_def.sxprint(line, "Postprocessing ISAC 2D averages starts")
if not masterdir:
timestring = time.strftime("_%d_%b_%Y_%H_%M_%S", time.localtime())
masterdir = "sharpen_" + Tracker["constants"]["isac_dir"]
os.makedirs(masterdir)
else:
if os.path.exists(masterdir):
sp_global_def.sxprint("%s already exists" % masterdir)
else:
os.makedirs(masterdir)
sp_global_def.write_command(masterdir)
subdir_path = os.path.join(masterdir, "ali2d_local_params_avg")
if not os.path.exists(subdir_path):
os.mkdir(subdir_path)
subdir_path = os.path.join(masterdir, "params_avg")
if not os.path.exists(subdir_path):
os.mkdir(subdir_path)
li = len(masterdir)
else:
li = 0
li = mpi.mpi_bcast(li, 1, mpi.MPI_INT, Blockdata["main_node"],
mpi.MPI_COMM_WORLD)[0]
masterdir = mpi.mpi_bcast(masterdir, li, mpi.MPI_CHAR,
Blockdata["main_node"], mpi.MPI_COMM_WORLD)
masterdir = b"".join(masterdir).decode('latin1')
Tracker["constants"]["masterdir"] = masterdir
log_main = sp_logger.Logger(sp_logger.BaseLogger_Files())
log_main.prefix = Tracker["constants"]["masterdir"] + "/"
while not os.path.exists(Tracker["constants"]["masterdir"]):
sp_global_def.sxprint(
"Node ",
Blockdata["myid"],
" waiting...",
Tracker["constants"]["masterdir"],
)
time.sleep(1)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if Blockdata["myid"] == Blockdata["main_node"]:
init_dict = {}
sp_global_def.sxprint(Tracker["constants"]["isac_dir"])
Tracker["directory"] = os.path.join(Tracker["constants"]["isac_dir"],
"2dalignment")
core = sp_utilities.read_text_row(
os.path.join(Tracker["directory"], "initial2Dparams.txt"))
for im in range(len(core)):
init_dict[im] = core[im]
del core
else:
init_dict = 0
init_dict = sp_utilities.wrap_mpi_bcast(init_dict,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
###
do_ctf = True
if options.noctf:
do_ctf = False
if Blockdata["myid"] == Blockdata["main_node"]:
if do_ctf:
sp_global_def.sxprint("CTF correction is on")
else:
sp_global_def.sxprint("CTF correction is off")
if options.local_alignment:
sp_global_def.sxprint("local refinement is on")
else:
sp_global_def.sxprint("local refinement is off")
if B_enhance:
sp_global_def.sxprint("Bfactor is to be applied on averages")
elif adjust_to_given_pw2:
sp_global_def.sxprint(
"PW of averages is adjusted to a given 1D PW curve")
elif adjust_to_analytic_model:
sp_global_def.sxprint(
"PW of averages is adjusted to analytical model")
else:
sp_global_def.sxprint("PW of averages is not adjusted")
# Tracker["constants"]["orgstack"] = "bdb:"+ os.path.join(Tracker["constants"]["isac_dir"],"../","sparx_stack")
image = sp_utilities.get_im(Tracker["constants"]["orgstack"], 0)
Tracker["constants"]["nnxo"] = image.get_xsize()
if Tracker["constants"]["pixel_size"] == -1.0:
sp_global_def.sxprint(
"Pixel size value is not provided by user. extracting it from ctf header entry of the original stack."
)
try:
ctf_params = image.get_attr("ctf")
Tracker["constants"]["pixel_size"] = ctf_params.apix
except:
sp_global_def.ERROR(
"Pixel size could not be extracted from the original stack.",
myid=Blockdata["myid"],
)
## Now fill in low-pass filter
isac_shrink_path = os.path.join(Tracker["constants"]["isac_dir"],
"README_shrink_ratio.txt")
if not os.path.exists(isac_shrink_path):
sp_global_def.ERROR(
"%s does not exist in the specified ISAC run output directory"
% (isac_shrink_path),
myid=Blockdata["myid"],
)
isac_shrink_file = open(isac_shrink_path, "r")
isac_shrink_lines = isac_shrink_file.readlines()
isac_shrink_ratio = float(
isac_shrink_lines[5]
) # 6th line: shrink ratio (= [target particle radius]/[particle radius]) used in the ISAC run
isac_radius = float(
isac_shrink_lines[6]
) # 7th line: particle radius at original pixel size used in the ISAC run
isac_shrink_file.close()
print("Extracted parameter values")
print("ISAC shrink ratio : {0}".format(isac_shrink_ratio))
print("ISAC particle radius : {0}".format(isac_radius))
Tracker["ini_shrink"] = isac_shrink_ratio
else:
Tracker["ini_shrink"] = 0.0
Tracker = sp_utilities.wrap_mpi_bcast(Tracker,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
# print(Tracker["constants"]["pixel_size"], "pixel_size")
x_range = max(
Tracker["constants"]["xrange"],
int(old_div(1.0, Tracker["ini_shrink"]) + 0.99999),
)
a_range = y_range = x_range
if Blockdata["myid"] == Blockdata["main_node"]:
parameters = sp_utilities.read_text_row(
os.path.join(Tracker["constants"]["isac_dir"],
"all_parameters.txt"))
else:
parameters = 0
parameters = sp_utilities.wrap_mpi_bcast(parameters,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
params_dict = {}
list_dict = {}
# parepare params_dict
# navg = min(Tracker["constants"]["navg"]*Blockdata["nproc"], EMUtil.get_image_count(os.path.join(Tracker["constants"]["isac_dir"], "class_averages.hdf")))
navg = min(
Tracker["constants"]["navg"],
EMAN2_cppwrap.EMUtil.get_image_count(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf")),
)
global_dict = {}
ptl_list = []
memlist = []
if Blockdata["myid"] == Blockdata["main_node"]:
sp_global_def.sxprint("Number of averages computed in this run is %d" %
navg)
for iavg in range(navg):
params_of_this_average = []
image = sp_utilities.get_im(
os.path.join(Tracker["constants"]["isac_dir"],
"class_averages.hdf"),
iavg,
)
members = sorted(image.get_attr("members"))
memlist.append(members)
for im in range(len(members)):
abs_id = members[im]
global_dict[abs_id] = [iavg, im]
P = sp_utilities.combine_params2(
init_dict[abs_id][0],
init_dict[abs_id][1],
init_dict[abs_id][2],
init_dict[abs_id][3],
parameters[abs_id][0],
old_div(parameters[abs_id][1], Tracker["ini_shrink"]),
old_div(parameters[abs_id][2], Tracker["ini_shrink"]),
parameters[abs_id][3],
)
if parameters[abs_id][3] == -1:
sp_global_def.sxprint(
"WARNING: Image #{0} is an unaccounted particle with invalid 2D alignment parameters and should not be the member of any classes. Please check the consitency of input dataset."
.format(abs_id)
) # How to check what is wrong about mirror = -1 (Toshio 2018/01/11)
params_of_this_average.append([P[0], P[1], P[2], P[3], 1.0])
ptl_list.append(abs_id)
params_dict[iavg] = params_of_this_average
list_dict[iavg] = members
sp_utilities.write_text_row(
params_of_this_average,
os.path.join(
Tracker["constants"]["masterdir"],
"params_avg",
"params_avg_%03d.txt" % iavg,
),
)
ptl_list.sort()
init_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
init_params[im] = [ptl_list[im]] + params_dict[global_dict[
ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
sp_utilities.write_text_row(
init_params,
os.path.join(Tracker["constants"]["masterdir"],
"init_isac_params.txt"),
)
else:
params_dict = 0
list_dict = 0
memlist = 0
params_dict = sp_utilities.wrap_mpi_bcast(params_dict,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
list_dict = sp_utilities.wrap_mpi_bcast(list_dict,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
memlist = sp_utilities.wrap_mpi_bcast(memlist,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
# Now computing!
del init_dict
tag_sharpen_avg = 1000
## always apply low pass filter to B_enhanced images to suppress noise in high frequencies
enforced_to_H1 = False
if B_enhance:
if Tracker["constants"]["low_pass_filter"] == -1.0:
enforced_to_H1 = True
# distribute workload among mpi processes
image_start, image_end = sp_applications.MPI_start_end(
navg, Blockdata["nproc"], Blockdata["myid"])
if Blockdata["myid"] == Blockdata["main_node"]:
cpu_dict = {}
for iproc in range(Blockdata["nproc"]):
local_image_start, local_image_end = sp_applications.MPI_start_end(
navg, Blockdata["nproc"], iproc)
for im in range(local_image_start, local_image_end):
cpu_dict[im] = iproc
else:
cpu_dict = 0
cpu_dict = sp_utilities.wrap_mpi_bcast(cpu_dict,
Blockdata["main_node"],
communicator=mpi.MPI_COMM_WORLD)
slist = [None for im in range(navg)]
ini_list = [None for im in range(navg)]
avg1_list = [None for im in range(navg)]
avg2_list = [None for im in range(navg)]
data_list = [None for im in range(navg)]
plist_dict = {}
if Blockdata["myid"] == Blockdata["main_node"]:
if B_enhance:
sp_global_def.sxprint(
"Avg ID B-factor FH1(Res before ali) FH2(Res after ali)")
else:
sp_global_def.sxprint(
"Avg ID FH1(Res before ali) FH2(Res after ali)")
FH_list = [[0, 0.0, 0.0] for im in range(navg)]
for iavg in range(image_start, image_end):
mlist = EMAN2_cppwrap.EMData.read_images(
Tracker["constants"]["orgstack"], list_dict[iavg])
for im in range(len(mlist)):
sp_utilities.set_params2D(mlist[im],
params_dict[iavg][im],
xform="xform.align2d")
if options.local_alignment:
new_avg, plist, FH2 = sp_applications.refinement_2d_local(
mlist,
Tracker["constants"]["radius"],
a_range,
x_range,
y_range,
CTF=do_ctf,
SNR=1.0e10,
)
plist_dict[iavg] = plist
FH1 = -1.0
else:
new_avg, frc, plist = compute_average(
mlist, Tracker["constants"]["radius"], do_ctf)
FH1 = get_optimistic_res(frc)
FH2 = -1.0
FH_list[iavg] = [iavg, FH1, FH2]
if B_enhance:
new_avg, gb = apply_enhancement(
new_avg,
Tracker["constants"]["B_start"],
Tracker["constants"]["pixel_size"],
Tracker["constants"]["Bfactor"],
)
sp_global_def.sxprint(" %6d %6.3f %4.3f %4.3f" %
(iavg, gb, FH1, FH2))
elif adjust_to_given_pw2:
roo = sp_utilities.read_text_file(Tracker["constants"]["modelpw"],
-1)
roo = roo[0] # always on the first column
new_avg = adjust_pw_to_model(new_avg,
Tracker["constants"]["pixel_size"],
roo)
sp_global_def.sxprint(" %6d %4.3f %4.3f " %
(iavg, FH1, FH2))
elif adjust_to_analytic_model:
new_avg = adjust_pw_to_model(new_avg,
Tracker["constants"]["pixel_size"],
None)
sp_global_def.sxprint(" %6d %4.3f %4.3f " %
(iavg, FH1, FH2))
elif no_adjustment:
pass
if Tracker["constants"]["low_pass_filter"] != -1.0:
if Tracker["constants"]["low_pass_filter"] == 0.0:
low_pass_filter = FH1
elif Tracker["constants"]["low_pass_filter"] == 1.0:
low_pass_filter = FH2
if not options.local_alignment:
low_pass_filter = FH1
else:
low_pass_filter = Tracker["constants"]["low_pass_filter"]
if low_pass_filter >= 0.45:
low_pass_filter = 0.45
new_avg = sp_filter.filt_tanl(new_avg, low_pass_filter, 0.02)
else: # No low pass filter but if enforced
if enforced_to_H1:
new_avg = sp_filter.filt_tanl(new_avg, FH1, 0.02)
if B_enhance:
new_avg = sp_fundamentals.fft(new_avg)
new_avg.set_attr("members", list_dict[iavg])
new_avg.set_attr("n_objects", len(list_dict[iavg]))
slist[iavg] = new_avg
sp_global_def.sxprint(
time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()) + " =>",
"Refined average %7d" % iavg,
)
## send to main node to write
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
for im in range(navg):
# avg
if (cpu_dict[im] == Blockdata["myid"]
and Blockdata["myid"] != Blockdata["main_node"]):
sp_utilities.send_EMData(slist[im], Blockdata["main_node"],
tag_sharpen_avg)
elif (cpu_dict[im] == Blockdata["myid"]
and Blockdata["myid"] == Blockdata["main_node"]):
slist[im].set_attr("members", memlist[im])
slist[im].set_attr("n_objects", len(memlist[im]))
slist[im].write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"),
im,
)
elif (cpu_dict[im] != Blockdata["myid"]
and Blockdata["myid"] == Blockdata["main_node"]):
new_avg_other_cpu = sp_utilities.recv_EMData(
cpu_dict[im], tag_sharpen_avg)
new_avg_other_cpu.set_attr("members", memlist[im])
new_avg_other_cpu.set_attr("n_objects", len(memlist[im]))
new_avg_other_cpu.write_image(
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"),
im,
)
if options.local_alignment:
if cpu_dict[im] == Blockdata["myid"]:
sp_utilities.write_text_row(
plist_dict[im],
os.path.join(
Tracker["constants"]["masterdir"],
"ali2d_local_params_avg",
"ali2d_local_params_avg_%03d.txt" % im,
),
)
if (cpu_dict[im] == Blockdata["myid"]
and cpu_dict[im] != Blockdata["main_node"]):
sp_utilities.wrap_mpi_send(plist_dict[im],
Blockdata["main_node"],
mpi.MPI_COMM_WORLD)
sp_utilities.wrap_mpi_send(FH_list, Blockdata["main_node"],
mpi.MPI_COMM_WORLD)
elif (cpu_dict[im] != Blockdata["main_node"]
and Blockdata["myid"] == Blockdata["main_node"]):
dummy = sp_utilities.wrap_mpi_recv(cpu_dict[im],
mpi.MPI_COMM_WORLD)
plist_dict[im] = dummy
dummy = sp_utilities.wrap_mpi_recv(cpu_dict[im],
mpi.MPI_COMM_WORLD)
FH_list[im] = dummy[im]
else:
if (cpu_dict[im] == Blockdata["myid"]
and cpu_dict[im] != Blockdata["main_node"]):
sp_utilities.wrap_mpi_send(FH_list, Blockdata["main_node"],
mpi.MPI_COMM_WORLD)
elif (cpu_dict[im] != Blockdata["main_node"]
and Blockdata["myid"] == Blockdata["main_node"]):
dummy = sp_utilities.wrap_mpi_recv(cpu_dict[im],
mpi.MPI_COMM_WORLD)
FH_list[im] = dummy[im]
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if options.local_alignment:
if Blockdata["myid"] == Blockdata["main_node"]:
ali3d_local_params = [None for im in range(len(ptl_list))]
for im in range(len(ptl_list)):
ali3d_local_params[im] = [ptl_list[im]] + plist_dict[
global_dict[ptl_list[im]][0]][global_dict[ptl_list[im]][1]]
sp_utilities.write_text_row(
ali3d_local_params,
os.path.join(Tracker["constants"]["masterdir"],
"ali2d_local_params.txt"),
)
sp_utilities.write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
else:
if Blockdata["myid"] == Blockdata["main_node"]:
sp_utilities.write_text_row(
FH_list,
os.path.join(Tracker["constants"]["masterdir"], "FH_list.txt"))
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
target_xr = 3
target_yr = 3
if Blockdata["myid"] == 0:
cmd = "{} {} {} {} {} {} {} {} {} {}".format(
"sp_chains.py",
os.path.join(Tracker["constants"]["masterdir"],
"class_averages.hdf"),
os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"),
os.path.join(Tracker["constants"]["masterdir"],
"ordered_class_averages.hdf"),
"--circular",
"--radius=%d" % Tracker["constants"]["radius"],
"--xr=%d" % (target_xr + 1),
"--yr=%d" % (target_yr + 1),
"--align",
">/dev/null",
)
junk = sp_utilities.cmdexecute(cmd)
cmd = "{} {}".format(
"rm -rf",
os.path.join(Tracker["constants"]["masterdir"], "junk.hdf"))
junk = sp_utilities.cmdexecute(cmd)
return
def helicalshiftali_MPI(stack,
maskfile=None,
maxit=100,
CTF=False,
snr=1.0,
Fourvar=False,
search_rng=-1):
nproc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("helical-shiftali_MPI")
max_iter = int(maxit)
if (myid == main_node):
infils = EMUtil.get_all_attributes(stack, "filament")
ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord')
filaments = ordersegments(infils, ptlcoords)
total_nfils = len(filaments)
inidl = [0] * total_nfils
for i in range(total_nfils):
inidl[i] = len(filaments[i])
linidl = sum(inidl)
nima = linidl
tfilaments = []
for i in range(total_nfils):
tfilaments += filaments[i]
del filaments
else:
total_nfils = 0
linidl = 0
total_nfils = bcast_number_to_all(total_nfils, source_node=main_node)
if myid != main_node:
inidl = [-1] * total_nfils
inidl = bcast_list_to_all(inidl, myid, source_node=main_node)
linidl = bcast_number_to_all(linidl, source_node=main_node)
if myid != main_node:
tfilaments = [-1] * linidl
tfilaments = bcast_list_to_all(tfilaments, myid, source_node=main_node)
filaments = []
iendi = 0
for i in range(total_nfils):
isti = iendi
iendi = isti + inidl[i]
filaments.append(tfilaments[isti:iendi])
del tfilaments, inidl
if myid == main_node:
print_msg("total number of filaments: %d" % total_nfils)
if total_nfils < nproc:
ERROR(
'number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'
% (nproc, total_nfils),
myid=myid)
# balanced load
temp = chunks_distribution([[len(filaments[i]), i]
for i in range(len(filaments))],
nproc)[myid:myid + 1][0]
filaments = [filaments[temp[i][1]] for i in range(len(temp))]
nfils = len(filaments)
#filaments = [[0,1]]
#print "filaments",filaments
list_of_particles = []
indcs = []
k = 0
for i in range(nfils):
list_of_particles += filaments[i]
k1 = k + len(filaments[i])
indcs.append([k, k1])
k = k1
data = EMData.read_images(stack, list_of_particles)
ldata = len(data)
sxprint("ldata=", ldata)
nx = data[0].get_xsize()
ny = data[0].get_ysize()
if maskfile == None:
mrad = min(nx, ny) // 2 - 2
mask = pad(model_blank(2 * mrad + 1, ny, 1, 1.0), nx, ny, 1, 0.0)
else:
mask = get_im(maskfile)
# apply initial xform.align2d parameters stored in header
init_params = []
for im in range(ldata):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'],
p['mirror'], p['scale'])
if CTF:
from sp_filter import filt_ctf
from sp_morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
ctf_abs_sum = None
from sp_utilities import info
for im in range(ldata):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
qctf = data[im].get_attr("ctf_applied")
if qctf == 0:
data[im] = filt_ctf(fft(data[im]), ctf_params)
data[im].set_attr('ctf_applied', 1)
elif qctf != 1:
ERROR('Incorrectly set qctf flag', myid=myid)
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
else:
data[im] = fft(data[im])
del list_of_particles
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
tsnr = 1. / snr
for i in range(0, nx + 2, 2):
for j in range(ny):
temp.set_value_at(i, j, tsnr)
temp.set_value_at(i + 1, j, 0.0)
#info(ctf_2_sum)
Util.add_img(ctf_2_sum, temp)
#info(ctf_2_sum)
del temp
total_iter = 0
shift_x = [0.0] * ldata
for Iter in range(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n" % (total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in range(ldata):
Util.add_img(avg, fshift(data[im], shift_x[im]))
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF: tavg = Util.divn_filter(avg, ctf_2_sum)
else: tavg = Util.mult_scalar(avg, 1.0 / float(nima))
else:
tavg = model_blank(nx, ny)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask(tavg, mask, False)
tavg -= stat[0]
Util.mul_img(tavg, mask)
tavg.write_image("tavg.hdf", Iter)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
tavg = fft(tavg)
sx_sum = 0.0
nxc = nx // 2
for ifil in range(nfils):
"""
# Calculate filament average
avg = EMData(nx, ny, 1, False)
filnima = 0
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
Util.add_img(avg, data[im])
filnima += 1
tavg = Util.mult_scalar(avg, 1.0/float(filnima))
"""
# Calculate 1D ccf between each segment and filament average
nsegms = indcs[ifil][1] - indcs[ifil][0]
ctx = [None] * nsegms
pcoords = [None] * nsegms
for im in range(indcs[ifil][0], indcs[ifil][1]):
ctx[im - indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx,
1)
pcoords[im - indcs[ifil][0]] = data[im].get_attr(
'ptcl_source_coord')
#ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0])
#print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True)
# search for best x-shift
cents = nsegms // 2
dst = sqrt(
max((pcoords[cents][0] - pcoords[0][0])**2 +
(pcoords[cents][1] - pcoords[0][1])**2,
(pcoords[cents][0] - pcoords[-1][0])**2 +
(pcoords[cents][1] - pcoords[-1][1])**2))
maxincline = atan2(ny // 2 - 2 - float(search_rng), dst)
kang = int(dst * tan(maxincline) + 0.5)
#print " settings ",nsegms,cents,dst,search_rng,maxincline,kang
# ## C code for alignment. @ming
results = [0.0] * 3
results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline,
kang, search_rng, nxc)
sib = int(results[0])
bang = results[1]
qm = results[2]
#print qm, sib, bang
# qm = -1.e23
#
# for six in xrange(-search_rng, search_rng+1,1):
# q0 = ctx[cents].get_value_at(six+nxc)
# for incline in xrange(kang+1):
# qt = q0
# qu = q0
# if(kang>0): tang = tan(maxincline/kang*incline)
# else: tang = 0.0
# for kim in xrange(cents+1,nsegms):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# #print " A ", ifil,six,incline,kim,xl,ixl,dxl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# for kim in xrange(cents):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# if( qt > qm ):
# qm = qt
# sib = six
# bang = tang
# if( qu > qm ):
# qm = qu
# sib = six
# bang = -tang
#if incline == 0: print "incline = 0 ",six,tang,qt,qu
#print qm,six,sib,bang
#print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True)
for im in range(indcs[ifil][0], indcs[ifil][1]):
kim = im - indcs[ifil][0]
dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 +
(pcoords[cents][1] - pcoords[kim][1])**2)
if (kim < cents): xl = -dst * bang + sib
else: xl = dst * bang + sib
shift_x[im] = xl
# Average shift
sx_sum += shift_x[indcs[ifil][0] + cents]
# #print myid,sx_sum,total_nfils
sx_sum = mpi.mpi_reduce(sx_sum, 1, mpi.MPI_FLOAT, mpi.MPI_SUM,
main_node, mpi.MPI_COMM_WORLD)
if myid == main_node:
sx_sum = float(sx_sum[0]) / total_nfils
print_msg("Average shift %6.2f\n" % (sx_sum))
else:
sx_sum = 0.0
sx_sum = 0.0
sx_sum = bcast_number_to_all(sx_sum, source_node=main_node)
for im in range(ldata):
shift_x[im] -= sx_sum
#print " %3d %6.3f"%(im,shift_x[im])
#exit()
# combine shifts found with the original parameters
for im in range(ldata):
t1 = Transform()
##import random
##shix=random.randint(-10, 10)
##t1.set_params({"type":"2D","tx":shix})
t1.set_params({"type": "2D", "tx": shift_x[im]})
# combine t0 and t1
tt = t1 * init_params[im]
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from sp_utilities import file_type
if (file_type(stack) == "bdb"):
from sp_utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata,
nproc)
else:
from sp_utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc)
else:
send_attr_dict(main_node, data, par_str, 0, ldata)
if myid == main_node: print_end_msg("helical-shiftali_MPI")
def main():
def params_3D_2D_NEW(phi, theta, psi, s2x, s2y, mirror):
# the final ali2d parameters already combine shifts operation first and rotation operation second for parameters converted from 3D
if mirror:
m = 1
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
540.0 - psi, 0, 0, 1.0)
else:
m = 0
alpha, sx, sy, scalen = compose_transform2(0, s2x, s2y, 1.0,
360.0 - psi, 0, 0, 1.0)
return alpha, sx, sy, m
progname = os.path.basename(sys.argv[0])
usage = progname + " prj_stack --ave2D= --var2D= --ave3D= --var3D= --img_per_grp= --fl= --aa= --sym=symmetry --CTF"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--output_dir",
type="string",
default="./",
help="Output directory")
parser.add_option("--ave2D",
type="string",
default=False,
help="Write to the disk a stack of 2D averages")
parser.add_option("--var2D",
type="string",
default=False,
help="Write to the disk a stack of 2D variances")
parser.add_option("--ave3D",
type="string",
default=False,
help="Write to the disk reconstructed 3D average")
parser.add_option("--var3D",
type="string",
default=False,
help="Compute 3D variability (time consuming!)")
parser.add_option(
"--img_per_grp",
type="int",
default=100,
help="Number of neighbouring projections.(Default is 100)")
parser.add_option(
"--no_norm",
action="store_true",
default=False,
help="Do not use normalization.(Default is to apply normalization)")
#parser.add_option("--radius", type="int" , default=-1 , help="radius for 3D variability" )
parser.add_option(
"--npad",
type="int",
default=2,
help=
"Number of time to pad the original images.(Default is 2 times padding)"
)
parser.add_option("--sym",
type="string",
default="c1",
help="Symmetry. (Default is no symmetry)")
parser.add_option(
"--fl",
type="float",
default=0.0,
help=
"Low pass filter cutoff in absolute frequency (0.0 - 0.5) and is applied to decimated images. (Default - no filtration)"
)
parser.add_option(
"--aa",
type="float",
default=0.02,
help=
"Fall off of the filter. Use default value if user has no clue about falloff (Default value is 0.02)"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="Use CFT correction.(Default is no CTF correction)")
#parser.add_option("--MPI" , action="store_true", default=False, help="use MPI version")
#parser.add_option("--radiuspca", type="int" , default=-1 , help="radius for PCA" )
#parser.add_option("--iter", type="int" , default=40 , help="maximum number of iterations (stop criterion of reconstruction process)" )
#parser.add_option("--abs", type="float" , default=0.0 , help="minimum average absolute change of voxels' values (stop criterion of reconstruction process)" )
#parser.add_option("--squ", type="float" , default=0.0 , help="minimum average squared change of voxels' values (stop criterion of reconstruction process)" )
parser.add_option(
"--VAR",
action="store_true",
default=False,
help="Stack of input consists of 2D variances (Default False)")
parser.add_option(
"--decimate",
type="float",
default=0.25,
help="Image decimate rate, a number less than 1. (Default is 0.25)")
parser.add_option(
"--window",
type="int",
default=0,
help=
"Target image size relative to original image size. (Default value is zero.)"
)
#parser.add_option("--SND", action="store_true", default=False, help="compute squared normalized differences (Default False)")
#parser.add_option("--nvec", type="int" , default=0 , help="Number of eigenvectors, (Default = 0 meaning no PCA calculated)")
parser.add_option(
"--symmetrize",
action="store_true",
default=False,
help="Prepare input stack for handling symmetry (Default False)")
parser.add_option("--overhead",
type="float",
default=0.5,
help="python overhead per CPU.")
(options, args) = parser.parse_args()
#####
from mpi import mpi_comm_rank, mpi_comm_size, mpi_recv, MPI_COMM_WORLD
from mpi import mpi_barrier, mpi_reduce, mpi_bcast, mpi_send, MPI_FLOAT, MPI_SUM, MPI_INT, MPI_MAX
#from mpi import *
from sp_applications import MPI_start_end
from sp_reconstruction import recons3d_em, recons3d_em_MPI
from sp_reconstruction import recons3d_4nn_MPI, recons3d_4nn_ctf_MPI
from sp_utilities import print_begin_msg, print_end_msg, print_msg
from sp_utilities import read_text_row, get_image, get_im, wrap_mpi_send, wrap_mpi_recv
from sp_utilities import bcast_EMData_to_all, bcast_number_to_all
from sp_utilities import get_symt
# This is code for handling symmetries by the above program. To be incorporated. PAP 01/27/2015
from EMAN2db import db_open_dict
# Set up global variables related to bdb cache
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
# Set up global variables related to ERROR function
sp_global_def.BATCH = True
# detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
if RUNNING_UNDER_MPI: sp_global_def.MPI = True
if options.output_dir == "./":
current_output_dir = os.path.abspath(options.output_dir)
else:
current_output_dir = options.output_dir
if options.symmetrize:
if mpi.mpi_comm_size(MPI_COMM_WORLD) > 1:
ERROR("Cannot use more than one CPU for symmetry preparation")
if not os.path.exists(current_output_dir):
os.makedirs(current_output_dir)
sp_global_def.write_command(current_output_dir)
from sp_logger import Logger, BaseLogger_Files
if os.path.exists(os.path.join(current_output_dir, "log.txt")):
os.remove(os.path.join(current_output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(current_output_dir, "./")
instack = args[0]
sym = options.sym.lower()
if (sym == "c1"):
ERROR("There is no need to symmetrize stack for C1 symmetry")
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
if (instack[:4] != "bdb:"):
#if output_dir =="./": stack = "bdb:data"
stack = "bdb:" + current_output_dir + "/data"
delete_bdb(stack)
junk = cmdexecute("sp_cpy.py " + instack + " " + stack)
else:
stack = instack
qt = EMUtil.get_all_attributes(stack, 'xform.projection')
na = len(qt)
ts = get_symt(sym)
ks = len(ts)
angsa = [None] * na
for k in range(ks):
#Qfile = "Q%1d"%k
#if options.output_dir!="./": Qfile = os.path.join(options.output_dir,"Q%1d"%k)
Qfile = os.path.join(current_output_dir, "Q%1d" % k)
#delete_bdb("bdb:Q%1d"%k)
delete_bdb("bdb:" + Qfile)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
junk = cmdexecute("e2bdb.py " + stack + " --makevstack=bdb:" +
Qfile)
#DB = db_open_dict("bdb:Q%1d"%k)
DB = db_open_dict("bdb:" + Qfile)
for i in range(na):
ut = qt[i] * ts[k]
DB.set_attr(i, "xform.projection", ut)
#bt = ut.get_params("spider")
#angsa[i] = [round(bt["phi"],3)%360.0, round(bt["theta"],3)%360.0, bt["psi"], -bt["tx"], -bt["ty"]]
#write_text_row(angsa, 'ptsma%1d.txt'%k)
#junk = cmdexecute("e2bdb.py "+stack+" --makevstack=bdb:Q%1d"%k)
#junk = cmdexecute("sxheader.py bdb:Q%1d --params=xform.projection --import=ptsma%1d.txt"%(k,k))
DB.close()
#if options.output_dir =="./": delete_bdb("bdb:sdata")
delete_bdb("bdb:" + current_output_dir + "/" + "sdata")
#junk = cmdexecute("e2bdb.py . --makevstack=bdb:sdata --filt=Q")
sdata = "bdb:" + current_output_dir + "/" + "sdata"
sxprint(sdata)
junk = cmdexecute("e2bdb.py " + current_output_dir +
" --makevstack=" + sdata + " --filt=Q")
#junk = cmdexecute("ls EMAN2DB/sdata*")
#a = get_im("bdb:sdata")
a = get_im(sdata)
a.set_attr("variabilitysymmetry", sym)
#a.write_image("bdb:sdata")
a.write_image(sdata)
else:
from sp_fundamentals import window2d
myid = mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
shared_comm = mpi_comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED,
0, MPI_INFO_NULL)
myid_on_node = mpi_comm_rank(shared_comm)
no_of_processes_per_group = mpi_comm_size(shared_comm)
masters_from_groups_vs_everything_else_comm = mpi_comm_split(
MPI_COMM_WORLD, main_node == myid_on_node, myid_on_node)
color, no_of_groups, balanced_processor_load_on_nodes = get_colors_and_subsets(main_node, MPI_COMM_WORLD, myid, \
shared_comm, myid_on_node, masters_from_groups_vs_everything_else_comm)
overhead_loading = options.overhead * number_of_proc
#memory_per_node = options.memory_per_node
#if memory_per_node == -1.: memory_per_node = 2.*no_of_processes_per_group
keepgoing = 1
current_window = options.window
current_decimate = options.decimate
if len(args) == 1: stack = args[0]
else:
sxprint("Usage: " + usage)
sxprint("Please run \'" + progname + " -h\' for detailed options")
ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
t0 = time()
# obsolete flags
options.MPI = True
#options.nvec = 0
options.radiuspca = -1
options.iter = 40
options.abs = 0.0
options.squ = 0.0
if options.fl > 0.0 and options.aa == 0.0:
ERROR("Fall off has to be given for the low-pass filter",
myid=myid)
#if options.VAR and options.SND:
# ERROR( "Only one of var and SND can be set!",myid=myid )
if options.VAR and (options.ave2D or options.ave3D or options.var2D):
ERROR(
"When VAR is set, the program cannot output ave2D, ave3D or var2D",
myid=myid)
#if options.SND and (options.ave2D or options.ave3D):
# ERROR( "When SND is set, the program cannot output ave2D or ave3D", myid=myid )
#if options.nvec > 0 :
# ERROR( "PCA option not implemented", myid=myid )
#if options.nvec > 0 and options.ave3D == None:
# ERROR( "When doing PCA analysis, one must set ave3D", myid=myid )
if current_decimate > 1.0 or current_decimate < 0.0:
ERROR("Decimate rate should be a value between 0.0 and 1.0",
myid=myid)
if current_window < 0.0:
ERROR("Target window size should be always larger than zero",
myid=myid)
if myid == main_node:
img = get_image(stack, 0)
nx = img.get_xsize()
ny = img.get_ysize()
if (min(nx, ny) < current_window): keepgoing = 0
keepgoing = bcast_number_to_all(keepgoing, main_node, MPI_COMM_WORLD)
if keepgoing == 0:
ERROR(
"The target window size cannot be larger than the size of decimated image",
myid=myid)
import string
options.sym = options.sym.lower()
# if global_def.CACHE_DISABLE:
# from utilities import disable_bdb_cache
# disable_bdb_cache()
# global_def.BATCH = True
if myid == main_node:
if not os.path.exists(current_output_dir):
os.makedirs(current_output_dir
) # Never delete output_dir in the program!
img_per_grp = options.img_per_grp
#nvec = options.nvec
radiuspca = options.radiuspca
from sp_logger import Logger, BaseLogger_Files
#if os.path.exists(os.path.join(options.output_dir, "log.txt")): os.remove(os.path.join(options.output_dir, "log.txt"))
log_main = Logger(BaseLogger_Files())
log_main.prefix = os.path.join(current_output_dir, "./")
if myid == main_node:
line = ""
for a in sys.argv:
line += " " + a
log_main.add(line)
log_main.add("-------->>>Settings given by all options<<<-------")
log_main.add("Symmetry : %s" % options.sym)
log_main.add("Input stack : %s" % stack)
log_main.add("Output_dir : %s" % current_output_dir)
if options.ave3D:
log_main.add("Ave3d : %s" % options.ave3D)
if options.var3D:
log_main.add("Var3d : %s" % options.var3D)
if options.ave2D:
log_main.add("Ave2D : %s" % options.ave2D)
if options.var2D:
log_main.add("Var2D : %s" % options.var2D)
if options.VAR: log_main.add("VAR : True")
else: log_main.add("VAR : False")
if options.CTF: log_main.add("CTF correction : True ")
else: log_main.add("CTF correction : False ")
log_main.add("Image per group : %5d" % options.img_per_grp)
log_main.add("Image decimate rate : %4.3f" % current_decimate)
log_main.add("Low pass filter : %4.3f" % options.fl)
current_fl = options.fl
if current_fl == 0.0: current_fl = 0.5
log_main.add(
"Current low pass filter is equivalent to cutoff frequency %4.3f for original image size"
% round((current_fl * current_decimate), 3))
log_main.add("Window size : %5d " % current_window)
log_main.add("sx3dvariability begins")
symbaselen = 0
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
nnxo = nx
nnyo = ny
if options.sym != "c1":
imgdata = get_im(stack)
try:
i = imgdata.get_attr("variabilitysymmetry").lower()
if (i != options.sym):
ERROR(
"The symmetry provided does not agree with the symmetry of the input stack",
myid=myid)
except:
ERROR(
"Input stack is not prepared for symmetry, please follow instructions",
myid=myid)
from sp_utilities import get_symt
i = len(get_symt(options.sym))
if ((nima / i) * i != nima):
ERROR(
"The length of the input stack is incorrect for symmetry processing",
myid=myid)
symbaselen = nima / i
else:
symbaselen = nima
else:
nima = 0
nx = 0
ny = 0
nnxo = 0
nnyo = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
nnxo = bcast_number_to_all(nnxo)
nnyo = bcast_number_to_all(nnyo)
if current_window > max(nx, ny):
ERROR("Window size is larger than the original image size")
if current_decimate == 1.:
if current_window != 0:
nx = current_window
ny = current_window
else:
if current_window == 0:
nx = int(nx * current_decimate + 0.5)
ny = int(ny * current_decimate + 0.5)
else:
nx = int(current_window * current_decimate + 0.5)
ny = nx
symbaselen = bcast_number_to_all(symbaselen)
# check FFT prime number
from sp_fundamentals import smallprime
is_fft_friendly = (nx == smallprime(nx))
if not is_fft_friendly:
if myid == main_node:
log_main.add(
"The target image size is not a product of small prime numbers"
)
log_main.add("Program adjusts the input settings!")
### two cases
if current_decimate == 1.:
nx = smallprime(nx)
ny = nx
current_window = nx # update
if myid == main_node:
log_main.add("The window size is updated to %d." %
current_window)
else:
if current_window == 0:
nx = smallprime(int(nx * current_decimate + 0.5))
current_decimate = float(nx) / nnxo
ny = nx
if (myid == main_node):
log_main.add("The decimate rate is updated to %f." %
current_decimate)
else:
nx = smallprime(
int(current_window * current_decimate + 0.5))
ny = nx
current_window = int(nx / current_decimate + 0.5)
if (myid == main_node):
log_main.add("The window size is updated to %d." %
current_window)
if myid == main_node:
log_main.add("The target image size is %d" % nx)
if radiuspca == -1: radiuspca = nx / 2 - 2
if myid == main_node:
log_main.add("%-70s: %d\n" % ("Number of projection", nima))
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
"""
if options.SND:
from sp_projection import prep_vol, prgs
from sp_statistics import im_diff
from sp_utilities import get_im, model_circle, get_params_proj, set_params_proj
from sp_utilities import get_ctf, generate_ctf
from sp_filter import filt_ctf
imgdata = EMData.read_images(stack, range(img_begin, img_end))
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
bcast_EMData_to_all(vol, myid)
volft, kb = prep_vol(vol)
mask = model_circle(nx/2-2, nx, ny)
varList = []
for i in xrange(img_begin, img_end):
phi, theta, psi, s2x, s2y = get_params_proj(imgdata[i-img_begin])
ref_prj = prgs(volft, kb, [phi, theta, psi, -s2x, -s2y])
if options.CTF:
ctf_params = get_ctf(imgdata[i-img_begin])
ref_prj = filt_ctf(ref_prj, generate_ctf(ctf_params))
diff, A, B = im_diff(ref_prj, imgdata[i-img_begin], mask)
diff2 = diff*diff
set_params_proj(diff2, [phi, theta, psi, s2x, s2y])
varList.append(diff2)
mpi_barrier(MPI_COMM_WORLD)
"""
if options.VAR: # 2D variance images have no shifts
#varList = EMData.read_images(stack, range(img_begin, img_end))
from EMAN2 import Region
for index_of_particle in range(img_begin, img_end):
image = get_im(stack, index_of_proj)
if current_window > 0:
varList.append(
fdecimate(
window2d(image, current_window, current_window),
nx, ny))
else:
varList.append(fdecimate(image, nx, ny))
else:
from sp_utilities import bcast_number_to_all, bcast_list_to_all, send_EMData, recv_EMData
from sp_utilities import set_params_proj, get_params_proj, params_3D_2D, get_params2D, set_params2D, compose_transform2
from sp_utilities import model_blank, nearest_proj, model_circle, write_text_row, wrap_mpi_gatherv
from sp_applications import pca
from sp_statistics import avgvar, avgvar_ctf, ccc
from sp_filter import filt_tanl
from sp_morphology import threshold, square_root
from sp_projection import project, prep_vol, prgs
from sets import Set
from sp_utilities import wrap_mpi_recv, wrap_mpi_bcast, wrap_mpi_send
import numpy as np
if myid == main_node:
t1 = time()
proj_angles = []
aveList = []
tab = EMUtil.get_all_attributes(stack, 'xform.projection')
for i in range(nima):
t = tab[i].get_params('spider')
phi = t['phi']
theta = t['theta']
psi = t['psi']
x = theta
if x > 90.0: x = 180.0 - x
x = x * 10000 + psi
proj_angles.append([x, t['phi'], t['theta'], t['psi'], i])
t2 = time()
log_main.add(
"%-70s: %d\n" %
("Number of neighboring projections", img_per_grp))
log_main.add("...... Finding neighboring projections\n")
log_main.add("Number of images per group: %d" % img_per_grp)
log_main.add("Now grouping projections")
proj_angles.sort()
proj_angles_list = np.full((nima, 4), 0.0, dtype=np.float32)
for i in range(nima):
proj_angles_list[i][0] = proj_angles[i][1]
proj_angles_list[i][1] = proj_angles[i][2]
proj_angles_list[i][2] = proj_angles[i][3]
proj_angles_list[i][3] = proj_angles[i][4]
else:
proj_angles_list = 0
proj_angles_list = wrap_mpi_bcast(proj_angles_list, main_node,
MPI_COMM_WORLD)
proj_angles = []
for i in range(nima):
proj_angles.append([
proj_angles_list[i][0], proj_angles_list[i][1],
proj_angles_list[i][2],
int(proj_angles_list[i][3])
])
del proj_angles_list
proj_list, mirror_list = nearest_proj(proj_angles, img_per_grp,
range(img_begin, img_end))
all_proj = Set()
for im in proj_list:
for jm in im:
all_proj.add(proj_angles[jm][3])
all_proj = list(all_proj)
index = {}
for i in range(len(all_proj)):
index[all_proj[i]] = i
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("%-70s: %.2f\n" %
("Finding neighboring projections lasted [s]",
time() - t2))
log_main.add("%-70s: %d\n" %
("Number of groups processed on the main node",
len(proj_list)))
log_main.add("Grouping projections took: %12.1f [m]" %
((time() - t2) / 60.))
log_main.add("Number of groups on main node: ", len(proj_list))
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
log_main.add("...... Calculating the stack of 2D variances \n")
# Memory estimation. There are two memory consumption peaks
# peak 1. Compute ave, var;
# peak 2. Var volume reconstruction;
# proj_params = [0.0]*(nima*5)
aveList = []
varList = []
#if nvec > 0: eigList = [[] for i in range(nvec)]
dnumber = len(
all_proj) # all neighborhood set for assigned to myid
pnumber = len(proj_list) * 2. + img_per_grp # aveList and varList
tnumber = dnumber + pnumber
vol_size2 = nx**3 * 4. * 8 / 1.e9
vol_size1 = 2. * nnxo**3 * 4. * 8 / 1.e9
proj_size = nnxo * nnyo * len(
proj_list) * 4. * 2. / 1.e9 # both aveList and varList
orig_data_size = nnxo * nnyo * 4. * tnumber / 1.e9
reduced_data_size = nx * nx * 4. * tnumber / 1.e9
full_data = np.full((number_of_proc, 2), -1., dtype=np.float16)
full_data[myid] = orig_data_size, reduced_data_size
if myid != main_node:
wrap_mpi_send(full_data, main_node, MPI_COMM_WORLD)
if myid == main_node:
for iproc in range(number_of_proc):
if iproc != main_node:
dummy = wrap_mpi_recv(iproc, MPI_COMM_WORLD)
full_data[np.where(dummy > -1)] = dummy[np.where(
dummy > -1)]
del dummy
mpi_barrier(MPI_COMM_WORLD)
full_data = wrap_mpi_bcast(full_data, main_node, MPI_COMM_WORLD)
# find the CPU with heaviest load
minindx = np.argsort(full_data, 0)
heavy_load_myid = minindx[-1][1]
total_mem = sum(full_data)
if myid == main_node:
if current_window == 0:
log_main.add(
"Nx: current image size = %d. Decimated by %f from %d"
% (nx, current_decimate, nnxo))
else:
log_main.add(
"Nx: current image size = %d. Windowed to %d, and decimated by %f from %d"
% (nx, current_window, current_decimate, nnxo))
log_main.add("Nproj: number of particle images.")
log_main.add("Navg: number of 2D average images.")
log_main.add("Nvar: number of 2D variance images.")
log_main.add(
"Img_per_grp: user defined image per group for averaging = %d"
% img_per_grp)
log_main.add(
"Overhead: total python overhead memory consumption = %f"
% overhead_loading)
log_main.add("Total memory) = 4.0*nx^2*(nproj + navg +nvar+ img_per_grp)/1.0e9 + overhead: %12.3f [GB]"%\
(total_mem[1] + overhead_loading))
del full_data
mpi_barrier(MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add(
"Begin reading and preprocessing images on processor. Wait... "
)
ttt = time()
#imgdata = EMData.read_images(stack, all_proj)
imgdata = [None for im in range(len(all_proj))]
for index_of_proj in range(len(all_proj)):
#image = get_im(stack, all_proj[index_of_proj])
if (current_window > 0):
imgdata[index_of_proj] = fdecimate(
window2d(get_im(stack, all_proj[index_of_proj]),
current_window, current_window), nx, ny)
else:
imgdata[index_of_proj] = fdecimate(
get_im(stack, all_proj[index_of_proj]), nx, ny)
if (current_decimate > 0.0 and options.CTF):
ctf = imgdata[index_of_proj].get_attr("ctf")
ctf.apix = ctf.apix / current_decimate
imgdata[index_of_proj].set_attr("ctf", ctf)
if myid == heavy_load_myid and index_of_proj % 100 == 0:
log_main.add(" ...... %6.2f%% " %
(index_of_proj / float(len(all_proj)) * 100.))
mpi_barrier(MPI_COMM_WORLD)
if myid == heavy_load_myid:
log_main.add("All_proj preprocessing cost %7.2f m" %
((time() - ttt) / 60.))
log_main.add("Wait untill reading on all CPUs done...")
'''
imgdata2 = EMData.read_images(stack, range(img_begin, img_end))
if options.fl > 0.0:
for k in xrange(len(imgdata2)):
imgdata2[k] = filt_tanl(imgdata2[k], options.fl, options.aa)
if options.CTF:
vol = recons3d_4nn_ctf_MPI(myid, imgdata2, 1.0, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
else:
vol = recons3d_4nn_MPI(myid, imgdata2, symmetry=options.sym, npad=options.npad, xysize=-1, zsize=-1)
if myid == main_node:
vol.write_image("vol_ctf.hdf")
print_msg("Writing to the disk volume reconstructed from averages as : %s\n"%("vol_ctf.hdf"))
del vol, imgdata2
mpi_barrier(MPI_COMM_WORLD)
'''
from sp_applications import prepare_2d_forPCA
from sp_utilities import model_blank
from EMAN2 import Transform
if not options.no_norm:
mask = model_circle(nx / 2 - 2, nx, nx)
if options.CTF:
from sp_utilities import pad
from sp_filter import filt_ctf
from sp_filter import filt_tanl
if myid == heavy_load_myid:
log_main.add("Start computing 2D aveList and varList. Wait...")
ttt = time()
inner = nx // 2 - 4
outer = inner + 2
xform_proj_for_2D = [None for i in range(len(proj_list))]
for i in range(len(proj_list)):
ki = proj_angles[proj_list[i][0]][3]
if ki >= symbaselen: continue
mi = index[ki]
dpar = Util.get_transform_params(imgdata[mi],
"xform.projection", "spider")
phiM, thetaM, psiM, s2xM, s2yM = dpar["phi"], dpar[
"theta"], dpar[
"psi"], -dpar["tx"] * current_decimate, -dpar[
"ty"] * current_decimate
grp_imgdata = []
for j in range(img_per_grp):
mj = index[proj_angles[proj_list[i][j]][3]]
cpar = Util.get_transform_params(imgdata[mj],
"xform.projection",
"spider")
alpha, sx, sy, mirror = params_3D_2D_NEW(
cpar["phi"], cpar["theta"], cpar["psi"],
-cpar["tx"] * current_decimate,
-cpar["ty"] * current_decimate, mirror_list[i][j])
if thetaM <= 90:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, phiM - cpar["phi"], 0.0,
0.0, 1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, 180 - (phiM - cpar["phi"]),
0.0, 0.0, 1.0)
else:
if mirror == 0:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0, -(phiM - cpar["phi"]), 0.0,
0.0, 1.0)
else:
alpha, sx, sy, scale = compose_transform2(
alpha, sx, sy, 1.0,
-(180 - (phiM - cpar["phi"])), 0.0, 0.0, 1.0)
imgdata[mj].set_attr(
"xform.align2d",
Transform({
"type": "2D",
"alpha": alpha,
"tx": sx,
"ty": sy,
"mirror": mirror,
"scale": 1.0
}))
grp_imgdata.append(imgdata[mj])
if not options.no_norm:
for k in range(img_per_grp):
ave, std, minn, maxx = Util.infomask(
grp_imgdata[k], mask, False)
grp_imgdata[k] -= ave
grp_imgdata[k] /= std
if options.fl > 0.0:
for k in range(img_per_grp):
grp_imgdata[k] = filt_tanl(grp_imgdata[k], options.fl,
options.aa)
# Because of background issues, only linear option works.
if options.CTF:
ave, var = aves_wiener(grp_imgdata,
SNR=1.0e5,
interpolation_method="linear")
else:
ave, var = ave_var(grp_imgdata)
# Switch to std dev
# threshold is not really needed,it is just in case due to numerical accuracy something turns out negative.
var = square_root(threshold(var))
set_params_proj(ave, [phiM, thetaM, 0.0, 0.0, 0.0])
set_params_proj(var, [phiM, thetaM, 0.0, 0.0, 0.0])
aveList.append(ave)
varList.append(var)
xform_proj_for_2D[i] = [phiM, thetaM, 0.0, 0.0, 0.0]
'''
if nvec > 0:
eig = pca(input_stacks=grp_imgdata, subavg="", mask_radius=radiuspca, nvec=nvec, incore=True, shuffle=False, genbuf=True)
for k in range(nvec):
set_params_proj(eig[k], [phiM, thetaM, 0.0, 0.0, 0.0])
eigList[k].append(eig[k])
"""
if myid == 0 and i == 0:
for k in xrange(nvec):
eig[k].write_image("eig.hdf", k)
"""
'''
if (myid == heavy_load_myid) and (i % 100 == 0):
log_main.add(" ......%6.2f%% " %
(i / float(len(proj_list)) * 100.))
del imgdata, grp_imgdata, cpar, dpar, all_proj, proj_angles, index
if not options.no_norm: del mask
if myid == main_node: del tab
# At this point, all averages and variances are computed
mpi_barrier(MPI_COMM_WORLD)
if (myid == heavy_load_myid):
log_main.add("Computing aveList and varList took %12.1f [m]" %
((time() - ttt) / 60.))
xform_proj_for_2D = wrap_mpi_gatherv(xform_proj_for_2D, main_node,
MPI_COMM_WORLD)
if (myid == main_node):
write_text_row([str(entry) for entry in xform_proj_for_2D],
os.path.join(current_output_dir, "params.txt"))
del xform_proj_for_2D
mpi_barrier(MPI_COMM_WORLD)
if options.ave2D:
from sp_fundamentals import fpol
from sp_applications import header
if myid == main_node:
log_main.add("Compute ave2D ... ")
km = 0
for i in range(number_of_proc):
if i == main_node:
for im in range(len(aveList)):
aveList[im].write_image(
os.path.join(current_output_dir,
options.ave2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im + i + 70000)
"""
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('members', map(int, members))
members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pix_err', map(float, members))
members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', map(float, members))
"""
tmpvol = fpol(ave, nx, nx, 1)
tmpvol.write_image(
os.path.join(current_output_dir,
options.ave2D), km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in range(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
"""
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pix_err')
mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0,-999.0,-999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
"""
if myid == main_node:
header(os.path.join(current_output_dir, options.ave2D),
params='xform.projection',
fimport=os.path.join(current_output_dir,
"params.txt"))
mpi_barrier(MPI_COMM_WORLD)
if options.ave3D:
from sp_fundamentals import fpol
t5 = time()
if myid == main_node: log_main.add("Reconstruct ave3D ... ")
ave3D = recons3d_4nn_MPI(myid,
aveList,
symmetry=options.sym,
npad=options.npad)
bcast_EMData_to_all(ave3D, myid)
if myid == main_node:
if current_decimate != 1.0:
ave3D = resample(ave3D, 1. / current_decimate)
ave3D = fpol(ave3D, nnxo, nnxo,
nnxo) # always to the orignal image size
set_pixel_size(ave3D, 1.0)
ave3D.write_image(
os.path.join(current_output_dir, options.ave3D))
log_main.add("Ave3D reconstruction took %12.1f [m]" %
((time() - t5) / 60.0))
log_main.add("%-70s: %s\n" %
("The reconstructed ave3D is saved as ",
options.ave3D))
mpi_barrier(MPI_COMM_WORLD)
del ave, var, proj_list, stack, alpha, sx, sy, mirror, aveList
'''
if nvec > 0:
for k in range(nvec):
if myid == main_node:log_main.add("Reconstruction eigenvolumes", k)
cont = True
ITER = 0
mask2d = model_circle(radiuspca, nx, nx)
while cont:
#print "On node %d, iteration %d"%(myid, ITER)
eig3D = recons3d_4nn_MPI(myid, eigList[k], symmetry=options.sym, npad=options.npad)
bcast_EMData_to_all(eig3D, myid, main_node)
if options.fl > 0.0:
eig3D = filt_tanl(eig3D, options.fl, options.aa)
if myid == main_node:
eig3D.write_image(os.path.join(options.outpout_dir, "eig3d_%03d.hdf"%(k, ITER)))
Util.mul_img( eig3D, model_circle(radiuspca, nx, nx, nx) )
eig3Df, kb = prep_vol(eig3D)
del eig3D
cont = False
icont = 0
for l in range(len(eigList[k])):
phi, theta, psi, s2x, s2y = get_params_proj(eigList[k][l])
proj = prgs(eig3Df, kb, [phi, theta, psi, s2x, s2y])
cl = ccc(proj, eigList[k][l], mask2d)
if cl < 0.0:
icont += 1
cont = True
eigList[k][l] *= -1.0
u = int(cont)
u = mpi_reduce([u], 1, MPI_INT, MPI_MAX, main_node, MPI_COMM_WORLD)
icont = mpi_reduce([icont], 1, MPI_INT, MPI_SUM, main_node, MPI_COMM_WORLD)
if myid == main_node:
u = int(u[0])
log_main.add(" Eigenvector: ",k," number changed ",int(icont[0]))
else: u = 0
u = bcast_number_to_all(u, main_node)
cont = bool(u)
ITER += 1
del eig3Df, kb
mpi_barrier(MPI_COMM_WORLD)
del eigList, mask2d
'''
if options.ave3D: del ave3D
if options.var2D:
from sp_fundamentals import fpol
from sp_applications import header
if myid == main_node:
log_main.add("Compute var2D...")
km = 0
for i in range(number_of_proc):
if i == main_node:
for im in range(len(varList)):
tmpvol = fpol(varList[im], nx, nx, 1)
tmpvol.write_image(
os.path.join(current_output_dir,
options.var2D), km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i,
SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im + i + 70000)
tmpvol = fpol(ave, nx, nx, 1)
tmpvol.write_image(
os.path.join(current_output_dir,
options.var2D), km)
km += 1
else:
mpi_send(len(varList), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
for im in range(len(varList)):
send_EMData(varList[im], main_node, im + myid +
70000) # What with the attributes??
mpi_barrier(MPI_COMM_WORLD)
if myid == main_node:
from sp_applications import header
header(os.path.join(current_output_dir, options.var2D),
params='xform.projection',
fimport=os.path.join(current_output_dir,
"params.txt"))
mpi_barrier(MPI_COMM_WORLD)
if options.var3D:
if myid == main_node: log_main.add("Reconstruct var3D ...")
t6 = time()
# radiusvar = options.radius
# if( radiusvar < 0 ): radiusvar = nx//2 -3
res = recons3d_4nn_MPI(myid,
varList,
symmetry=options.sym,
npad=options.npad)
#res = recons3d_em_MPI(varList, vol_stack, options.iter, radiusvar, options.abs, True, options.sym, options.squ)
if myid == main_node:
from sp_fundamentals import fpol
if current_decimate != 1.0:
res = resample(res, 1. / current_decimate)
res = fpol(res, nnxo, nnxo, nnxo)
set_pixel_size(res, 1.0)
res.write_image(os.path.join(current_output_dir,
options.var3D))
log_main.add(
"%-70s: %s\n" %
("The reconstructed var3D is saved as ", options.var3D))
log_main.add("Var3D reconstruction took %f12.1 [m]" %
((time() - t6) / 60.0))
log_main.add("Total computation time %f12.1 [m]" %
((time() - t0) / 60.0))
log_main.add("sx3dvariability finishes")
if RUNNING_UNDER_MPI:
sp_global_def.MPI = False
sp_global_def.BATCH = False