def calc_fsc(model_path1, model_path2):
V1 = mrc.readMRC(model_path1)
V2 = mrc.readMRC(model_path2)
N = V1.shape[0]
V1, _ = cryoem.align_density(V1)
V2, _ = cryoem.align_density(V2)
VF1 = np.fft.fftshift(np.fft.fftn(V1))
VF2 = np.fft.fftshift(np.fft.fftn(V2))
# test_VF = np.fft.fftshift(np.fft.fftn(test_V))
maxrad = 1
rads, fsc, thresholds, resolutions = cryoem.compute_fsc(VF1, VF2, maxrad)
return rads, fsc, thresholds, resolutions
def calc_fsc(model_path1, model_path2):
V1 = mrc.readMRC(model_path1)
V2 = mrc.readMRC(model_path2)
N = V1.shape[0]
V1, _ = cryoem.align_density(V1)
V2, _ = cryoem.align_density(V2)
VF1 = np.fft.fftshift(np.fft.fftn(V1))
VF2 = np.fft.fftshift(np.fft.fftn(V2))
# test_VF = np.fft.fftshift(np.fft.fftn(test_V))
maxrad = 1
rads, fsc, thresholds, resolutions = cryoem.compute_fsc(VF1, VF2, maxrad)
return rads, fsc, thresholds, resolutions
def __init__(self, parent=None, mrcfiles=[]):
QtGui.QWidget.__init__(self, parent)
Ms = [mrc.readMRC(mrcfile) for mrcfile in mrcfiles]
if len(mrcfiles) < 6:
hbox = True
layout = QtGui.QHBoxLayout(self)
else:
hbox = False
layout = QtGui.QGridLayout(self)
maxcol = int(len(mrcfiles) / 3) + 1
layout.setContentsMargins(0, 0, 0, 0)
layout.setSpacing(0)
for i, M in enumerate(Ms):
filename = os.path.basename(mrcfiles[i])
self.splitter_main_bottom = QtGui.QSplitter(self)
if hbox:
layout.addWidget(self.splitter_main_bottom)
else:
row, col = np.unravel_index(i, (3, maxcol))
layout.addWidget(self.splitter_main_bottom, row, col)
self.splitter_main_bottom.setOrientation(QtCore.Qt.Horizontal)
# self.sliceplot_widget = SlicePlotQWidget()
# self.splitter_main_bottom.addWidget(self.sliceplot_widget)
self.densityplot_widget = MayaviQWidget()
self.splitter_main_bottom.addWidget(self.densityplot_widget)
self.alignedM, self.R = cryoem.align_density(M, upsamp=1.0)
self.densityplot_widget.setup(self.alignedM, filename=filename)
def calc_fsc(model_path1, model_path2):
V1 = mrc.readMRC(model_path1)
V2 = mrc.readMRC(model_path2)
N = V1.shape[0]
alignedV1, R1 = cryoem.align_density(V1)
alignedV1 = cryoem.rotate_density(alignedV1, R1)
alignedV2, R1 = cryoem.align_density(V2)
alignedV2 = cryoem.rotate_density(alignedV2, R1)
VF1 = np.fft.fftshift(np.fft.fftn(alignedV1))
VF2 = np.fft.fftshift(np.fft.fftn(alignedV2))
maxrad = 1
rads, fsc, thresholds, resolutions = cryoem.compute_fsc(VF1, VF2, maxrad)
return rads, fsc, thresholds, resolutions
def loaddensity(self, fname, key):
if os.path.isfile(os.path.join(self.lbase, fname)):
if fname.upper().endswith('.MRC'):
return mrc.readMRC(os.path.join(self.lbase, fname))
else:
with open(os.path.join(self.lbase, fname), 'rb') as f:
return pickle.load(f)[-1][key]
else:
return None
def loaddensity(self, fname, key):
if os.path.isfile(os.path.join(self.lbase,fname)):
if fname.upper().endswith('.MRC'):
return mrc.readMRC(os.path.join(self.lbase,fname))
else:
with open(os.path.join(self.lbase,fname), 'rb') as f:
return cPickle.load(f)[-1][key]
else:
return None
def premult_test(model, kernel='lanczos', kernsize=6):
if isinstance(model, str):
M = mrc.readMRC(model)
elif isinstance(model, np.ndarray):
M = model
shape = np.asarray(M.shape)
assert (shape - shape.mean()).sum() == 0
N = M.shape[0]
rad = 0.6
premult = cryoops.compute_premultiplier(N, kernel, kernsize)
TtoF = sincint.gentrunctofull(N=N, rad=rad)
premulter = premult.reshape((1, 1, -1)) \
* premult.reshape((1, -1, 1)) \
* premult.reshape((-1, 1, 1))
fM = density.real_to_fspace(M)
prefM = density.real_to_fspace(premulter * M)
pt = np.random.randn(3)
pt /= np.linalg.norm(pt)
psi = 2 * np.pi * np.random.rand()
ea = geometry.genEA(pt)[0]
ea[2] = psi
print('project model for Euler angel: ({:.2f}, {:.2f}, {:.2f}) degree'.
format(*np.rad2deg(ea)))
rot_matrix = geometry.rotmat3D_EA(*ea)[:, 0:2]
slop = cryoops.compute_projection_matrix([rot_matrix], N, kernel, kernsize,
rad, 'rots')
trunc_slice = slop.dot(fM.reshape((-1, )))
premult_trunc_slice = slop.dot(prefM.reshape((-1, )))
proj = density.fspace_to_real(TtoF.dot(trunc_slice).reshape(N, N))
premult_proj = density.fspace_to_real(
TtoF.dot(premult_trunc_slice).reshape(N, N))
fig, ax = plt.subplots(1, 3, figsize=(14.4, 4.8))
im_proj = ax[0].imshow(proj, origin='lower')
fig.colorbar(im_proj, ax=ax[0])
ax[0].set_title('no premulter')
im_pre = ax[1].imshow(premult_proj, origin='lower')
fig.colorbar(im_pre, ax=ax[1])
ax[1].set_title('with premulter')
im_diff = ax[2].imshow(proj - premult_proj, origin='lower')
fig.colorbar(im_diff, ax=ax[2])
ax[2].set_title('difference of two image')
fig.tight_layout()
plt.show()
def __init__(self, parent=None, mrcfiles=[]):
QtGui.QWidget.__init__(self, parent)
Ms = [mrc.readMRC(mrcfile) for mrcfile in mrcfiles]
layout = QtGui.QVBoxLayout(self)
layout.setContentsMargins(0, 0, 0, 0)
layout.setSpacing(0)
for M in Ms:
self.splitter_main_bottom = QtGui.QSplitter(self)
layout.addWidget(self.splitter_main_bottom)
self.splitter_main_bottom.setOrientation(QtCore.Qt.Horizontal)
self.sliceplot_widget = SlicePlotQWidget()
self.splitter_main_bottom.addWidget(self.sliceplot_widget)
self.densityplot_widget = MayaviQWidget()
self.splitter_main_bottom.addWidget(self.densityplot_widget)
self.alignedM, self.R = c.align_density(M, upsamp=1.0)
self.densityplot_widget.setup(self.alignedM)
self.sliceplot_widget.setup(M, self.R)
def __init__(self, parent=None, mrcfiles=[]):
QtGui.QWidget.__init__(self, parent)
Ms = [mrc.readMRC(mrcfile) for mrcfile in mrcfiles]
layout = QtGui.QVBoxLayout(self)
layout.setContentsMargins(0,0,0,0)
layout.setSpacing(0)
for M in Ms:
self.splitter_main_bottom = QtGui.QSplitter(self)
layout.addWidget(self.splitter_main_bottom)
self.splitter_main_bottom.setOrientation(QtCore.Qt.Horizontal)
self.sliceplot_widget = SlicePlotQWidget()
self.splitter_main_bottom.addWidget(self.sliceplot_widget)
self.densityplot_widget = MayaviQWidget()
self.splitter_main_bottom.addWidget(self.densityplot_widget)
self.alignedM,self.R = c.align_density(M, upsamp=1.0)
self.densityplot_widget.setup(self.alignedM)
self.sliceplot_widget.setup(M, self.R)
def __init__(self, parent=None, mrcfiles=[]):
QtGui.QWidget.__init__(self, parent)
Ms = [mrc.readMRC(mrcfile) for mrcfile in mrcfiles]
layout = QtGui.QHBoxLayout(self)
layout.setContentsMargins(0,0,0,0)
layout.setSpacing(0)
for i, M in enumerate(Ms):
filename = os.path.basename(mrcfiles[i])
self.splitter_main_bottom = QtGui.QSplitter(self)
layout.addWidget(self.splitter_main_bottom)
self.splitter_main_bottom.setOrientation(QtCore.Qt.Horizontal)
# self.sliceplot_widget = SlicePlotQWidget()
# self.splitter_main_bottom.addWidget(self.sliceplot_widget)
self.densityplot_widget = MayaviQWidget()
self.splitter_main_bottom.addWidget(self.densityplot_widget)
self.alignedM,self.R = cryoem.align_density(M, upsamp=1.0)
self.densityplot_widget.setup(self.alignedM, filename=filename)
def project(model, euler_angles, rad=0.95, truncate=False):
if isinstance(model, str):
M = mrc.readMRC(model)
elif isinstance(model, np.ndarray):
M = model
N = M.shape[0]
kernel = 'lanczos'
ksize = 6
premult = cryoops.compute_premultiplier(N, kernel, ksize)
TtoF = sincint.gentrunctofull(N=N, rad=rad)
premulter = premult.reshape((1, 1, -1)) \
* premult.reshape((1, -1, 1)) \
* premult.reshape((-1, 1, 1))
# premulter = 1
fM = density.real_to_fspace(premulter * M)
euler_angles = euler_angles.reshape((-1, 3))
num_projs = euler_angles.shape[0]
if truncate:
projs = np.zeros((num_projs, TtoF.shape[1]), dtype=fM.dtype)
else:
projs = np.zeros((num_projs, N, N), dtype=M.dtype)
for i, ea in enumerate(euler_angles):
rot_matrix = geometry.rotmat3D_EA(*ea)[:, 0:2]
slop = cryoops.compute_projection_matrix([rot_matrix], N, kernel, ksize, rad, 'rots')
trunc_slice = slop.dot(fM.reshape((-1,)))
if truncate:
projs[i, :] = trunc_slice
else:
projs[i, :, :] = density.fspace_to_real(TtoF.dot(trunc_slice).reshape(N, N))
if num_projs == 1 and not truncate:
projs = projs.reshape((N, N))
return projs
def gen_slices(model_files, fspace=False, log_scale=True):
for model in model_files:
M = mrc.readMRC(model)
N = M.shape[0]
print('model size: {0}x{0}x{0}'.format(N))
oversampling_factor = 3
zeropad = oversampling_factor - 1 # oversampling factor = zeropad + 1
psize = 3.0 * oversampling_factor
beamstop_freq = 0.003
mask = geometry.gen_dense_beamstop_mask(N, 2, beamstop_freq, psize=psize)
# mask = None
if fspace:
fM = M
else:
M_totalmass = 1000000
M *= M_totalmass / M.sum()
V = density.real_to_fspace_with_oversampling(M, oversampling_factor)
fM = V.real ** 2 + V.imag ** 2
mask_3D = geometry.gen_dense_beamstop_mask(N, 3, beamstop_freq, psize=psize)
fM *= mask_3D
mrc.writeMRC('particle/{}_fM_totalmass_{}_oversampling_{}.mrc'.format(
os.path.splitext(os.path.basename(model))[0], str(int(M_totalmass)).zfill(5), oversampling_factor
), fM, psz=psize)
slicing_func = RegularGridInterpolator([np.arange(N),]*3, fM, bounds_error=False, fill_value=0.0)
coords = geometry.gencoords_base(N, 2)
fig, axes = plt.subplots(3, 3, figsize=(12.9, 9.6))
for i, ax in enumerate(axes.flat):
row, col = np.unravel_index(i, (3, 3))
# Randomly generate the viewing direction/shift
pt = np.random.randn(3)
pt /= np.linalg.norm(pt)
psi = 2 * np.pi * np.random.rand()
EA = geometry.genEA(pt)[0]
EA[2] = psi
R = geometry.rotmat3D_EA(*EA)[:, 0:2]
rotated_coords = R.dot(coords.T).T + int(N/2)
img = slicing_func(rotated_coords).reshape(N, N)
img = np.require(np.random.poisson(img), dtype=np.float32)
if log_scale:
img = np.log(np.maximum(img, 0))
if mask is not None:
img *= mask
im = ax.imshow(img, origin='lower') # cmap='Greys'
ticks = [0, int(N/4.0), int(N/2.0), int(N*3.0/4.0), int(N-1)]
if row == 2:
ax.set_xticks(ticks)
else:
ax.set_xticks([])
if col == 0:
ax.set_yticks(ticks)
else:
ax.set_yticks([])
fig.colorbar(im, ax=ax)
# fig.subplots_adjust(right=0.8)
# cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7])
# fig.colorbar(im, cax=cbar_ax)
fig.suptitle('simulated experimental data of XFEL for {}'.format(model))
# fig.tight_layout()
plt.show()
import numpy as np
import matplotlib.pyplot as plt
from cryoio import mrc
import density, cryoops
import geometry
from quadrature import healpix
from cryoem import cryoem
import pyximport
pyximport.install(setup_args={"include_dirs": np.get_include()},
reload_support=True)
import sincint
M = mrc.readMRC('./particle/EMD-6044.mrc')
# M = mrc.readMRC('./particle/1AON.mrc')
# M = M / np.sum(M)
M = M[:124, :124, :124]
mrc.writeMRC('./particle/EMD-6044-cropped.mrc', M, psz=3.0)
N = M.shape[0]
print(M.shape)
rad = 1
kernel = 'lanczos'
ksize = 4
xy, trunc_xy, truncmask = geometry.gencoords(N, 2, rad, True)
# premult = cryoops.compute_premultiplier(N, kernel='lanczos', kernsize=6)
premult = cryoops.compute_premultiplier(N, kernel, ksize)
TtoF = sincint.gentrunctofull(N=N, rad=rad)
def sagd_init(data_dir, model_file, use_angular_correlation=False):
data_params = {
'dataset_name': "1AON",
'inpath': os.path.join(data_dir, 'imgdata.mrc'),
'ctfpath': os.path.join(data_dir, 'defocus.txt'),
'microscope_params': {
'akv': 200,
'wgh': 0.07,
'cs': 2.0
},
'resolution': 2.8,
'sigma': 'noise_std',
'sigma_out': 'data_std',
'minisize': 20,
'test_imgs': 20,
'partition': 0,
'num_partitions': 0,
'random_seed': 1,
# 'symmetry': 'C7'
}
# Setup dataset
print("Loading dataset %s" % data_dir)
cryodata, _ = dataset_loading_test(data_params)
# mleDC, _, mleDC_est_std = cryodata.get_dc_estimate()
# modelscale = (np.abs(mleDC) + 2*mleDC_est_std)/cryodata.N
modelscale = 1.0
if model_file is not None:
print("Loading density map %s" % model_file)
M = readMRC(model_file)
else:
print("Generating random initial density map ...")
M = cryoem.generate_phantom_density(cryodata.N, 0.95 * cryodata.N / 2.0, \
5 * cryodata.N / 128.0, 30, seed=0)
M *= modelscale / M.sum()
slice_interp = {
'kern': 'lanczos',
'kernsize': 4,
'zeropad': 0,
'dopremult': True
}
# fM = SimpleKernel.get_fft(M, slice_interp)
M_totalmass = 5000
M *= M_totalmass / M.sum()
N = M.shape[0]
kernel = 'lanczos'
ksize = 6
premult = cryoops.compute_premultiplier(N, kernel, ksize)
V = density.real_to_fspace(
premult.reshape((1, 1, -1)) * premult.reshape(
(1, -1, 1)) * premult.reshape((-1, 1, 1)) * M)
M = V.real**2 + V.imag**2
freqs_3d = geometry.gencoords_base(N, 3) / (N * data_params['resolution'])
freq_radius_3d = np.sqrt((freqs_3d**2).sum(axis=1))
mask_3d_outlier = np.require(np.float_(freq_radius_3d > 0.015).reshape(
(N, N, N)),
dtype=density.real_t)
fM = M * mask_3d_outlier
cparams = {
'use_angular_correlation': use_angular_correlation,
'likelihood': 'UnknownRSLikelihood()',
'kernel': 'multicpu',
'prior_name': "'Null'",
'sparsity_lambda': 0.9,
'prior': 'NullPrior()',
# 'prior_name': "'CAR'",
# 'prior': 'CARPrior()',
# 'car_type': 'gauss0.5',
# 'car_tau': 75.0,
'iteration': 0,
'pixel_size': cryodata.pixel_size,
'max_frequency': 0.02,
'num_batches': cryodata.N_batches,
'interp_kernel_R': 'lanczos',
'interp_kernel_size_R': 4,
'interp_zeropad_R': 0.0,
'interp_premult_R': True,
'interp_kernel_I': 'lanczos',
'interp_kernel_size_I': 8,
'interp_zeropad_I': 0.0, # 1.0,
'interp_premult_I': True,
'sigma': cryodata.noise_var,
'modelscale': modelscale,
# 'symmetry': 'C7'
}
is_params = {
# importance sampling
# Ignore the first 50 iterations entirely
'is_prior_prob': max(0.05,
2**(-0.005 * max(0, cparams['iteration'] - 50))),
'is_temperature': max(1.0,
2**(750.0 / max(1, cparams['iteration'] - 50))),
'is_ess_scale': 10,
'is_fisher_chirality_flip': cparams['iteration'] < 2500,
'is_on_R': True,
'is_global_prob_R': 0.9,
'is_on_I': True,
'is_global_prob_I': 1e-10,
'is_on_S': True,
'is_global_prob_S': 0.9,
'is_gaussian_sigmascale_S': 0.67,
}
cparams.update(is_params)
return cryodata, (M, fM), cparams
from __future__ import print_function, division
import os
import sys
sys.path.append(os.path.dirname(sys.path[0]))
import argparse
from matplotlib import pyplot as plt
from numpy import unravel_index, log, maximum
from cryoio import mrc
from geometry import gen_dense_beamstop_mask
parser = argparse.ArgumentParser()
parser.add_argument("mrcs_files", help="list of mrcs files.", nargs='+')
args = parser.parse_args()
mrcs_files = args.mrcs_files
if not isinstance(mrcs_files, list):
mrcs_files = [mrcs_files]
for ph in mrcs_files:
M = mrc.readMRC(ph)
N = M.shape[0]
mask_3D = gen_dense_beamstop_mask(N, 3, 0.01, psize = 18)
mrc.writeMRC(ph, M*mask_3D, psz=18)
def sparse(mode, totalmass, use_ac, freq_start, freq_end):
# 1000000
dataset_dir = 'data/EMD6044_xfel_5000_totalmass_{}_oversampling_6'.format(
str(totalmass).zfill(7))
data_params = {
'dataset_name': "1AON",
'inpath': os.path.join(dataset_dir, 'imgdata.mrc'),
'gtpath': os.path.join(dataset_dir, 'ctf_gt.par'),
'ctfpath': os.path.join(dataset_dir, 'defocus.txt'),
'microscope_params': {
'akv': 200,
'wgh': 0.07,
'cs': 2.0
},
'resolution': 3.0 * 6,
'pixel_size': 3.0 * 6,
# 'sigma': 'noise_std',
# 'sigma_out': 'data_std',
# 'minisize': 150,
'num_images': 200,
# 'euler_angles': None,
}
# euler_angles = []
# with open(data_params['gtpath']) as par:
# par.readline()
# # 'C PHI THETA PSI SHX SHY FILM DF1 DF2 ANGAST'
# while True:
# try:
# line = par.readline().split()
# euler_angles.append([float(line[1]), float(line[2]), float(line[3])])
# except Exception:
# break
# data_params['euler_angles'] = np.deg2rad(np.asarray(euler_angles))[0:data_params['num_images']]
# from scripts.analyze_sparse import get_quadrature
# quad_domain_R, _ = get_quadrature(N=124, const_rad=0.8)
# euler_angles = geometry.genEA(quad_domain_R.dirs)
# rand_idxs = np.arange(euler_angles.shape[0])
# np.random.shuffle(rand_idxs)
# data_params['euler_angles'] = euler_angles[rand_idxs]
refined_model = mrc.readMRC(
'particle/EMD-6044-cropped_fM_totalmass_{}_oversampling_6.mrc'.format(
str(totalmass).zfill(7)))
cryodata = SimpleDataset(None, data_params, None)
if use_ac == 1:
use_angular_correlation = True
elif use_ac == 0:
use_angular_correlation = False
else:
raise NotImplementedError
cparams = {
'max_frequency': None,
'beamstop_freq': 0.01,
'learn_like_envelope_bfactor': 500
}
# print("Using angular correlation patterns:", use_angular_correlation)
sk = SimpleKernel(cryodata,
use_angular_correlation=use_angular_correlation)
# if mode == 1:
# for freq in np.arange(0.015, 0.055, 0.005):
# for freq in np.arange(0.025, 0.055, 0.005):
# for freq in np.arange(0.035, 0.055, 0.005):
# cparams['max_frequency'] = freq
# sk.set_data(refined_model, cparams)
# # for i in range(cryodata.num_images):
# for i in range(500):
# tic = time.time()
# workspace = sk.worker(i)
# print("idx: {}, time per worker: {}".format(i, time.time()-tic))
# # sk.plot_distribution(workspace, sk.quad_domain_R, sk.quad_domain_I,
# # correct_ea=cryodata.euler_angles[i], lognorm=False)
# with open('sparse/mode-{}-totalmass-{}-use_ac-{}-freq-{}-workspace.pkl'.format(mode, totalmass, use_angular_correlation, freq), 'wb') as pkl:
# pickle.dump(sk.cached_workspace, pkl)
# elif mode == 2:
# for freq_start, freq_end in zip(np.arange(0.010, 0.05, 0.005), np.arange(0.015, 0.055, 0.005)):
# for freq_start, freq_end in zip(np.arange(0.020, 0.05, 0.005), np.arange(0.025, 0.055, 0.005)):
# for freq_start, freq_end in zip(np.arange(0.030, 0.05, 0.005), np.arange(0.035, 0.055, 0.005)):
cparams['max_frequency'] = freq_end
cparams['beamstop_freq'] = freq_start
sk.set_data(refined_model, cparams)
# sk.concurrent_worker(range(200))
# sk.plot_distribution(sk.cached_cphi[1], sk.quad_domain_R, sk.quad_domain_I,
# correct_ea=cryodata.euler_angles[1], lognorm=False)
timer = list()
for i in range(4):
tic = time.time()
_ = sk.worker(i)
toc = time.time() - tic
# print("idx: {}, time per worker: {}".format(i, toc))
timer.append(toc)
# workspace = sk.worker(i)
# sk.plot_distribution(workspace, sk.quad_domain_R, sk.quad_domain_I,
# correct_ea=cryodata.euler_angles[i], lognorm=False)
print("freq_start-{:.3f}-freq_end-{:.3f}-time-{}".format(
freq_start, freq_end,
sum(timer[2:]) / len(timer[2:])))
axes[2].imshow(ac_proj, origin='lower')
axes[2].set_title('angular correlation image', fontdict={'size': 14})
for ax in axes:
ax.set_xticks([0, int(N/4), int(N/2), int(N*3/4), N-1])
ax.set_yticks([0, int(N/4), int(N/2), int(N*3/4), N-1])
fig.tight_layout()
fig.savefig('angular_correlation_vis.png', dpi=300)
plt.show()
if __name__ == '__main__':
print(sys.argv)
mrc_file = sys.argv[1]
M = mrc.readMRC(mrc_file)
# M[M<30.4] = 0
# realspace_benchmark(M, calc_corr_img=True)
# realspace_benchmark_comparison(M)
# realspace_benchmark_new_algorithm(M, calc_corr_img=False)
# realspace_benchmark_new_algorithm_comparison(M)
# fourierspace_benchmark_new_algorithm_comparison(M, rad=0.6)
# fourierspace_benchmark_new_algorithm_comparison(M, modulus=True)
# fourierspace_benchmark_comparison(M)
# fourierspace_benchmark_comparison(M, modulus=True)
# ea = np.deg2rad([60, 60, 0])
# realspace_benchmark_comparison(M, save_animation=True, animation_name='realspace_old')
import numpy as np
import matplotlib.pyplot as plt
from cryoio import mrc
import density, cryoops
import geometry
from quadrature import healpix
from cryoem import cryoem
import pyximport; pyximport.install(
setup_args={"include_dirs": np.get_include()}, reload_support=True)
import sincint
M = mrc.readMRC('./particle/EMD-6044.mrc')
# M = mrc.readMRC('./particle/1AON.mrc')
# M = M / np.sum(M)
M = M[:124, :124, :124]
mrc.writeMRC('./particle/EMD-6044-cropped.mrc', M, psz=3.0)
N = M.shape[0]
print(M.shape)
rad = 1
kernel = 'lanczos'
ksize = 4
xy, trunc_xy, truncmask = geometry.gencoords(N, 2, rad, True)
# premult = cryoops.compute_premultiplier(N, kernel='lanczos', kernsize=6)
premult = cryoops.compute_premultiplier(N, kernel, ksize)
TtoF = sincint.gentrunctofull(N=N, rad=rad)
from cryoem import relion
from cryoio import mrc, star
import geometry
from quadrature import healpix as hp
WD = '/Users/lqhuang/Git/SOD-cryoem'
# phantompath = os.path.join(WD, 'particle', '1AON.mrc')
phantompath = os.path.join(WD, 'particle', 'EMD-6044.mrc')
recom_contour = 17.0
N = 128
sigma_noise = 25.0
M_totalmass = 80000
M = mrc.readMRC(phantompath)
M[M < recom_contour] = 0
if M_totalmass is not None:
M *= M_totalmass / M.sum()
tic = time()
def gen_euler_angles(num_EAs):
EAs = list()
for i in range(num_EAs):
# Randomly generate the viewing direction/shift
pt = np.random.randn(3)
pt /= np.linalg.norm(pt)
psi = 2 * np.pi * np.random.rand()
EA = geometry.genEA(pt)[0]
import cryoem
from quadrature import SK97Quadrature
import pyximport
pyximport.install(setup_args={"include_dirs": np.get_include()},
reload_support=True)
import sincint
if __name__ == '__main__':
psize = 3.0 * 3
freq = 0.005
rad = freq * 2.0 * psize
beamstop_freq = 0.003
beamstop_rad = beamstop_freq * 2.0 * psize
fM = mrc.readMRC('../particle/EMD-6044-cropped-non-negative-256.mrc')
N = fM.shape[0]
TtoF = sincint.gentrunctofull(N=N, rad=rad, beamstop_rad=beamstop_rad)
theta = np.arange(0, 2 * np.pi, 2 * np.pi / 12)
degree_R, resolution_R = SK97Quadrature.compute_degree(N, rad, 1.0)
dirs, weights = SK97Quadrature.get_quad_points(degree_R, None)
Rs = np.vstack([
geometry.rotmat3D_dir(vec)[:, 0:2].reshape((1, 3, 2)) for vec in dirs
])
N_R = dirs.shape[0]
N_I = theta.shape[0]
N_T = TtoF.shape[1]
print(N_R, N_I, N_T)
def load_kernel(data_dir, model_file, use_angular_correlation=False, sample_shifts=False):
data_params = {
'dataset_name': "1AON",
'inpath': os.path.join(data_dir, 'imgdata.mrc'),
'ctfpath': os.path.join(data_dir, 'defocus.txt'),
'microscope_params': {'akv': 200, 'wgh': 0.07, 'cs': 2.0},
'resolution': 2.8,
'sigma': 'noise_std',
'sigma_out': 'data_std',
'minisize': 20,
'test_imgs': 20,
'partition': 0,
'num_partitions': 0,
'random_seed': 1,
# 'symmetry': 'C7'
}
print("Loading dataset %s" % data_dir)
cryodata, _ = dataset_loading_test(data_params)
# mleDC, _, mleDC_est_std = cryodata.get_dc_estimate()
# modelscale = (np.abs(mleDC) + 2*mleDC_est_std)/cryodata.N
modelscale = 1.0
if model_file is not None:
print("Loading density map %s" % model_file)
M = mrc.readMRC(model_file)
else:
print("Generating random initial density map ...")
M = cryoem.generate_phantom_density(cryodata.N, 0.95 * cryodata.N / 2.0, \
5 * cryodata.N / 128.0, 30, seed=0)
M *= modelscale/M.sum()
slice_interp = {'kern': 'lanczos', 'kernsize': 4, 'zeropad': 0, 'dopremult': True}
fM = M
minibatch = cryodata.get_next_minibatch(shuffle_minibatches=False)
is_sym = get_symmetryop(data_params.get('symmetry',None))
sampler_R = FixedFisherImportanceSampler('_R', is_sym)
sampler_I = FixedFisherImportanceSampler('_I')
sampler_S = None
cparams = {
'use_angular_correlation': use_angular_correlation,
'iteration': 0,
'pixel_size': cryodata.pixel_size,
'max_frequency': 0.02,
'interp_kernel_R': 'lanczos',
'interp_kernel_size_R': 4,
'interp_zeropad_R': 0,
'interp_premult_R': True,
'interp_kernel_I': 'lanczos',
'interp_kernel_size_I': 4,
'interp_zeropad_I': 0.0,
'interp_premult_I': True,
'quad_shiftsigma': 10,
'quad_shiftextent': 60,
'sigma': cryodata.noise_var,
# 'symmetry': 'C7'
}
kernel = UnknownRSThreadedCPUKernel()
kernel.setup(cparams, None, None, None)
kernel.set_samplers(sampler_R, sampler_I, sampler_S)
kernel.set_dataset(cryodata)
kernel.precomp_slices = None
kernel.set_data(cparams, minibatch)
kernel.using_precomp_slicing = False
kernel.using_precomp_inplane = False
kernel.M = M
kernel.fM = fM
return kernel
for i in range(data_params['num_images']):
tic = time.time()
workspace = sk.worker(i)
print("idx: {}, time per worker: {}".format(i, time.time()-tic))
sk.plot_distribution(workspace, sk.quad_domain_R, sk.quad_domain_I,
correct_ea=cryodata.euler_angles[i], lognorm=False)
# workspace['cphi_R'] = workspace['full_like_cphi_R']
# workspace['cphi_I'] = workspace['full_like_cphi_I']
# sk.plot_distribution(workspace, sk.quad_domain_R, sk.quad_domain_I,
# correct_ea=cryodata.euler_angles[i], lognorm=False)
# sk.plot_rmsd()
if __name__ == '__main__':
model_file = sys.argv[1]
# refined_model_file = sys.argv[2]
print("Loading 3D density: %s" % model_file)
# print("Loading refined 3D density: %s" % refined_model_file)
# data_dir = 'data/1AON_xfel_5000_totalmass_05000_oversampling'
# kernel = kernel_test(data_dir, model_file, use_angular_correlation=False)
model = mrc.readMRC(model_file)
refined_model = model # = mrc.readMRC(refined_model_file)
likelihood_estimate(model, refined_model, use_angular_correlation=False)
def genphantomdata(N_D, phantompath):
mscope_params = {
'akv': 200,
'wgh': 0.07,
'cs': 2.0,
'psize': 3.0,
'bfactor': 500.0
}
M = mrc.readMRC(phantompath)
N = M.shape[0]
rad = 0.95
M_totalmass = 1000000
# M_totalmass = 1500000
kernel = 'lanczos'
ksize = 6
tic = time.time()
N_D = int(N_D)
N = int(N)
rad = float(rad)
psize = mscope_params['psize']
bfactor = mscope_params['bfactor']
M_totalmass = float(M_totalmass)
ctfparfile = 'particle/examplectfs.par'
srcctf_stack = CTFStack(ctfparfile, mscope_params)
genctf_stack = GeneratedCTFStack(
mscope_params, parfields=['PHI', 'THETA', 'PSI', 'SHX', 'SHY'])
TtoF = sincint.gentrunctofull(N=N, rad=rad)
Cmap = np.sort(
np.random.random_integers(0,
srcctf_stack.get_num_ctfs() - 1, N_D))
cryoem.window(M, 'circle')
M[M < 0] = 0
if M_totalmass is not None:
M *= M_totalmass / M.sum()
# oversampling
oversampling_factor = 3
psize = psize * oversampling_factor
V = density.real_to_fspace_with_oversampling(M, oversampling_factor)
fM = V.real**2 + V.imag**2
# mrc.writeMRC('particle/EMD6044_fM_totalmass_{}_oversampling_{}.mrc'.format(str(int(M_totalmass)).zfill(5), oversampling_factor), fM, psz=psize)
print("Generating data...")
sys.stdout.flush()
imgdata = np.empty((N_D, N, N), dtype=density.real_t)
pardata = {'R': []}
prevctfI = None
coords = geometry.gencoords(N, 2, rad)
slicing_func = RegularGridInterpolator((np.arange(N), ) * 3,
fM,
bounds_error=False,
fill_value=0.0)
for i, srcctfI in enumerate(Cmap):
ellapse_time = time.time() - tic
remain_time = float(N_D - i) * ellapse_time / max(i, 1)
print("\r%.2f Percent.. (Elapsed: %s, Remaining: %s)" %
(i / float(N_D) * 100.0, format_timedelta(ellapse_time),
format_timedelta(remain_time)),
end='')
sys.stdout.flush()
# Get the CTF for this image
cCTF = srcctf_stack.get_ctf(srcctfI)
if prevctfI != srcctfI:
genctfI = genctf_stack.add_ctf(cCTF)
C = cCTF.dense_ctf(N, psize, bfactor).reshape((N, N))
prevctfI = srcctfI
# Randomly generate the viewing direction/shift
pt = np.random.randn(3)
pt /= np.linalg.norm(pt)
psi = 2 * np.pi * np.random.rand()
EA = geometry.genEA(pt)[0]
EA[2] = psi
# Rotate coordinates and get slice image by interpolation
R = geometry.rotmat3D_EA(*EA)[:, 0:2]
rotated_coords = R.dot(coords.T).T + int(N / 2)
slice_data = slicing_func(rotated_coords)
intensity = TtoF.dot(slice_data)
np.maximum(intensity, 0.0, out=intensity)
# Add poisson noise
img = np.float_(np.random.poisson(intensity.reshape(N, N)))
np.maximum(1e-8, img, out=img)
imgdata[i] = np.require(img, dtype=density.real_t)
genctf_stack.add_img(genctfI,
PHI=EA[0] * 180.0 / np.pi,
THETA=EA[1] * 180.0 / np.pi,
PSI=EA[2] * 180.0 / np.pi,
SHX=0.0,
SHY=0.0)
pardata['R'].append(R)
print("\n\rDone in ", time.time() - tic, " seconds.")
return imgdata, genctf_stack, pardata, mscope_params
theta = np.arange(0, 2*np.pi, 2*np.pi/36)
EAs = np.asarray([[i, j] for i in np.arange(0, np.pi, np.pi/12.0) for j in np.arange(0, 2*np.pi, 2*np.pi/6.0)])
print(EAs.shape)
EAs = np.vstack((EAs.T, np.zeros(EAs.shape[0]))).T
dirs = geometry.genDir(EAs)
N_R = dirs.shape[0]
N_I = theta.shape[0]
N_T = FtoT.shape[0]
print("length of truncation slice", N_T)
# M = cryoem.generate_phantom_density(N, 0.95*N/2.0, 5 * N / 128.0, 30, seed=1)
M = mrc.readMRC('particle/1AON.mrc')
zeropad = 1
zeropad_size = int(zeropad * (N / 2))
zp_N = zeropad_size * 2 + N
zp_M_shape = (zp_N,) * 3
ZP_M = np.zeros(zp_M_shape, dtype=density.real_t)
zp_M_slicer = (slice( zeropad_size, (N + zeropad_size) ),) * 3
M_totalmass = 5000
M *= M_totalmass / M.sum()
ZP_M[zp_M_slicer] = M
N = M.shape[0]
kernel = 'lanczos'
fig, axes = plt.subplots(4, 4, figsize=(12.8, 8))
im_real = axes[0, 0].imshow(real_proj)
im_fourier = axes[1, 0].imshow(np.log(np.abs(fourier_slice)))
for i, ax in enumerate(axes[:, 1:].T):
shift = np.random.randn(2) * (N / 4.0)
S = cryoops.compute_shift_phases(shift.reshape(1, 2), N, rad)[0]
shift_trunc_slice = S * trunc_slice
shift_fourier_slice = TtoF.dot(shift_trunc_slice).reshape(N, N)
shift_real_proj = density.fspace_to_real(shift_fourier_slice)
ax[0].imshow(shift_real_proj)
ax[1].imshow(np.log(np.abs(shift_fourier_slice)))
ax[2].imshow(np.log(shift_fourier_slice.real))
ax[3].imshow(np.log(shift_fourier_slice.imag))
fig.tight_layout()
plt.show()
if __name__ == '__main__':
# M = mrc.readMRC('./particle/EMD-6044.mrc')
# M = M[:124, :124, :124]
# mrc.writeMRC('./particle/EMD-6044-cropped.mrc', M, psz=3.0)
print(sys.argv)
M = mrc.readMRC(sys.argv[1])
shift_vis(M)
def genphantomdata(N_D, phantompath, ctfparfile):
# mscope_params = {'akv': 200, 'wgh': 0.07,
# 'cs': 2.0, 'psize': 2.8, 'bfactor': 500.0}
mscope_params = {'akv': 200, 'wgh': 0.07,
'cs': 2.0, 'psize': 3.0, 'bfactor': 500.0}
M = mrc.readMRC(phantompath)
N = M.shape[0]
rad = 0.95
shift_sigma = 3.0
sigma_noise = 25.0
M_totalmass = 80000
kernel = 'lanczos'
ksize = 6
premult = cryoops.compute_premultiplier(N, kernel, ksize)
tic = time.time()
N_D = int(N_D)
N = int(N)
rad = float(rad)
psize = mscope_params['psize']
bfactor = mscope_params['bfactor']
shift_sigma = float(shift_sigma)
sigma_noise = float(sigma_noise)
M_totalmass = float(M_totalmass)
srcctf_stack = CTFStack(ctfparfile, mscope_params)
genctf_stack = GeneratedCTFStack(mscope_params, parfields=[
'PHI', 'THETA', 'PSI', 'SHX', 'SHY'])
TtoF = sincint.gentrunctofull(N=N, rad=rad)
Cmap = np.sort(np.random.random_integers(
0, srcctf_stack.get_num_ctfs() - 1, N_D))
cryoem.window(M, 'circle')
M[M < 0] = 0
if M_totalmass is not None:
M *= M_totalmass / M.sum()
V = density.real_to_fspace(
premult.reshape((1, 1, -1)) * premult.reshape((1, -1, 1)) * premult.reshape((-1, 1, 1)) * M)
print("Generating data...")
sys.stdout.flush()
imgdata = np.empty((N_D, N, N), dtype=density.real_t)
pardata = {'R': [], 't': []}
prevctfI = None
for i, srcctfI in enumerate(Cmap):
ellapse_time = time.time() - tic
remain_time = float(N_D - i) * ellapse_time / max(i, 1)
print("\r%.2f Percent.. (Elapsed: %s, Remaining: %s)" % (i / float(N_D)
* 100.0, format_timedelta(ellapse_time), format_timedelta(remain_time)))
sys.stdout.flush()
# Get the CTF for this image
cCTF = srcctf_stack.get_ctf(srcctfI)
if prevctfI != srcctfI:
genctfI = genctf_stack.add_ctf(cCTF)
C = cCTF.dense_ctf(N, psize, bfactor).reshape((N**2,))
prevctfI = srcctfI
# Randomly generate the viewing direction/shift
pt = np.random.randn(3)
pt /= np.linalg.norm(pt)
psi = 2 * np.pi * np.random.rand()
EA = geometry.genEA(pt)[0]
EA[2] = psi
shift = np.random.randn(2) * shift_sigma
R = geometry.rotmat3D_EA(*EA)[:, 0:2]
slop = cryoops.compute_projection_matrix(
[R], N, kernel, ksize, rad, 'rots')
S = cryoops.compute_shift_phases(shift.reshape((1, 2)), N, rad)[0]
D = slop.dot(V.reshape((-1,)))
D *= S
imgdata[i] = density.fspace_to_real((C * TtoF.dot(D)).reshape((N, N))) + np.require(
np.random.randn(N, N) * sigma_noise, dtype=density.real_t)
genctf_stack.add_img(genctfI,
PHI=EA[0] * 180.0 / np.pi, THETA=EA[1] * 180.0 / np.pi, PSI=EA[2] * 180.0 / np.pi,
SHX=shift[0], SHY=shift[1])
pardata['R'].append(R)
pardata['t'].append(shift)
print("\rDone in ", time.time() - tic, " seconds.")
return imgdata, genctf_stack, pardata, mscope_params
def __init__(self, expbase, cmdparams=None):
"""cryodata is a CryoData instance.
expbase is a path to the base of folder where this experiment's files
will be stored. The folder above expbase will also be searched
for .params files. These will be loaded first."""
BackgroundWorker.__init__(self)
# Create a background thread which handles IO
self.io_queue = Queue()
self.io_thread = Thread(target=self.ioworker)
self.io_thread.daemon = True
self.io_thread.start()
# General setup ----------------------------------------------------
self.expbase = expbase
self.outbase = None
# Paramter setup ---------------------------------------------------
# search above expbase for params files
_,_,filenames = os.walk(opj(expbase,'../')).next()
self.paramfiles = [opj(opj(expbase,'../'), fname) \
for fname in filenames if fname.endswith('.params')]
# search expbase for params files
_,_,filenames = os.walk(opj(expbase)).next()
self.paramfiles += [opj(expbase,fname) \
for fname in filenames if fname.endswith('.params')]
if 'local.params' in filenames:
self.paramfiles += [opj(expbase,'local.params')]
# load parameter files
self.params = Params(self.paramfiles)
self.cparams = None
if cmdparams is not None:
# Set parameter specified on the command line
for k,v in cmdparams.iteritems():
self.params[k] = v
# Dataset setup -------------------------------------------------------
self.imgpath = self.params['inpath']
psize = self.params['resolution']
if not isinstance(self.imgpath,list):
imgstk = MRCImageStack(self.imgpath,psize)
else:
imgstk = CombinedImageStack([MRCImageStack(cimgpath,psize) for cimgpath in self.imgpath])
if self.params.get('float_images',True):
imgstk.float_images()
self.ctfpath = self.params['ctfpath']
mscope_params = self.params['microscope_params']
if not isinstance(self.ctfpath,list):
ctfstk = CTFStack(self.ctfpath,mscope_params)
else:
ctfstk = CombinedCTFStack([CTFStack(cctfpath,mscope_params) for cctfpath in self.ctfpath])
self.cryodata = CryoDataset(imgstk,ctfstk)
self.cryodata.compute_noise_statistics()
if self.params.get('window_images',True):
imgstk.window_images()
minibatch_size = self.params['minisize']
testset_size = self.params['test_imgs']
partition = self.params.get('partition',0)
num_partitions = self.params.get('num_partitions',1)
seed = self.params['random_seed']
if isinstance(partition,str):
partition = eval(partition)
if isinstance(num_partitions,str):
num_partitions = eval(num_partitions)
if isinstance(seed,str):
seed = eval(seed)
self.cryodata.divide_dataset(minibatch_size,testset_size,partition,num_partitions,seed)
self.cryodata.set_datasign(self.params.get('datasign','auto'))
if self.params.get('normalize_data',True):
self.cryodata.normalize_dataset()
self.voxel_size = self.cryodata.pixel_size
# Iterations setup -------------------------------------------------
self.iteration = 0
self.tic_epoch = None
self.num_data_evals = 0
self.eval_params()
outdir = self.cparams.get('outdir',None)
if outdir is None:
if self.cparams.get('num_partitions',1) > 1:
outdir = 'partition{0}'.format(self.cparams['partition'])
else:
outdir = ''
self.outbase = opj(self.expbase,outdir)
if not os.path.isdir(self.outbase):
os.makedirs(self.outbase)
# Output setup -----------------------------------------------------
self.ostream = OutputStream(opj(self.outbase,'stdout'))
self.ostream(80*"=")
self.ostream("Experiment: " + expbase + \
" Kernel: " + self.params['kernel'])
self.ostream("Started on " + socket.gethostname() + \
" At: " + time.strftime('%B %d %Y: %I:%M:%S %p'))
self.ostream("Git SHA1: " + gitutil.git_get_SHA1())
self.ostream(80*"=")
gitutil.git_info_dump(opj(self.outbase, 'gitinfo'))
self.startdatetime = datetime.now()
# for diagnostics and parameters
self.diagout = Output(opj(self.outbase, 'diag'),runningout=False)
# for stats (per image etc)
self.statout = Output(opj(self.outbase, 'stat'),runningout=True)
# for likelihoods of individual images
self.likeout = Output(opj(self.outbase, 'like'),runningout=False)
self.img_likes = n.empty(self.cryodata.N_D)
self.img_likes[:] = n.inf
# optimization state vars ------------------------------------------
init_model = self.cparams.get('init_model',None)
if init_model is not None:
filename = init_model
if filename.upper().endswith('.MRC'):
M = readMRC(filename)
else:
with open(filename) as fp:
M = cPickle.load(fp)
if type(M)==list:
M = M[-1]['M']
if M.shape != 3*(self.cryodata.N,):
M = cryoem.resize_ndarray(M,3*(self.cryodata.N,),axes=(0,1,2))
else:
init_seed = self.cparams.get('init_random_seed',0) + self.cparams.get('partition',0)
print "Randomly generating initial density (init_random_seed = {0})...".format(init_seed), ; sys.stdout.flush()
tic = time.time()
M = cryoem.generate_phantom_density(self.cryodata.N, 0.95*self.cryodata.N/2.0, \
5*self.cryodata.N/128.0, 30, seed=init_seed)
print "done in {0}s".format(time.time() - tic)
tic = time.time()
print "Windowing and aligning initial density...", ; sys.stdout.flush()
# window the initial density
wfunc = self.cparams.get('init_window','circle')
cryoem.window(M,wfunc)
# Center and orient the initial density
cryoem.align_density(M)
print "done in {0:.2f}s".format(time.time() - tic)
# apply the symmetry operator
init_sym = get_symmetryop(self.cparams.get('init_symmetry',self.cparams.get('symmetry',None)))
if init_sym is not None:
tic = time.time()
print "Applying symmetry operator...", ; sys.stdout.flush()
M = init_sym.apply(M)
print "done in {0:.2f}s".format(time.time() - tic)
tic = time.time()
print "Scaling initial model...", ; sys.stdout.flush()
modelscale = self.cparams.get('modelscale','auto')
mleDC, _, mleDC_est_std = self.cryodata.get_dc_estimate()
if modelscale == 'auto':
# Err on the side of a weaker prior by using a larger value for modelscale
modelscale = (n.abs(mleDC) + 2*mleDC_est_std)/self.cryodata.N
print "estimated modelscale = {0:.3g}...".format(modelscale), ; sys.stdout.flush()
self.params['modelscale'] = modelscale
self.cparams['modelscale'] = modelscale
M *= modelscale/M.sum()
print "done in {0:.2f}s".format(time.time() - tic)
if mleDC_est_std/n.abs(mleDC) > 0.05:
print " WARNING: the DC component estimate has a high relative variance, it may be inaccurate!"
if ((modelscale*self.cryodata.N - n.abs(mleDC)) / mleDC_est_std) > 3:
print " WARNING: the selected modelscale value is more than 3 std devs different than the estimated one. Be sure this is correct."
self.M = n.require(M,dtype=density.real_t)
self.fM = density.real_to_fspace(M)
self.dM = density.zeros_like(self.M)
self.step = eval(self.cparams['optim_algo'])
self.step.setup(self.cparams, self.diagout, self.statout, self.ostream)
# Objective function setup --------------------------------------------
param_type = self.cparams.get('parameterization','real')
cplx_param = param_type in ['complex','complex_coeff','complex_herm_coeff']
self.like_func = eval_objective(self.cparams['likelihood'])
self.prior_func = eval_objective(self.cparams['prior'])
if self.cparams.get('penalty',None) is not None:
self.penalty_func = eval_objective(self.cparams['penalty'])
prior_func = SumObjectives(self.prior_func.fspace, \
[self.penalty_func,self.prior_func], None)
else:
prior_func = self.prior_func
self.obj = SumObjectives(cplx_param,
[self.like_func,prior_func], [None,None])
self.obj.setup(self.cparams, self.diagout, self.statout, self.ostream)
self.obj.set_dataset(self.cryodata)
self.obj_wrapper = ObjectiveWrapper(param_type)
self.last_save = time.time()
self.logpost_history = FiniteRunningSum()
self.like_history = FiniteRunningSum()
# Importance Samplers -------------------------------------------------
self.is_sym = get_symmetryop(self.cparams.get('is_symmetry',self.cparams.get('symmetry',None)))
self.sampler_R = FixedFisherImportanceSampler('_R',self.is_sym)
self.sampler_I = FixedFisherImportanceSampler('_I')
self.sampler_S = FixedGaussianImportanceSampler('_S')
self.like_func.set_samplers(sampler_R=self.sampler_R,sampler_I=self.sampler_I,sampler_S=self.sampler_S)
from cryoio import mrc, star
import geometry
from quadrature import healpix as hp
WD = '/Users/lqhuang/Git/SOD-cryoem'
# phantompath = os.path.join(WD, 'particle', '1AON.mrc')
phantompath = os.path.join(WD, 'particle', 'EMD-6044.mrc')
recom_contour = 17.0
N = 128
sigma_noise = 25.0
M_totalmass = 80000
M = mrc.readMRC(phantompath)
M[M<recom_contour] = 0
if M_totalmass is not None:
M *= M_totalmass / M.sum()
tic = time()
def gen_euler_angles(num_EAs):
EAs = list()
for i in range(num_EAs):
# Randomly generate the viewing direction/shift
pt = np.random.randn(3)
pt /= np.linalg.norm(pt)
psi = 2 * np.pi * np.random.rand()
EA = geometry.genEA(pt)[0]
EA[2] = psi
pcimg = np.zeros((int(radius), int(angle)))
print(pcimg.shape)
return pcimg
def get_corr_img(img, pcimg_interpolation='nearest'):
"""
get a angular correlation image
"""
if 'nearest' in pcimg_interpolation.lower():
pcimg = imgpolarcoord(img)
elif 'linear' in pcimg_interpolation.lower():
pcimg = imgpolarcoord3(img)
pcimg_fourier = np.fft.fftshift(np.fft.fft(pcimg, axis=1))
corr_img = np.fft.ifft(np.fft.ifftshift(pcimg_fourier*np.conjugate(pcimg_fourier)), axis=1)
return corr_img.real
if __name__ == '__main__':
from cryoio import mrc
from matplotlib import pyplot as plt
map_file = '../particle/EMD-2325.map'
model = mrc.readMRC(map_file)
proj1 = np.sum(model, axis=2)
corr_img = get_corr_img(proj1, pcimg_interpolation='linear')
plt.figure(1)
plt.imshow(proj1)
# plt.figure(2)
# plt.imshow(c2_img)
plt.show()
def plot_figures():
oversampling_factor = 6
psize = 3 * oversampling_factor
freq = 0.05
rad = 0.5 * 2.0 * psize
beamstop_freq = 0.005
beamstop_rad = beamstop_freq * 2.0 * psize
# M = mrc.readMRC('particle/EMD-6044-cropped.mrc')
# M[M < 0] = 0
# M_totalmass = 2000000
# if M_totalmass is not None:
# M *= M_totalmass / M.sum()
# fM = density.real_to_fspace_with_oversampling(M, oversampling_factor=oversampling_factor)
# fM = fM.real ** 2 + fM.imag ** 2
# mrc.writeMRC("particle/EMD-6044-cropped_fM_totalmass_%d_oversampling_%d.mrc" % (M_totalmass, oversampling_factor), fM, psz=psize)
# exit()
fM = mrc.readMRC(
'particle/EMD-6044-cropped_fM_totalmass_2000000_oversampling_6.mrc')
N = fM.shape[0]
TtoF = sincint.gentrunctofull(N=N, rad=rad, beamstop_rad=beamstop_rad)
theta = np.arange(0, 2 * np.pi, 2 * np.pi / 60)
degree_R, resolution_R = SK97Quadrature.compute_degree(N, 0.3, 1.0)
dirs, weights = SK97Quadrature.get_quad_points(degree_R, None)
Rs = np.vstack([
geometry.rotmat3D_dir(vec)[:, 0:2].reshape((1, 3, 2)) for vec in dirs
])
N_R = dirs.shape[0]
N_I = theta.shape[0]
N_T = TtoF.shape[1]
# generate slicing operators
dir_slice_interp = {
'projdirs': dirs,
'N': N,
'kern': 'lanczos',
'kernsize': 6,
'projdirtype': 'dirs',
'onlyRs': True,
'rad': rad,
'sym': None
} # 'zeropad': 0, 'dopremult': True
R_slice_interp = {
'projdirs': Rs,
'N': N,
'kern': 'lanczos',
'kernsize': 6,
'projdirtype': 'rots',
'onlyRs': True,
'rad': rad,
'sym': None
} # 'zeropad': 0, 'dopremult': True
inplane_interp = {
'thetas': theta,
'N': N,
'kern': 'lanczos',
'kernsize': 6,
'onlyRs': True,
'rad': rad
} # 'zeropad': 1, 'dopremult': True
inplane_interp['N_src'] = N # zp_N
slice_ops = cryoops.compute_projection_matrix(**dir_slice_interp)
inplane_ops = cryoops.compute_inplanerot_matrix(**inplane_interp)
# generate slices and inplane-rotated slices
slices_sampled = cryoem.getslices_interp(
fM, slice_ops, rad, beamstop_rad=beamstop_rad).reshape((N_R, N_T))
curr_img = TtoF.dot(slices_sampled[0]).reshape(N, N)
rotd_sampled = cryoem.getslices_interp(curr_img,
inplane_ops,
rad,
beamstop_rad=beamstop_rad).reshape(
(N_I, N_T))
## plot figures
fig, axes = plt.subplots(3, 4, figsize=(9.6, 7.2))
for i, ax in enumerate(axes.flatten()):
img = TtoF.dot(slices_sampled[i]).reshape(N, N)
ax.imshow(img, origin='lower')
fig.suptitle('slices_sampled')
fig, axes = plt.subplots(3, 4, figsize=(9.6, 7.2))
for i, ax in enumerate(axes.flatten()):
img = TtoF.dot(slices_sampled[i]).reshape(N, N)
ax.imshow(img, origin='lower')
fig.suptitle('rotd_sampled')
plt.show()
def genphantomdata(N_D, phantompath, ctfparfile):
mscope_params = {
'akv': 200,
'wgh': 0.07,
'cs': 2.0,
'psize': 2.8,
'bfactor': 500.0
}
N = 128
rad = 0.95
shift_sigma = 3.0
sigma_noise = 25.0
M_totalmass = 80000
kernel = 'lanczos'
ksize = 6
premult = cryoops.compute_premultiplier(N, kernel, ksize)
tic = time.time()
N_D = int(N_D)
N = int(N)
rad = float(rad)
psize = mscope_params['psize']
bfactor = mscope_params['bfactor']
shift_sigma = float(shift_sigma)
sigma_noise = float(sigma_noise)
M_totalmass = float(M_totalmass)
srcctf_stack = CTFStack(ctfparfile, mscope_params)
genctf_stack = GeneratedCTFStack(
mscope_params, parfields=['PHI', 'THETA', 'PSI', 'SHX', 'SHY'])
TtoF = sincint.gentrunctofull(N=N, rad=rad)
Cmap = n.sort(
n.random.random_integers(0,
srcctf_stack.get_num_ctfs() - 1, N_D))
M = mrc.readMRC(phantompath)
cryoem.window(M, 'circle')
M[M < 0] = 0
if M_totalmass is not None:
M *= M_totalmass / M.sum()
V = density.real_to_fspace(
premult.reshape((1, 1, -1)) * premult.reshape(
(1, -1, 1)) * premult.reshape((-1, 1, 1)) * M)
print "Generating data..."
sys.stdout.flush()
imgdata = n.empty((N_D, N, N), dtype=density.real_t)
pardata = {'R': [], 't': []}
prevctfI = None
for i, srcctfI in enumerate(Cmap):
ellapse_time = time.time() - tic
remain_time = float(N_D - i) * ellapse_time / max(i, 1)
print "\r%.2f Percent.. (Elapsed: %s, Remaining: %s) " % (
i / float(N_D) * 100.0, format_timedelta(ellapse_time),
format_timedelta(remain_time)),
sys.stdout.flush()
# Get the CTF for this image
cCTF = srcctf_stack.get_ctf(srcctfI)
if prevctfI != srcctfI:
genctfI = genctf_stack.add_ctf(cCTF)
C = cCTF.dense_ctf(N, psize, bfactor).reshape((N**2, ))
prevctfI = srcctfI
# Randomly generate the viewing direction/shift
pt = n.random.randn(3)
pt /= n.linalg.norm(pt)
psi = 2 * n.pi * n.random.rand()
EA = geom.genEA(pt)[0]
EA[2] = psi
shift = n.random.randn(2) * shift_sigma
R = geom.rotmat3D_EA(*EA)[:, 0:2]
slop = cryoops.compute_projection_matrix([R], N, kernel, ksize, rad,
'rots')
S = cryoops.compute_shift_phases(shift.reshape((1, 2)), N, rad)[0]
D = slop.dot(V.reshape((-1, )))
D *= S
imgdata[i] = density.fspace_to_real((C * TtoF.dot(D)).reshape(
(N, N))) + n.require(n.random.randn(N, N) * sigma_noise,
dtype=density.real_t)
genctf_stack.add_img(genctfI,
PHI=EA[0] * 180.0 / n.pi,
THETA=EA[1] * 180.0 / n.pi,
PSI=EA[2] * 180.0 / n.pi,
SHX=shift[0],
SHY=shift[1])
pardata['R'].append(R)
pardata['t'].append(shift)
print "\rDone in ", time.time() - tic, " seconds."
return imgdata, genctf_stack, pardata, mscope_params
par_file = os.path.join(data_dir, 'pardata.pkl')
# with open(par_file, 'rb') as pkl_file:
# pardata = pickle.load(pkl_file)
euler_angles = list()
with open(os.path.join(data_dir, 'ctf_gt.par')) as par:
par.readline()
# 'C PHI THETA PSI SHX SHY FILM DF1 DF2 ANGAST'
while True:
try:
line = par.readline().split()
euler_angles.append([float(line[1]), float(line[2]), float(line[3])])
except Exception:
break
euler_angles = np.deg2rad(np.asarray(euler_angles))
imgdata = mrc.readMRC(img_file)
num_data = imgdata.shape[2]
rad = 0.8
FtoT = sincint.genfulltotrunc(N=N, rad=rad)
N_T = FtoT.shape[0]
slices = np.zeros((num_data, N_T), dtype=np.float32)
for i in range(num_data):
curr_img = imgdata[:, :, i]
slices[i] = FtoT.dot(curr_img.reshape(-1, 1)).reshape(-1)
Rs = np.vstack([geometry.rotmat3D_EA(*ea)[:, 0:2].reshape((1, 3, 2)) for ea in euler_angles])
trunc = FtoT.dot(image.flatten())
corr_trunc = calc_angular_correlation(trunc, N, rad, **ac_kwargs)
full_angular_correlation = TtoF.dot(corr_trunc)
if outside:
_, _, outside_mask = gencoords_outside(N, 2, rad, True)
corr_trunc_outside = calc_angular_correlation(
image[outside_mask.reshape(N, N)].flatten(),
N,
rad,
outside=True,
**ac_kwargs)
full_angular_correlation[outside_mask] = corr_trunc_outside
return full_angular_correlation.reshape(N, N)
if __name__ == '__main__':
from cryoio import mrc
from matplotlib import pyplot as plt
map_file = '../particle/1AON.mrc'
model = mrc.readMRC(map_file)
proj = np.sum(model, axis=2)
c2_img_nearest = get_corr_img(proj, pcimg_interpolation='nearest')
c2_img_linear = get_corr_img(proj, pcimg_interpolation='linear')
plt.figure(1)
plt.imshow(proj)
plt.figure(2)
plt.imshow(c2_img_linear)
plt.show()
from cryoio.mrc import writeMRC, readMRC filename = 'Data/Beta/Particles/Falcon_2012_06_12-14_33_35_0.mrc' filename = 'exp/Beta_sagd_noinit/model.mrc' filename = 'Data/Beta/init.mrc' m, hdr = readMRC(filename, inc_header=True) print hdr