def dump_subvol(self,picking_result):
from aitom.classify.deep.unsupervised.autoencoder.autoencoder_util import peaks_to_subvolumes
subvols_loc = os.path.join(self.dump_path,"demo_single_particle_subvolumes.pickle")
a = io_file.read_mrc_data(self.path)
d = peaks_to_subvolumes(im_vol_util.cub_img(a)['vt'], picking_result, 32)
io_file.pickle_dump(d, subvols_loc)
print("Save subvolumes .pickle file to:", subvols_loc)
def picking(path,
s1,
s2,
t,
find_maxima=True,
partition_op=None,
multiprocessing_process_num=0):
a = io_file.read_mrc_data(path)
print("file has been read")
temp = im_vol_util.cub_img(a)
a_im = temp['im'] # image data
a_vt = temp['vt'] # volume data
peaks = peak__partition(
a_vt,
s1=s1,
s2=s2,
find_maxima=find_maxima,
partition_op=partition_op,
multiprocessing_process_num=multiprocessing_process_num)
# using DoG to detect all peaks, may contain peaks caused by noise
# calculate threshold T and delete peaks whose val are smaller than threshold
# Related paper: Pei L, Xu M, Frazier Z, Alber F. Simulating Cryo-Electron Tomograms of Crowded Mixtures of Macromolecular Complexes and Assessment of Particle Picking. BMC Bioinformatics. 2016; 17: 405.
M = peaks[0]['val'] # max val of all peaks
m = peaks[len(peaks) - 1]['val'] # min val of all peaks
T = m + t * (M - m) / 20
for i in range(len(peaks)):
if peaks[i]['val'] < T:
res = peaks[0:i - 1]
break
print("T=m+t*(M-m)/20 \nT=%f m=%f t=%f M=%f" % (T, m, t, M))
return res
def picking(path,
s1,
s2,
t,
find_maxima=True,
partition_op=None,
multiprocessing_process_num=0,
pick_num=None):
'''
parameters:
path:file path s1:sigma1 s2:sigma2 t:threshold level find_maxima:peaks appears at the maximum/minimum multiprocessing_process_num: number of multiporcessing
partition_op: partition the volume for multithreading, is a dict consists 'nonoverlap_width', 'overlap_width' and 'save_vg'
pick_num: the max number of particles to pick out
# Take a two-dimensional image as an example, if the image size is 210*150(all in pixels), nonoverlap_width is 60 and overlap_width is 30.
# It will be divided into 6 pieces for different threads to process. The ranges of their X and Y are
# (first line) (0-90)*(0-90) (60-150)*(0-90) (120-210)*(0-90) (0-90)
# (second line) (0-90)*(60-150) (60-150)*(60-150) (120-210)*(60-150)
In general, s2=1.1*s1, s1 and t depend on particle size and noise. In practice, s1 should be roughly equal to the particle radius(in pixels). In related paper, the model achieves highest comprehensive score when s1=7 and t=3.
return:
a list including all peaks information (in descending order of value), each element in the return list looks like:
{'val': 281.4873046875, 'x': [1178, 1280, 0], 'uuid': '6ad66107-088c-471e-b65f-0b3b2fdc35b0'}
'val' is the score of the peak when picking, only the score is higher than the threshold will the peak be selected.
'x' is the center of the peak in the tomogram.
'uuid' is an unique id for each peak.
'''
a = io_file.read_mrc_data(path)
print("file has been read")
temp = im_vol_util.cub_img(a)
a_im = temp['im'] # image data
a_vt = temp['vt'] # volume data
# using DoG to detect all peaks, may contain peaks caused by noise
peaks = peak__partition(
a_vt,
s1=s1,
s2=s2,
find_maxima=find_maxima,
partition_op=partition_op,
multiprocessing_process_num=multiprocessing_process_num)
# calculate threshold T and delete peaks whose val are smaller than threshold
# Related paper: Pei L, Xu M, Frazier Z, Alber F. Simulating Cryo-Electron Tomograms of Crowded Mixtures of Macromolecular Complexes and Assessment of Particle Picking. BMC Bioinformatics. 2016; 17: 405.
M = peaks[0]['val'] # max val of all peaks
m = peaks[len(peaks) - 1]['val'] # min val of all peaks
T = m + t * (M - m) / 20
peak_vals_neg = [-peak['val'] * find_maxima for peak in peaks]
res = peaks[:bisect(peak_vals_neg, -T * find_maxima) - 1]
assert res[-1]['val'] >= T
print("%d particles detected, containing redundant peaks" % len(res))
result = do_filter(pp=res, peak_dist_min=s1,
op=None) # remove redundant peaks
print("peak number reduced to %d" % len(result))
if pick_num is None:
pass
elif pick_num < len(res):
res = res[:pick_num]
print("T=m+t*(M-m)/20 \nT=%f m=%f t=%f M=%f" % (T, m, t, M))
return res
def kmeans_centers_plot(clus_center_dir):
kmeans_clus = AIF.pickle_load(op_join(clus_center_dir, 'kmeans.pickle'))
ccents = AIF.pickle_load(op_join(clus_center_dir, 'ccents.pickle'))
# export slices for visual inspection
if False:
for i in ccents:
auto.dsp_cub(ccents[i])
else:
clus_center_figure_dir = op_join(clus_center_dir, 'fig')
if os.path.isdir(clus_center_figure_dir):
shutil.rmtree(clus_center_figure_dir)
os.makedirs(clus_center_figure_dir)
# normalize across all images
min_t = N.min([ccents[_].min() for _ in ccents])
max_t = N.max([ccents[_].max() for _ in ccents])
assert max_t > min_t
ccents_t = {_: ((ccents[_] - min_t) / (max_t - min_t)) for _ in ccents}
ccents_t = ccents
for i in ccents_t:
clus_siz = len(kmeans_clus[i])
t = AIVU.cub_img(ccents_t[i])['im']
AIIO.save_png(t,
op_join(clus_center_figure_dir,
'%003d--%d.png' % (i, clus_siz)),
normalize=False)
def particle_picking(mrc_header):
sigma1 = max(int(7 / voxel_spacing_in_nm),
2) # In general, 7 is optimal sigma1 val in nm according to the paper and sigma1 should at least be 2
print('sigma1=%d' % sigma1)
# For particular tomogram, larger sigma1 value may have better results. Use IMOD to display selected peaks and determine best sigma1.
# For 'aitom_demo_cellular_tomogram.mrc', sigma1 is 5 rather than 3 for better performance(in this tomogram, 7nm corresponds to 3.84 pixels)
# print(mrc_header['MRC']['xlen'], mrc_header['MRC']['nx'], voxel_spacing_in_nm, sigma1)
partition_op = {'nonoverlap_width': sigma1 * 20, 'overlap_width': sigma1 * 10, 'save_vg': False}
result = picking(path, s1=sigma1, s2=sigma1 * 1.1, t=3, find_maxima=False, partition_op=partition_op,
multiprocessing_process_num=10, pick_num=1000)
print("DoG done, %d particles picked" % len(result))
pprint(result[:5])
# (Optional) Save subvolumes of peaks for autoencoder input
dump_subvols = True
if dump_subvols: # use later for autoencoder
subvols_loc = "demo_single_particle_subvolumes.pickle"
from aitom.classify.deep.unsupervised.autoencoder.autoencoder_util import peaks_to_subvolumes
a = io_file.read_mrc_data(path)
d = peaks_to_subvolumes(im_vol_util.cub_img(a)['vt'], result, 32)
io_file.pickle_dump(d, subvols_loc)
print("Save subvolumes .pickle file to:", subvols_loc)
def active_contour_slice(v, sigma=3.5, membrane_thickness=5, display_slice=-1, out_dir = './output', save_flag=True):
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
os.makedirs(out_dir)
mask_v = np.zeros(v.shape)
print(v.shape)
vg = FG.smooth(v, sigma)
for slice_num in range(mask_v.shape[2]):
save_flag_slice = False
if slice_num == display_slice and save_flag:
save_flag_slice = True
print('processing slice ', slice_num)
ori_img = (v[:, :, slice_num]).copy()
if save_flag_slice: plt.imsave(os.path.join(out_dir, 'original.png'), ori_img, cmap='gray')
img = (vg[:, :, slice_num]).copy()
if save_flag_slice: plt.imsave(os.path.join(out_dir, 'original_smooth.png'), img, cmap='gray')
# generate init circle
s = np.linspace(0, 2*np.pi, 400)
x = 80 + 60*np.sin(s)
y = 80 + 60*np.cos(s)
init = np.array([x, y]).T
# init = create_sphere(20,20,20,10) # sphere for 3d active contour
snake = active_contour(img, init, alpha=0.015, beta=10, gamma=0.001)
# Note: The default format of the returned coordinates is (x,y) instead of (row,col) in skimage 0.15.x - 0.17.x
snake[:,[0,1]]= snake[:, [1,0]]
r = snake[:,0].astype(int)
c = snake[:,1].astype(int)
img2 = img.copy()
colour = np.min(img2)
img2[r,c] = colour
if save_flag_slice: plt.imsave(os.path.join(out_dir, 'contour.png'), img2, cmap='gray')
mask = np.zeros(ori_img.shape)
rr,cc = polygon(r, c)
mask[rr,cc] = 2
mask[r,c] = 1
for i in range(membrane_thickness):
mask = contour_shrink(mask)
mask[mask==0] = 3
if save_flag_slice: plt.imsave(os.path.join(out_dir, 'mask.png'), mask, cmap='gray')
mask_v [:,:,slice_num] = mask
mask_im = AIVU.cub_img(mask_v)['im']
# better visualize the results
for i in range(0,mask_im.shape[0],mask_v.shape[0]):
mask_im[i,:]=0
for i in range(0,mask_im.shape[1],mask_v.shape[1]):
mask_im[:,i]=0
if save_flag: plt.imsave(os.path.join(out_dir, 'result.png'), mask_im, cmap='gray')
return mask_v
def main():
# Download from: https://cmu.box.com/s/9hn3qqtqmivauus3kgtasg5uzlj53wxp
path = '/ldap_shared/home/v_zhenxi_zhu/data/aitom_demo_single_particle_tomogram.mrc'
# Also, we can crop and only use part of the mrc image instead of binning for tasks requiring higher resolution
# crop_path = 'cropped.mrc'
# crop_mrc(path, crop_path)
mrc_header = io_file.read_mrc_header(path)
voxel_spacing_in_nm = mrc_header['MRC']['xlen'] / mrc_header['MRC'][
'nx'] / 10
sigma1 = max(
int(7 / voxel_spacing_in_nm), 2
) # In general, 7 is optimal sigma1 val in nm according to the paper and sigma1 should at least be 2
print('sigma1=%d' % sigma1)
# For particular tomogram, larger sigma1 value may have better results. Use IMOD to display selected peaks and determine best sigma1.
# For 'aitom_demo_cellular_tomogram.mrc', sigma1 is 5 rather than 3 for better performance(in this tomogram, 7nm corresponds to 3.84 pixels)
# print(mrc_header['MRC']['xlen'], mrc_header['MRC']['nx'], voxel_spacing_in_nm, sigma1)
partition_op = {
'nonoverlap_width': sigma1 * 20,
'overlap_width': sigma1 * 10,
'save_vg': False
}
result = picking(path,
s1=sigma1,
s2=sigma1 * 1.1,
t=3,
find_maxima=False,
partition_op=partition_op,
multiprocessing_process_num=10,
pick_num=1000)
print("DoG done, %d particles picked" % len(result))
pprint(result[:5])
# (Optional) Save subvolumes of peaks for autoencoder input
dump_subvols = True
if dump_subvols: # use later for autoencoder
subvols_loc = "demo_single_particle_subvolumes.pickle"
from aitom.classify.deep.unsupervised.autoencoder.autoencoder_util import peaks_to_subvolumes
a = io_file.read_mrc_data(path)
d = peaks_to_subvolumes(im_vol_util.cub_img(a)['vt'], result, 32)
io_file.pickle_dump(d, subvols_loc)
print("Save subvolumes .pickle file to:", subvols_loc)
# Display selected peaks using imod/3dmod (http://bio3d.colorado.edu/imod/)
'''
#Optional: smooth original image
a = io_file.read_mrc_data(path)
path =path[:-5]+'_smoothed'+path[-4:]
temp = im_vol_util.cub_img(a)
s1 = sigma1
s2=sigma1*1.1
vg = dog_smooth(temp['vt'], s1,s2)
#vg = smooth(temp['vt'], s1)
TIM.write_data(vg,path)
'''
json_data = [] # generate file for 3dmod
for i in range(len(result)):
loc_np = result[i]['x']
loc = []
for j in range(len(loc_np)):
loc.append(loc_np[j].tolist())
json_data.append({'peak': {'loc': loc}})
with open('data_json_file.json', 'w') as f:
json.dump(json_data, f)
dj = json_data
x = N.zeros((len(dj), 3))
for i, d in enumerate(dj):
x[i, :] = N.array(d['peak']['loc'])
l = generate_lines(x_full=x, rad=sigma1)
display_map_with_lines(l=l, map_file=path)
def sphere_seg(v,
sigma,
init_level_set=None,
out_dir='./output',
save_flag=False):
"""
sphere_seg: membrane segmentation by sphere fitting
@params:
v: volume data sigma: gaussian sigma for denoising
init_level_set: initial level set for morphsnake
@return:
[x, y, z, R]: coordinates and radius of the sphere
"""
np.random.seed(12345)
if save_flag:
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
os.makedirs(out_dir)
# morphsnake
if init_level_set:
init = init_level_set
else:
circle_set = circle_level_set(v.shape[:2], (80, 80), 60)
init_mask = np.zeros(v.shape)
for i in range(init_mask.shape[2]):
init_mask[:, :, i] = circle_set
init = checkerboard_level_set(v.shape)
init = np.multiply(init, init_mask)
v_im = AIVU.cub_img(v)['im']
if save_flag:
plt.imsave(os.path.join(out_dir, 'original.png'), v_im, cmap='gray')
vg = FG.smooth(v, sigma)
mask_v = ACWE(image=vg, iterations=25, init_level_set=init)
mask_im = AIVU.cub_img(mask_v)['im']
if save_flag:
plt.imsave(os.path.join(out_dir, 'morphsnake_result.png'),
mask_im,
cmap='gray')
# RANSC
coords = np.array(np.where(mask_v == 1))
coords = coords.T # (N,3)
# robustly fit line only using inlier data with RANSAC algorithm
xyz = coords
model_robust, inliers = ransac(xyz,
CircleModel3D,
min_samples=20,
residual_threshold=3,
max_trials=50)
outliers = inliers == False
# print('inliers_num = ', sum(inliers), 'inliers_num = ', sum(outliers))
x, y, z, R = model_robust.params
v_RANSC = np.zeros(mask_v.shape)
assert len(inliers) == coords.shape[0]
if save_flag:
for i in range(len(inliers)):
if inliers[i]:
v_RANSC[coords[i][0]][coords[i][1]][coords[i][2]] = 2
else:
v_RANSC[coords[i][0]][coords[i][1]][coords[i][2]] = 1
vim_RANSC = AIVU.cub_img(v_RANSC)['im']
plt.imsave(os.path.join(out_dir, 'RANSC_result.png'),
vim_RANSC,
cmap='gray')
a = FG.smooth(v, sigma)
a.flags.writeable = True
color = np.min(a)
thickness = 1
center = (x, y, z)
grid = np.mgrid[[slice(i) for i in a.shape]]
grid = (grid.T - center).T
phi1 = R - np.sqrt(np.sum((grid)**2, 0))
phi2 = np.max(R - thickness, 0) - np.sqrt(np.sum((grid)**2, 0))
res = np.int8(phi1 > 0) - np.int8(phi2 > 0)
a[res == 1] = color
vim = AIVU.cub_img(a)['im']
plt.imsave(os.path.join(out_dir, 'result.png'), vim, cmap='gray')
return model_robust.params
def visualize(img, path):
t = AIVU.cub_img(img)['im']
AIIO.save_png(t, path)
# http://ftp.ebi.ac.uk/pub/databases/empiar/archive/10045/data/ribosomes/AnticipatedResults/Particles/Tomograms/05/IS002_291013_005_subtomo000001.mrc
import aitom.io.file as IF
a = IF.read_mrc_data("data/IS002_291013_005_subtomo000001.mrc")
# denoising using gaussian filter for visualization
from aitom.filter.gaussian import smooth
a = smooth(a, sigma=8)
# display image using image module
import aitom.image.vol.util as IVU
'''
a_im is a dict:
'im': image data, type of numpy.ndarray, elements in [0, 1]
'vt': volume data
'''
a_im = IVU.cub_img(a)
print(type(a_im['im']))
print(a_im['im'].shape)
print(a_im['im'][1][1])
import matplotlib.pyplot as plt
plt.imshow(a_im['im'], cmap='gray')
# save the figure into a png file
import aitom.image.io as IIO
IIO.save_png(m=a_im['im'], name='/tmp/a_im.png')
# display image using `image.util.dsp_cub`
IVU.dsp_cub(a)
# save slices of a into a png file
def map2png(map, file):
import aitom.image.io as IIO
import aitom.image.vol.util as TIVU
IIO.save_png(TIVU.cub_img(map)['im'], file)
def save_cub_img(v, name, view_dir=2):
import aitom.image.vol.util as TIVU
m = TIVU.cub_img(v=v, view_dir=view_dir)['im']
save_png(m=m, name=name)
http://ftp.ebi.ac.uk/pub/databases/empiar/archive/10045/data/ribosomes/AnticipatedResults/Particles/Tomograms/05/IS002_291013_005_subtomo000001.mrc
'''
import aitom.io.file as io_file
a = io_file.read_mrc_data("data/IS002_291013_005_subtomo000001.mrc")
# denoising using gaussian filter for visualization
from aitom.filter.gaussian import smooth
a = smooth(a, sigma=8)
# display image using image module
import aitom.image.vol.util as im_vol_util
a_im = im_vol_util.cub_img(a)
'''
a_im is a dict:
'im': image data
'vt': volume data
image data is type of numpy.ndarray, elements $\in$ [0, 1]
'''
print(type(a_im['im']))
print(a_im['im'].shape)
print(a_im['im'][1][1])
import matplotlib.pyplot as plt
#%matplotlib notebook
def output_image(v, path):
"""
Saves the sliced image of model v to path
"""
AIIO.save_png(AIVU.cub_img(v)['im'], path)
import aitom.geometry.ang_loc as TGAL
import aitom.geometry.rotate as TGR
import random
import numpy as N
# randomly rotate and translate v
loc_proportion = 0.1
loc_max = N.array(v.shape, dtype=float) * loc_proportion
angle = TGAL.random_rotation_angle_zyz()
loc_r = (N.random.random(3) - 0.5) * loc_max
vr = TGR.rotate(v, angle=angle, loc_r=loc_r, default_val=0.0)
#--------------------------------
# align vr against v
import aitom.align.util as TAU
al = TAU.align_vols_no_mask(v, vr)
print('rigid transform of alignment', al)
vr_i = TGR.rotate(
vr, angle=al['angle'], loc_r=al['loc'], default_val=0.0
) # rotate vr according to the alignment, expected to produce an image similiar to v
# save images of the slices of the corresponding 3D iamges for visual inspection
import aitom.image.io as TIIO
import aitom.image.vol.util as TIVU
TIIO.save_png(TIVU.cub_img(v)['im'], "v.png")
TIIO.save_png(TIVU.cub_img(vr)['im'], "vr.png")
TIIO.save_png(TIVU.cub_img(vr_i)['im'], "vr_i.png")
def output_image(v, path):
TIIO.save_png(TIVU.cub_img(v)['im'], path)
loc_proportion = 0.1
loc_max = N.array(v.shape, dtype=float) * loc_proportion
angle = GAL.random_rotation_angle_zyz()
loc_r = (N.random.random(3) - 0.5) * loc_max
vr = GR.rotate(v, angle=angle, loc_r=loc_r, default_val=0.0)
# generate simulated subtomogram vb from v
vb = TSRSC.do_reconstruction(vr, op, verbose=True)
print('vb', 'mean', vb.mean(), 'std', vb.std(), 'var', vb.var())
# save v and vb as 3D grey scale images
IF.put_mrc(vb, '/tmp/vb.mrc', overwrite=True)
IF.put_mrc(v, '/tmp/v.mrc', overwrite=True)
# save images of the slices of the corresponding 3D iamges for visual inspection
import aitom.image.io as IIO
import aitom.image.vol.util as IVU
IIO.save_png(IVU.cub_img(vb)['im'], "/tmp/vb.png")
IIO.save_png(IVU.cub_img(v)['im'], "/tmp/v.png")
if True:
# verify the correctness of SNR estimation
vb_rep = TSRSC.do_reconstruction(vr, op, verbose=True)
import scipy.stats as SS
# calculate SNR
vb_corr = SS.pearsonr(vb.flatten(), vb_rep.flatten())[0]
vb_snr = 2 * vb_corr / (1 - vb_corr)
print('SNR', 'parameter', op['model']['SNR'], 'estimated',
vb_snr) # fsc = ssnr / (2.0 + ssnr)
