def dump_subvol(self,picking_result): from aitom.classify.deep.unsupervised.autoencoder.autoencoder_util import peaks_to_subvolumes subvols_loc = os.path.join(self.dump_path,"demo_single_particle_subvolumes.pickle") a = io_file.read_mrc_data(self.path) d = peaks_to_subvolumes(im_vol_util.cub_img(a)['vt'], picking_result, 32) io_file.pickle_dump(d, subvols_loc) print("Save subvolumes .pickle file to:", subvols_loc)
def picking(path, s1, s2, t, find_maxima=True, partition_op=None, multiprocessing_process_num=0): a = io_file.read_mrc_data(path) print("file has been read") temp = im_vol_util.cub_img(a) a_im = temp['im'] # image data a_vt = temp['vt'] # volume data peaks = peak__partition( a_vt, s1=s1, s2=s2, find_maxima=find_maxima, partition_op=partition_op, multiprocessing_process_num=multiprocessing_process_num) # using DoG to detect all peaks, may contain peaks caused by noise # calculate threshold T and delete peaks whose val are smaller than threshold # Related paper: Pei L, Xu M, Frazier Z, Alber F. Simulating Cryo-Electron Tomograms of Crowded Mixtures of Macromolecular Complexes and Assessment of Particle Picking. BMC Bioinformatics. 2016; 17: 405. M = peaks[0]['val'] # max val of all peaks m = peaks[len(peaks) - 1]['val'] # min val of all peaks T = m + t * (M - m) / 20 for i in range(len(peaks)): if peaks[i]['val'] < T: res = peaks[0:i - 1] break print("T=m+t*(M-m)/20 \nT=%f m=%f t=%f M=%f" % (T, m, t, M)) return res
def picking(path, s1, s2, t, find_maxima=True, partition_op=None, multiprocessing_process_num=0, pick_num=None): ''' parameters: path:file path s1:sigma1 s2:sigma2 t:threshold level find_maxima:peaks appears at the maximum/minimum multiprocessing_process_num: number of multiporcessing partition_op: partition the volume for multithreading, is a dict consists 'nonoverlap_width', 'overlap_width' and 'save_vg' pick_num: the max number of particles to pick out # Take a two-dimensional image as an example, if the image size is 210*150(all in pixels), nonoverlap_width is 60 and overlap_width is 30. # It will be divided into 6 pieces for different threads to process. The ranges of their X and Y are # (first line) (0-90)*(0-90) (60-150)*(0-90) (120-210)*(0-90) (0-90) # (second line) (0-90)*(60-150) (60-150)*(60-150) (120-210)*(60-150) In general, s2=1.1*s1, s1 and t depend on particle size and noise. In practice, s1 should be roughly equal to the particle radius(in pixels). In related paper, the model achieves highest comprehensive score when s1=7 and t=3. return: a list including all peaks information (in descending order of value), each element in the return list looks like: {'val': 281.4873046875, 'x': [1178, 1280, 0], 'uuid': '6ad66107-088c-471e-b65f-0b3b2fdc35b0'} 'val' is the score of the peak when picking, only the score is higher than the threshold will the peak be selected. 'x' is the center of the peak in the tomogram. 'uuid' is an unique id for each peak. ''' a = io_file.read_mrc_data(path) print("file has been read") temp = im_vol_util.cub_img(a) a_im = temp['im'] # image data a_vt = temp['vt'] # volume data # using DoG to detect all peaks, may contain peaks caused by noise peaks = peak__partition( a_vt, s1=s1, s2=s2, find_maxima=find_maxima, partition_op=partition_op, multiprocessing_process_num=multiprocessing_process_num) # calculate threshold T and delete peaks whose val are smaller than threshold # Related paper: Pei L, Xu M, Frazier Z, Alber F. Simulating Cryo-Electron Tomograms of Crowded Mixtures of Macromolecular Complexes and Assessment of Particle Picking. BMC Bioinformatics. 2016; 17: 405. M = peaks[0]['val'] # max val of all peaks m = peaks[len(peaks) - 1]['val'] # min val of all peaks T = m + t * (M - m) / 20 peak_vals_neg = [-peak['val'] * find_maxima for peak in peaks] res = peaks[:bisect(peak_vals_neg, -T * find_maxima) - 1] assert res[-1]['val'] >= T print("%d particles detected, containing redundant peaks" % len(res)) result = do_filter(pp=res, peak_dist_min=s1, op=None) # remove redundant peaks print("peak number reduced to %d" % len(result)) if pick_num is None: pass elif pick_num < len(res): res = res[:pick_num] print("T=m+t*(M-m)/20 \nT=%f m=%f t=%f M=%f" % (T, m, t, M)) return res
def kmeans_centers_plot(clus_center_dir): kmeans_clus = AIF.pickle_load(op_join(clus_center_dir, 'kmeans.pickle')) ccents = AIF.pickle_load(op_join(clus_center_dir, 'ccents.pickle')) # export slices for visual inspection if False: for i in ccents: auto.dsp_cub(ccents[i]) else: clus_center_figure_dir = op_join(clus_center_dir, 'fig') if os.path.isdir(clus_center_figure_dir): shutil.rmtree(clus_center_figure_dir) os.makedirs(clus_center_figure_dir) # normalize across all images min_t = N.min([ccents[_].min() for _ in ccents]) max_t = N.max([ccents[_].max() for _ in ccents]) assert max_t > min_t ccents_t = {_: ((ccents[_] - min_t) / (max_t - min_t)) for _ in ccents} ccents_t = ccents for i in ccents_t: clus_siz = len(kmeans_clus[i]) t = AIVU.cub_img(ccents_t[i])['im'] AIIO.save_png(t, op_join(clus_center_figure_dir, '%003d--%d.png' % (i, clus_siz)), normalize=False)
def particle_picking(mrc_header): sigma1 = max(int(7 / voxel_spacing_in_nm), 2) # In general, 7 is optimal sigma1 val in nm according to the paper and sigma1 should at least be 2 print('sigma1=%d' % sigma1) # For particular tomogram, larger sigma1 value may have better results. Use IMOD to display selected peaks and determine best sigma1. # For 'aitom_demo_cellular_tomogram.mrc', sigma1 is 5 rather than 3 for better performance(in this tomogram, 7nm corresponds to 3.84 pixels) # print(mrc_header['MRC']['xlen'], mrc_header['MRC']['nx'], voxel_spacing_in_nm, sigma1) partition_op = {'nonoverlap_width': sigma1 * 20, 'overlap_width': sigma1 * 10, 'save_vg': False} result = picking(path, s1=sigma1, s2=sigma1 * 1.1, t=3, find_maxima=False, partition_op=partition_op, multiprocessing_process_num=10, pick_num=1000) print("DoG done, %d particles picked" % len(result)) pprint(result[:5]) # (Optional) Save subvolumes of peaks for autoencoder input dump_subvols = True if dump_subvols: # use later for autoencoder subvols_loc = "demo_single_particle_subvolumes.pickle" from aitom.classify.deep.unsupervised.autoencoder.autoencoder_util import peaks_to_subvolumes a = io_file.read_mrc_data(path) d = peaks_to_subvolumes(im_vol_util.cub_img(a)['vt'], result, 32) io_file.pickle_dump(d, subvols_loc) print("Save subvolumes .pickle file to:", subvols_loc)
def active_contour_slice(v, sigma=3.5, membrane_thickness=5, display_slice=-1, out_dir = './output', save_flag=True): if os.path.exists(out_dir): shutil.rmtree(out_dir) os.makedirs(out_dir) mask_v = np.zeros(v.shape) print(v.shape) vg = FG.smooth(v, sigma) for slice_num in range(mask_v.shape[2]): save_flag_slice = False if slice_num == display_slice and save_flag: save_flag_slice = True print('processing slice ', slice_num) ori_img = (v[:, :, slice_num]).copy() if save_flag_slice: plt.imsave(os.path.join(out_dir, 'original.png'), ori_img, cmap='gray') img = (vg[:, :, slice_num]).copy() if save_flag_slice: plt.imsave(os.path.join(out_dir, 'original_smooth.png'), img, cmap='gray') # generate init circle s = np.linspace(0, 2*np.pi, 400) x = 80 + 60*np.sin(s) y = 80 + 60*np.cos(s) init = np.array([x, y]).T # init = create_sphere(20,20,20,10) # sphere for 3d active contour snake = active_contour(img, init, alpha=0.015, beta=10, gamma=0.001) # Note: The default format of the returned coordinates is (x,y) instead of (row,col) in skimage 0.15.x - 0.17.x snake[:,[0,1]]= snake[:, [1,0]] r = snake[:,0].astype(int) c = snake[:,1].astype(int) img2 = img.copy() colour = np.min(img2) img2[r,c] = colour if save_flag_slice: plt.imsave(os.path.join(out_dir, 'contour.png'), img2, cmap='gray') mask = np.zeros(ori_img.shape) rr,cc = polygon(r, c) mask[rr,cc] = 2 mask[r,c] = 1 for i in range(membrane_thickness): mask = contour_shrink(mask) mask[mask==0] = 3 if save_flag_slice: plt.imsave(os.path.join(out_dir, 'mask.png'), mask, cmap='gray') mask_v [:,:,slice_num] = mask mask_im = AIVU.cub_img(mask_v)['im'] # better visualize the results for i in range(0,mask_im.shape[0],mask_v.shape[0]): mask_im[i,:]=0 for i in range(0,mask_im.shape[1],mask_v.shape[1]): mask_im[:,i]=0 if save_flag: plt.imsave(os.path.join(out_dir, 'result.png'), mask_im, cmap='gray') return mask_v
def main(): # Download from: https://cmu.box.com/s/9hn3qqtqmivauus3kgtasg5uzlj53wxp path = '/ldap_shared/home/v_zhenxi_zhu/data/aitom_demo_single_particle_tomogram.mrc' # Also, we can crop and only use part of the mrc image instead of binning for tasks requiring higher resolution # crop_path = 'cropped.mrc' # crop_mrc(path, crop_path) mrc_header = io_file.read_mrc_header(path) voxel_spacing_in_nm = mrc_header['MRC']['xlen'] / mrc_header['MRC'][ 'nx'] / 10 sigma1 = max( int(7 / voxel_spacing_in_nm), 2 ) # In general, 7 is optimal sigma1 val in nm according to the paper and sigma1 should at least be 2 print('sigma1=%d' % sigma1) # For particular tomogram, larger sigma1 value may have better results. Use IMOD to display selected peaks and determine best sigma1. # For 'aitom_demo_cellular_tomogram.mrc', sigma1 is 5 rather than 3 for better performance(in this tomogram, 7nm corresponds to 3.84 pixels) # print(mrc_header['MRC']['xlen'], mrc_header['MRC']['nx'], voxel_spacing_in_nm, sigma1) partition_op = { 'nonoverlap_width': sigma1 * 20, 'overlap_width': sigma1 * 10, 'save_vg': False } result = picking(path, s1=sigma1, s2=sigma1 * 1.1, t=3, find_maxima=False, partition_op=partition_op, multiprocessing_process_num=10, pick_num=1000) print("DoG done, %d particles picked" % len(result)) pprint(result[:5]) # (Optional) Save subvolumes of peaks for autoencoder input dump_subvols = True if dump_subvols: # use later for autoencoder subvols_loc = "demo_single_particle_subvolumes.pickle" from aitom.classify.deep.unsupervised.autoencoder.autoencoder_util import peaks_to_subvolumes a = io_file.read_mrc_data(path) d = peaks_to_subvolumes(im_vol_util.cub_img(a)['vt'], result, 32) io_file.pickle_dump(d, subvols_loc) print("Save subvolumes .pickle file to:", subvols_loc) # Display selected peaks using imod/3dmod (http://bio3d.colorado.edu/imod/) ''' #Optional: smooth original image a = io_file.read_mrc_data(path) path =path[:-5]+'_smoothed'+path[-4:] temp = im_vol_util.cub_img(a) s1 = sigma1 s2=sigma1*1.1 vg = dog_smooth(temp['vt'], s1,s2) #vg = smooth(temp['vt'], s1) TIM.write_data(vg,path) ''' json_data = [] # generate file for 3dmod for i in range(len(result)): loc_np = result[i]['x'] loc = [] for j in range(len(loc_np)): loc.append(loc_np[j].tolist()) json_data.append({'peak': {'loc': loc}}) with open('data_json_file.json', 'w') as f: json.dump(json_data, f) dj = json_data x = N.zeros((len(dj), 3)) for i, d in enumerate(dj): x[i, :] = N.array(d['peak']['loc']) l = generate_lines(x_full=x, rad=sigma1) display_map_with_lines(l=l, map_file=path)
def sphere_seg(v, sigma, init_level_set=None, out_dir='./output', save_flag=False): """ sphere_seg: membrane segmentation by sphere fitting @params: v: volume data sigma: gaussian sigma for denoising init_level_set: initial level set for morphsnake @return: [x, y, z, R]: coordinates and radius of the sphere """ np.random.seed(12345) if save_flag: if os.path.exists(out_dir): shutil.rmtree(out_dir) os.makedirs(out_dir) # morphsnake if init_level_set: init = init_level_set else: circle_set = circle_level_set(v.shape[:2], (80, 80), 60) init_mask = np.zeros(v.shape) for i in range(init_mask.shape[2]): init_mask[:, :, i] = circle_set init = checkerboard_level_set(v.shape) init = np.multiply(init, init_mask) v_im = AIVU.cub_img(v)['im'] if save_flag: plt.imsave(os.path.join(out_dir, 'original.png'), v_im, cmap='gray') vg = FG.smooth(v, sigma) mask_v = ACWE(image=vg, iterations=25, init_level_set=init) mask_im = AIVU.cub_img(mask_v)['im'] if save_flag: plt.imsave(os.path.join(out_dir, 'morphsnake_result.png'), mask_im, cmap='gray') # RANSC coords = np.array(np.where(mask_v == 1)) coords = coords.T # (N,3) # robustly fit line only using inlier data with RANSAC algorithm xyz = coords model_robust, inliers = ransac(xyz, CircleModel3D, min_samples=20, residual_threshold=3, max_trials=50) outliers = inliers == False # print('inliers_num = ', sum(inliers), 'inliers_num = ', sum(outliers)) x, y, z, R = model_robust.params v_RANSC = np.zeros(mask_v.shape) assert len(inliers) == coords.shape[0] if save_flag: for i in range(len(inliers)): if inliers[i]: v_RANSC[coords[i][0]][coords[i][1]][coords[i][2]] = 2 else: v_RANSC[coords[i][0]][coords[i][1]][coords[i][2]] = 1 vim_RANSC = AIVU.cub_img(v_RANSC)['im'] plt.imsave(os.path.join(out_dir, 'RANSC_result.png'), vim_RANSC, cmap='gray') a = FG.smooth(v, sigma) a.flags.writeable = True color = np.min(a) thickness = 1 center = (x, y, z) grid = np.mgrid[[slice(i) for i in a.shape]] grid = (grid.T - center).T phi1 = R - np.sqrt(np.sum((grid)**2, 0)) phi2 = np.max(R - thickness, 0) - np.sqrt(np.sum((grid)**2, 0)) res = np.int8(phi1 > 0) - np.int8(phi2 > 0) a[res == 1] = color vim = AIVU.cub_img(a)['im'] plt.imsave(os.path.join(out_dir, 'result.png'), vim, cmap='gray') return model_robust.params
def visualize(img, path): t = AIVU.cub_img(img)['im'] AIIO.save_png(t, path)
# http://ftp.ebi.ac.uk/pub/databases/empiar/archive/10045/data/ribosomes/AnticipatedResults/Particles/Tomograms/05/IS002_291013_005_subtomo000001.mrc import aitom.io.file as IF a = IF.read_mrc_data("data/IS002_291013_005_subtomo000001.mrc") # denoising using gaussian filter for visualization from aitom.filter.gaussian import smooth a = smooth(a, sigma=8) # display image using image module import aitom.image.vol.util as IVU ''' a_im is a dict: 'im': image data, type of numpy.ndarray, elements in [0, 1] 'vt': volume data ''' a_im = IVU.cub_img(a) print(type(a_im['im'])) print(a_im['im'].shape) print(a_im['im'][1][1]) import matplotlib.pyplot as plt plt.imshow(a_im['im'], cmap='gray') # save the figure into a png file import aitom.image.io as IIO IIO.save_png(m=a_im['im'], name='/tmp/a_im.png') # display image using `image.util.dsp_cub` IVU.dsp_cub(a) # save slices of a into a png file
def map2png(map, file): import aitom.image.io as IIO import aitom.image.vol.util as TIVU IIO.save_png(TIVU.cub_img(map)['im'], file)
def save_cub_img(v, name, view_dir=2): import aitom.image.vol.util as TIVU m = TIVU.cub_img(v=v, view_dir=view_dir)['im'] save_png(m=m, name=name)
http://ftp.ebi.ac.uk/pub/databases/empiar/archive/10045/data/ribosomes/AnticipatedResults/Particles/Tomograms/05/IS002_291013_005_subtomo000001.mrc ''' import aitom.io.file as io_file a = io_file.read_mrc_data("data/IS002_291013_005_subtomo000001.mrc") # denoising using gaussian filter for visualization from aitom.filter.gaussian import smooth a = smooth(a, sigma=8) # display image using image module import aitom.image.vol.util as im_vol_util a_im = im_vol_util.cub_img(a) ''' a_im is a dict: 'im': image data 'vt': volume data image data is type of numpy.ndarray, elements $\in$ [0, 1] ''' print(type(a_im['im'])) print(a_im['im'].shape) print(a_im['im'][1][1]) import matplotlib.pyplot as plt #%matplotlib notebook
def output_image(v, path): """ Saves the sliced image of model v to path """ AIIO.save_png(AIVU.cub_img(v)['im'], path)
import aitom.geometry.ang_loc as TGAL import aitom.geometry.rotate as TGR import random import numpy as N # randomly rotate and translate v loc_proportion = 0.1 loc_max = N.array(v.shape, dtype=float) * loc_proportion angle = TGAL.random_rotation_angle_zyz() loc_r = (N.random.random(3) - 0.5) * loc_max vr = TGR.rotate(v, angle=angle, loc_r=loc_r, default_val=0.0) #-------------------------------- # align vr against v import aitom.align.util as TAU al = TAU.align_vols_no_mask(v, vr) print('rigid transform of alignment', al) vr_i = TGR.rotate( vr, angle=al['angle'], loc_r=al['loc'], default_val=0.0 ) # rotate vr according to the alignment, expected to produce an image similiar to v # save images of the slices of the corresponding 3D iamges for visual inspection import aitom.image.io as TIIO import aitom.image.vol.util as TIVU TIIO.save_png(TIVU.cub_img(v)['im'], "v.png") TIIO.save_png(TIVU.cub_img(vr)['im'], "vr.png") TIIO.save_png(TIVU.cub_img(vr_i)['im'], "vr_i.png")
def output_image(v, path): TIIO.save_png(TIVU.cub_img(v)['im'], path)
loc_proportion = 0.1 loc_max = N.array(v.shape, dtype=float) * loc_proportion angle = GAL.random_rotation_angle_zyz() loc_r = (N.random.random(3) - 0.5) * loc_max vr = GR.rotate(v, angle=angle, loc_r=loc_r, default_val=0.0) # generate simulated subtomogram vb from v vb = TSRSC.do_reconstruction(vr, op, verbose=True) print('vb', 'mean', vb.mean(), 'std', vb.std(), 'var', vb.var()) # save v and vb as 3D grey scale images IF.put_mrc(vb, '/tmp/vb.mrc', overwrite=True) IF.put_mrc(v, '/tmp/v.mrc', overwrite=True) # save images of the slices of the corresponding 3D iamges for visual inspection import aitom.image.io as IIO import aitom.image.vol.util as IVU IIO.save_png(IVU.cub_img(vb)['im'], "/tmp/vb.png") IIO.save_png(IVU.cub_img(v)['im'], "/tmp/v.png") if True: # verify the correctness of SNR estimation vb_rep = TSRSC.do_reconstruction(vr, op, verbose=True) import scipy.stats as SS # calculate SNR vb_corr = SS.pearsonr(vb.flatten(), vb_rep.flatten())[0] vb_snr = 2 * vb_corr / (1 - vb_corr) print('SNR', 'parameter', op['model']['SNR'], 'estimated', vb_snr) # fsc = ssnr / (2.0 + ssnr)