def save_mrc(self, aligned=True, fname=None): if fname == None: params = self.diag['params'] name = params['name'] fname = '{0}.mrc'.format(name) mrc.writeMRC(fname,self.alignedM if aligned else self.M, psz=params['resolution'])
def reconstruct(projs_path, nside, iteration_loops, **kwargs): try: WD = kwargs['WD'] except KeyError as error: raise KeyError('lack working directory') exp_samples = mrc.readMRCimgs(projs_path, idx=0) input_shape = exp_samples.shape print('size of input images: {0}x{1}, number of input images: {2}'.format( *input_shape)) print('generating random phantom density') N = input_shape[0] M = cryoem.generate_phantom_density(N, 0.95 * N / 2.0, 5 * N / 128.0, 30) output_mrc = os.path.join(WD, 'initial_random_model.mrc') mrc.writeMRC(output_mrc, M) print('Start projection matching') total_iteration = iteration_loops for it in range(total_iteration): print('Iteration {0}'.format(it)) dir_suffix = 'it' + str(it).zfill(3) iter_dir = os.path.join(WD, dir_suffix) os.makedirs(iter_dir, exist_ok=True) tic = time.time() input_model = output_mrc ori = projection_matching(input_model, exp_samples, nside=nside, dir_suffix=dir_suffix, **kwargs) ori_star = os.path.join(WD, dir_suffix, 'orientations.star') star.easy_writeSTAR_xmipp(ori_star, EAs=ori, imgs_path=projs_path) output_mrc = os.path.join(WD, dir_suffix, dir_suffix + '_reco.mrc') xmipp.reconstruct_fourier(ori_star, output_mrc, thr=4) toc = time.time() - tic print('Iteration {0} finished, spread time: {1:.4f} seconds'.format( it, toc)) print('Total iteration is {0}, remian {1:.2f} minutes'.format( total_iteration, (total_iteration - it) * toc / 60)) print( '---------------------------------------------------------------------------------' )
def ioworker(self): while True: iotype,fname,data = self.io_queue.get() try: if iotype == 'mrc': writeMRC(fname,*data) elif iotype == 'pkl': with open(fname, 'wb') as f: cPickle.dump(data, f, protocol=-1) elif iotype == 'cp': copyfile(fname,data) except: print "ERROR DUMPING {0}: {1}".format(fname, sys.exc_info()[0]) self.io_queue.task_done()
def reconstruct(projs_path, nside, iteration_loops, **kwargs): try: WD = kwargs['WD'] except KeyError as error: raise KeyError('lack working directory') exp_samples = mrc.readMRCimgs(projs_path, idx=0) input_shape = exp_samples.shape print('size of input images: {0}x{1}, number of input images: {2}'.format( *input_shape)) print('generating random phantom density') N = input_shape[0] M = cryoem.generate_phantom_density(N, 0.95 * N / 2.0, 5 * N / 128.0, 30) output_mrc = os.path.join(WD, 'initial_random_model.mrc') mrc.writeMRC(output_mrc, M) print('Start projection matching') total_iteration = iteration_loops for it in range(total_iteration): print('Iteration {0}'.format(it)) dir_suffix = 'it' + str(it).zfill(3) iter_dir = os.path.join(WD, dir_suffix) os.makedirs(iter_dir, exist_ok=True) tic = time.time() input_model = output_mrc ori = projection_matching( input_model, exp_samples, nside=nside, dir_suffix=dir_suffix, **kwargs) ori_star = os.path.join(WD, dir_suffix, 'orientations.star') star.easy_writeSTAR_xmipp(ori_star, EAs=ori, imgs_path=projs_path) output_mrc = os.path.join(WD, dir_suffix, dir_suffix + '_reco.mrc') xmipp.reconstruct_fourier(ori_star, output_mrc, thr=4) toc = time.time() - tic print('Iteration {0} finished, spread time: {1:.4f} seconds'.format(it, toc)) print('Total iteration is {0}, remian {1:.2f} minutes'.format( total_iteration, (total_iteration-it)*toc/60)) print('---------------------------------------------------------------------------------')
%run genphantomdata num inputmrc inputpar output %run genphantomdata 5000 finalphantom.mrc Data/thermus/thermus_Nature2012_128x128.par Data/phantom_5000 """ sys.exit() imgdata, ctfstack, pardata, mscope_params = genphantomdata( sys.argv[1], sys.argv[2], sys.argv[3]) outpath = sys.argv[4] if not os.path.isdir(outpath): os.makedirs(outpath) tic = time.time() print "Dumping data...", sys.stdout.flush() def_path = os.path.join(outpath, 'defocus.txt') ctfstack.write_defocus_txt(def_path) par_path = os.path.join(outpath, 'ctf_gt.par') ctfstack.write_pardata(par_path) mrc_path = os.path.join(outpath, 'imgdata.mrc') mrc.writeMRC(mrc_path, n.transpose(imgdata, (1, 2, 0)), mscope_params['psize']) pard_path = os.path.join(outpath, 'pardata.pkl') with open(pard_path, 'wb') as fi: cPickle.dump(pardata, fi, protocol=2) print "Done in ", time.time() - tic, " seconds." sys.exit()
def gen_slices(model_files, fspace=False, log_scale=True): for model in model_files: M = mrc.readMRC(model) N = M.shape[0] print('model size: {0}x{0}x{0}'.format(N)) oversampling_factor = 3 zeropad = oversampling_factor - 1 # oversampling factor = zeropad + 1 psize = 3.0 * oversampling_factor beamstop_freq = 0.003 mask = geometry.gen_dense_beamstop_mask(N, 2, beamstop_freq, psize=psize) # mask = None if fspace: fM = M else: M_totalmass = 1000000 M *= M_totalmass / M.sum() V = density.real_to_fspace_with_oversampling(M, oversampling_factor) fM = V.real ** 2 + V.imag ** 2 mask_3D = geometry.gen_dense_beamstop_mask(N, 3, beamstop_freq, psize=psize) fM *= mask_3D mrc.writeMRC('particle/{}_fM_totalmass_{}_oversampling_{}.mrc'.format( os.path.splitext(os.path.basename(model))[0], str(int(M_totalmass)).zfill(5), oversampling_factor ), fM, psz=psize) slicing_func = RegularGridInterpolator([np.arange(N),]*3, fM, bounds_error=False, fill_value=0.0) coords = geometry.gencoords_base(N, 2) fig, axes = plt.subplots(3, 3, figsize=(12.9, 9.6)) for i, ax in enumerate(axes.flat): row, col = np.unravel_index(i, (3, 3)) # Randomly generate the viewing direction/shift pt = np.random.randn(3) pt /= np.linalg.norm(pt) psi = 2 * np.pi * np.random.rand() EA = geometry.genEA(pt)[0] EA[2] = psi R = geometry.rotmat3D_EA(*EA)[:, 0:2] rotated_coords = R.dot(coords.T).T + int(N/2) img = slicing_func(rotated_coords).reshape(N, N) img = np.require(np.random.poisson(img), dtype=np.float32) if log_scale: img = np.log(np.maximum(img, 0)) if mask is not None: img *= mask im = ax.imshow(img, origin='lower') # cmap='Greys' ticks = [0, int(N/4.0), int(N/2.0), int(N*3.0/4.0), int(N-1)] if row == 2: ax.set_xticks(ticks) else: ax.set_xticks([]) if col == 0: ax.set_yticks(ticks) else: ax.set_yticks([]) fig.colorbar(im, ax=ax) # fig.subplots_adjust(right=0.8) # cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7]) # fig.colorbar(im, cax=cbar_ax) fig.suptitle('simulated experimental data of XFEL for {}'.format(model)) # fig.tight_layout() plt.show()
import density, cryoops import geometry from quadrature import healpix from cryoem import cryoem import pyximport pyximport.install(setup_args={"include_dirs": np.get_include()}, reload_support=True) import sincint M = mrc.readMRC('./particle/EMD-6044.mrc') # M = mrc.readMRC('./particle/1AON.mrc') # M = M / np.sum(M) M = M[:124, :124, :124] mrc.writeMRC('./particle/EMD-6044-cropped.mrc', M, psz=3.0) N = M.shape[0] print(M.shape) rad = 1 kernel = 'lanczos' ksize = 4 xy, trunc_xy, truncmask = geometry.gencoords(N, 2, rad, True) # premult = cryoops.compute_premultiplier(N, kernel='lanczos', kernsize=6) premult = cryoops.compute_premultiplier(N, kernel, ksize) TtoF = sincint.gentrunctofull(N=N, rad=rad) fM = density.real_to_fspace(M) prefM = density.real_to_fspace( premult.reshape((1, 1, -1)) * premult.reshape(
def __init__(self, expbase, cmdparams=None): """cryodata is a CryoData instance. expbase is a path to the base of folder where this experiment's files will be stored. The folder above expbase will also be searched for .params files. These will be loaded first.""" BackgroundWorker.__init__(self) # Create a background thread which handles IO self.io_queue = Queue() self.io_thread = Thread(target=self.ioworker) self.io_thread.daemon = True self.io_thread.start() # General setup ---------------------------------------------------- self.expbase = os.path.join(expbase, 'logs') self.outbase = None # Paramter setup --------------------------------------------------- # search above expbase for params files _,_,filenames = next(os.walk(opj(expbase,'../'))) self.paramfiles = [opj(opj(expbase,'../'), fname) \ for fname in filenames if fname.endswith('.params')] # search expbase for params files _,_,filenames = next(os.walk(opj(expbase))) self.paramfiles += [opj(expbase,fname) \ for fname in filenames if fname.endswith('.params')] if 'local.params' in filenames: self.paramfiles += [opj(expbase,'local.params')] # load parameter files self.params = Params(self.paramfiles) self.cparams = None if cmdparams is not None: # Set parameter specified on the command line for k,v in cmdparams.items(): self.params[k] = v # Dataset setup ------------------------------------------------------- self.imgpath = self.params['inpath'] psize = self.params['resolution'] if not isinstance(self.imgpath,list): imgstk = MRCImageStack(self.imgpath,psize) else: imgstk = CombinedImageStack([MRCImageStack(cimgpath,psize) for cimgpath in self.imgpath]) if self.params.get('float_images',True): imgstk.float_images() self.ctfpath = self.params['ctfpath'] mscope_params = self.params['microscope_params'] if not isinstance(self.ctfpath,list): ctfstk = CTFStack(self.ctfpath,mscope_params) else: ctfstk = CombinedCTFStack([CTFStack(cctfpath,mscope_params) for cctfpath in self.ctfpath]) self.cryodata = CryoDataset(imgstk,ctfstk) self.cryodata.compute_noise_statistics() if self.params.get('window_images',True): imgstk.window_images() minibatch_size = self.params['minisize'] testset_size = self.params['test_imgs'] partition = self.params.get('partition',0) num_partitions = self.params.get('num_partitions',1) seed = self.params['random_seed'] if isinstance(partition,str): partition = eval(partition) if isinstance(num_partitions,str): num_partitions = eval(num_partitions) if isinstance(seed,str): seed = eval(seed) self.cryodata.divide_dataset(minibatch_size,testset_size,partition,num_partitions,seed) self.cryodata.set_datasign(self.params.get('datasign','auto')) if self.params.get('normalize_data',True): self.cryodata.normalize_dataset() self.voxel_size = self.cryodata.pixel_size # Iterations setup ------------------------------------------------- self.iteration = 0 self.tic_epoch = None self.num_data_evals = 0 self.eval_params() outdir = self.cparams.get('outdir',None) if outdir is None: if self.cparams.get('num_partitions',1) > 1: outdir = 'partition{0}'.format(self.cparams['partition']) else: outdir = '' self.outbase = opj(self.expbase,outdir) if not os.path.isdir(self.outbase): os.makedirs(self.outbase) # Output setup ----------------------------------------------------- self.ostream = OutputStream(opj(self.outbase,'stdout')) self.ostream(80*"=") self.ostream("Experiment: " + expbase + \ " Kernel: " + self.params['kernel']) self.ostream("Started on " + socket.gethostname() + \ " At: " + time.strftime('%B %d %Y: %I:%M:%S %p')) print('gitutil:', gitutil.git_get_SHA1().decode('utf-8')) self.ostream("Git SHA1: " + gitutil.git_get_SHA1().decode('utf-8')) self.ostream(80*"=") gitutil.git_info_dump(opj(self.outbase, 'gitinfo')) self.startdatetime = datetime.now() # for diagnostics and parameters self.diagout = Output(opj(self.outbase, 'diag'),runningout=False) # for stats (per image etc) self.statout = Output(opj(self.outbase, 'stat'),runningout=True) # for likelihoods of individual images self.likeout = Output(opj(self.outbase, 'like'),runningout=False) self.img_likes = np.empty(self.cryodata.N_D) self.img_likes[:] = np.inf # optimization state vars ------------------------------------------ init_model = self.cparams.get('init_model',None) if init_model is not None: filename = init_model if filename.upper().endswith('.MRC'): M = readMRC(filename) else: with open(filename) as fp: M = pickle.load(fp) if type(M)==list: M = M[-1]['M'] if M.shape != 3*(self.cryodata.N,): M = cryoem.resize_ndarray(M,3*(self.cryodata.N,),axes=(0,1,2)) else: init_seed = self.cparams.get('init_random_seed',0) + self.cparams.get('partition',0) print("Randomly generating initial density (init_random_seed = {0})...".format(init_seed)); sys.stdout.flush() tic = time.time() M = cryoem.generate_phantom_density(self.cryodata.N, 0.95*self.cryodata.N/2.0, \ 5*self.cryodata.N/128.0, 30, seed=init_seed) print("done in {0}s".format(time.time() - tic)) tic = time.time() print("Windowing and aligning initial density..."); sys.stdout.flush() # window the initial density wfunc = self.cparams.get('init_window','circle') cryoem.window(M,wfunc) # Center and orient the initial density cryoem.align_density(M) print("done in {0:.2f}s".format(time.time() - tic)) # apply the symmetry operator init_sym = get_symmetryop(self.cparams.get('init_symmetry',self.cparams.get('symmetry',None))) if init_sym is not None: tic = time.time() print("Applying symmetry operator..."); sys.stdout.flush() M = init_sym.apply(M) print("done in {0:.2f}s".format(time.time() - tic)) tic = time.time() print("Scaling initial model..."); sys.stdout.flush() modelscale = self.cparams.get('modelscale','auto') mleDC, _, mleDC_est_std = self.cryodata.get_dc_estimate() if modelscale == 'auto': # Err on the side of a weaker prior by using a larger value for modelscale modelscale = (np.abs(mleDC) + 2*mleDC_est_std)/self.cryodata.N print("estimated modelscale = {0:.3g}...".format(modelscale)); sys.stdout.flush() self.params['modelscale'] = modelscale self.cparams['modelscale'] = modelscale M *= modelscale/M.sum() print("done in {0:.2f}s".format(time.time() - tic)) if mleDC_est_std/np.abs(mleDC) > 0.05: print(" WARNING: the DC component estimate has a high relative variance, it may be inaccurate!") if ((modelscale*self.cryodata.N - np.abs(mleDC)) / mleDC_est_std) > 3: print(" WARNING: the selected modelscale value is more than 3 std devs different than the estimated one. Be sure this is correct.") # save initial model tic = time.time() print("Saving initial model..."); sys.stdout.flush() init_model_fname = os.path.join(self.expbase, 'init_model.mrc') writeMRC(init_model_fname, M, psz=self.cparams['pixel_size']) print("done in {0:.2f}s".format(time.time() - tic)) self.M = np.require(M,dtype=density.real_t) self.fM = density.real_to_fspace(M) self.dM = density.zeros_like(self.M) self.step = eval(self.cparams['optim_algo']) self.step.setup(self.cparams, self.diagout, self.statout, self.ostream) # Objective function setup -------------------------------------------- param_type = self.cparams.get('parameterization','real') cplx_param = param_type in ['complex','complex_coeff','complex_herm_coeff'] self.like_func = eval_objective(self.cparams['likelihood']) self.prior_func = eval_objective(self.cparams['prior']) if self.cparams.get('penalty',None) is not None: self.penalty_func = eval_objective(self.cparams['penalty']) prior_func = SumObjectives(self.prior_func.fspace, \ [self.penalty_func,self.prior_func], None) else: prior_func = self.prior_func self.obj = SumObjectives(cplx_param, [self.like_func,prior_func], [None,None]) self.obj.setup(self.cparams, self.diagout, self.statout, self.ostream) self.obj.set_dataset(self.cryodata) self.obj_wrapper = ObjectiveWrapper(param_type) self.last_save = time.time() self.logpost_history = FiniteRunningSum() self.like_history = FiniteRunningSum() # Importance Samplers ------------------------------------------------- self.is_sym = get_symmetryop(self.cparams.get('is_symmetry',self.cparams.get('symmetry',None))) self.sampler_R = FixedFisherImportanceSampler('_R',self.is_sym) self.sampler_I = FixedFisherImportanceSampler('_I') self.sampler_S = FixedGaussianImportanceSampler('_S') self.like_func.set_samplers(sampler_R=self.sampler_R,sampler_I=self.sampler_I,sampler_S=self.sampler_S)
from __future__ import print_function, division import os import sys sys.path.append(os.path.dirname(sys.path[0])) import argparse from matplotlib import pyplot as plt from numpy import unravel_index, log, maximum from cryoio import mrc from geometry import gen_dense_beamstop_mask parser = argparse.ArgumentParser() parser.add_argument("mrcs_files", help="list of mrcs files.", nargs='+') args = parser.parse_args() mrcs_files = args.mrcs_files if not isinstance(mrcs_files, list): mrcs_files = [mrcs_files] for ph in mrcs_files: M = mrc.readMRC(ph) N = M.shape[0] mask_3D = gen_dense_beamstop_mask(N, 3, 0.01, psize = 18) mrc.writeMRC(ph, M*mask_3D, psz=18)
print("""Wrong Number of Arguments. Usage: %run genphantomdata num inputmrc inputpar output %run genphantomdata 5000 finalphantom.mrc Data/thermus/thermus_Nature2012_128x128.par Data/phantom_5000 """) sys.exit() imgdata, ctfstack, pardata, mscope_params = genphantomdata(sys.argv[1], sys.argv[2], sys.argv[3]) outpath = sys.argv[4] if not os.path.isdir(outpath): os.makedirs(outpath) tic = time.time() print("Dumping data..."); sys.stdout.flush() def_path = os.path.join(outpath,'defocus.txt') ctfstack.write_defocus_txt(def_path) par_path = os.path.join(outpath,'ctf_gt.par') ctfstack.write_pardata(par_path) mrc_path = os.path.join(outpath,'imgdata.mrc') print(os.path.realpath(mrc_path)) mrc.writeMRC(mrc_path, np.transpose(imgdata,(1,2,0)), mscope_params['psize']) pard_path = os.path.join(outpath,'pardata.pkl') print(os.path.realpath(par_path)) with open(pard_path,'wb') as fi: pickle.dump(pardata, fi, protocol=2) print("Done in ", time.time()-tic, " seconds.") sys.exit()
# for xx in range(N): # for yy in range(N): # voxel_intensity = curr_img[xx, yy] # x = xx - cx # y = yy - cy # z = 0 # coord = np.asarray([x, y, z]).reshape(-1, 1) # rot_coord = np.dot(R_matrix, coord).reshape(1, -1)[0] # rot_coord = np.int_(np.round(rot_coord)) + cx # in_x = rot_coord[0] >= 0 and rot_coord[0] < N # in_y = rot_coord[1] >= 0 and rot_coord[1] < N # in_z = rot_coord[2] >= 0 and rot_coord[2] < N # if in_x and in_y and in_z: # num_valid_voxel += 1 # model_voxel_intensity = model[tuple(rot_coord)] # model_weight[tuple(rot_coord)] += 1 # voxel_weight = model_weight[tuple(rot_coord)] # delta_intensity = voxel_intensity - model_voxel_intensity # model_voxel_intensity += delta_intensity / voxel_weight # model[tuple(rot_coord)] = model_voxel_intensity model = cryoem.merge_slices(slices, Rs, N, rad, beamstop_rad=None) mrc.writeMRC('particle/merge.mrc', model, psz=2.8)
from cryoio import mrc import density, cryoops import geometry from quadrature import healpix from cryoem import cryoem import pyximport; pyximport.install( setup_args={"include_dirs": np.get_include()}, reload_support=True) import sincint M = mrc.readMRC('./particle/EMD-6044.mrc') # M = mrc.readMRC('./particle/1AON.mrc') # M = M / np.sum(M) M = M[:124, :124, :124] mrc.writeMRC('./particle/EMD-6044-cropped.mrc', M, psz=3.0) N = M.shape[0] print(M.shape) rad = 1 kernel = 'lanczos' ksize = 4 xy, trunc_xy, truncmask = geometry.gencoords(N, 2, rad, True) # premult = cryoops.compute_premultiplier(N, kernel='lanczos', kernsize=6) premult = cryoops.compute_premultiplier(N, kernel, ksize) TtoF = sincint.gentrunctofull(N=N, rad=rad) fM = density.real_to_fspace(M) prefM = density.real_to_fspace(premult.reshape( (1, 1, -1)) * premult.reshape((1, -1, 1)) * premult.reshape((-1, 1, 1)) * M)
if len(sys.argv) != 5: print """Wrong Number of Arguments. Usage: %run genphantomdata num inputmrc inputpar output %run genphantomdata 5000 finalphantom.mrc Data/thermus/thermus_Nature2012_128x128.par Data/phantom_5000 """ sys.exit() imgdata, ctfstack, pardata, mscope_params = genphantomdata(sys.argv[1], sys.argv[2], sys.argv[3]) outpath = sys.argv[4] if not os.path.isdir(outpath): os.makedirs(outpath) tic = time.time() print "Dumping data...", sys.stdout.flush() def_path = os.path.join(outpath, "defocus.txt") ctfstack.write_defocus_txt(def_path) par_path = os.path.join(outpath, "ctf_gt.par") ctfstack.write_pardata(par_path) mrc_path = os.path.join(outpath, "imgdata.mrc") mrc.writeMRC(mrc_path, n.transpose(imgdata, (1, 2, 0)), mscope_params["psize"]) pard_path = os.path.join(outpath, "pardata.pkl") with open(pard_path, "wb") as fi: cPickle.dump(pardata, fi, protocol=2) print "Done in ", time.time() - tic, " seconds." sys.exit()