def calc_fsc(model_path1, model_path2):
V1 = mrc.readMRC(model_path1)
V2 = mrc.readMRC(model_path2)
N = V1.shape[0]
alignedV1, R1 = cryoem.align_density(V1)
alignedV1 = cryoem.rotate_density(alignedV1, R1)
alignedV2, R1 = cryoem.align_density(V2)
alignedV2 = cryoem.rotate_density(alignedV2, R1)
VF1 = np.fft.fftshift(np.fft.fftn(alignedV1))
VF2 = np.fft.fftshift(np.fft.fftn(alignedV2))
maxrad = 1
rads, fsc, thresholds, resolutions = cryoem.compute_fsc(VF1, VF2, maxrad)
return rads, fsc, thresholds, resolutions
def __init__(self, parent=None, mrcfiles=[]):
QtGui.QWidget.__init__(self, parent)
Ms = [mrc.readMRC(mrcfile) for mrcfile in mrcfiles]
if len(mrcfiles) < 6:
hbox = True
layout = QtGui.QHBoxLayout(self)
else:
hbox = False
layout = QtGui.QGridLayout(self)
maxcol = int(len(mrcfiles) / 3) + 1
layout.setContentsMargins(0, 0, 0, 0)
layout.setSpacing(0)
for i, M in enumerate(Ms):
filename = os.path.basename(mrcfiles[i])
self.splitter_main_bottom = QtGui.QSplitter(self)
if hbox:
layout.addWidget(self.splitter_main_bottom)
else:
row, col = np.unravel_index(i, (3, maxcol))
layout.addWidget(self.splitter_main_bottom, row, col)
self.splitter_main_bottom.setOrientation(QtCore.Qt.Horizontal)
# self.sliceplot_widget = SlicePlotQWidget()
# self.splitter_main_bottom.addWidget(self.sliceplot_widget)
self.densityplot_widget = MayaviQWidget()
self.splitter_main_bottom.addWidget(self.densityplot_widget)
self.alignedM, self.R = cryoem.align_density(M, upsamp=1.0)
self.densityplot_widget.setup(self.alignedM, filename=filename)
def __init__(self, parent=None, mrcfiles=[]):
QtGui.QWidget.__init__(self, parent)
Ms = [mrc.readMRC(mrcfile) for mrcfile in mrcfiles]
layout = QtGui.QHBoxLayout(self)
layout.setContentsMargins(0,0,0,0)
layout.setSpacing(0)
for i, M in enumerate(Ms):
filename = os.path.basename(mrcfiles[i])
self.splitter_main_bottom = QtGui.QSplitter(self)
layout.addWidget(self.splitter_main_bottom)
self.splitter_main_bottom.setOrientation(QtCore.Qt.Horizontal)
# self.sliceplot_widget = SlicePlotQWidget()
# self.splitter_main_bottom.addWidget(self.sliceplot_widget)
self.densityplot_widget = MayaviQWidget()
self.splitter_main_bottom.addWidget(self.densityplot_widget)
self.alignedM,self.R = cryoem.align_density(M, upsamp=1.0)
self.densityplot_widget.setup(self.alignedM, filename=filename)
def __init__(self, expbase, cmdparams=None):
"""cryodata is a CryoData instance.
expbase is a path to the base of folder where this experiment's files
will be stored. The folder above expbase will also be searched
for .params files. These will be loaded first."""
BackgroundWorker.__init__(self)
# Create a background thread which handles IO
self.io_queue = Queue()
self.io_thread = Thread(target=self.ioworker)
self.io_thread.daemon = True
self.io_thread.start()
# General setup ----------------------------------------------------
self.expbase = expbase
self.outbase = None
# Paramter setup ---------------------------------------------------
# search above expbase for params files
_,_,filenames = os.walk(opj(expbase,'../')).next()
self.paramfiles = [opj(opj(expbase,'../'), fname) \
for fname in filenames if fname.endswith('.params')]
# search expbase for params files
_,_,filenames = os.walk(opj(expbase)).next()
self.paramfiles += [opj(expbase,fname) \
for fname in filenames if fname.endswith('.params')]
if 'local.params' in filenames:
self.paramfiles += [opj(expbase,'local.params')]
# load parameter files
self.params = Params(self.paramfiles)
self.cparams = None
if cmdparams is not None:
# Set parameter specified on the command line
for k,v in cmdparams.iteritems():
self.params[k] = v
# Dataset setup -------------------------------------------------------
self.imgpath = self.params['inpath']
psize = self.params['resolution']
if not isinstance(self.imgpath,list):
imgstk = MRCImageStack(self.imgpath,psize)
else:
imgstk = CombinedImageStack([MRCImageStack(cimgpath,psize) for cimgpath in self.imgpath])
if self.params.get('float_images',True):
imgstk.float_images()
self.ctfpath = self.params['ctfpath']
mscope_params = self.params['microscope_params']
if not isinstance(self.ctfpath,list):
ctfstk = CTFStack(self.ctfpath,mscope_params)
else:
ctfstk = CombinedCTFStack([CTFStack(cctfpath,mscope_params) for cctfpath in self.ctfpath])
self.cryodata = CryoDataset(imgstk,ctfstk)
self.cryodata.compute_noise_statistics()
if self.params.get('window_images',True):
imgstk.window_images()
minibatch_size = self.params['minisize']
testset_size = self.params['test_imgs']
partition = self.params.get('partition',0)
num_partitions = self.params.get('num_partitions',1)
seed = self.params['random_seed']
if isinstance(partition,str):
partition = eval(partition)
if isinstance(num_partitions,str):
num_partitions = eval(num_partitions)
if isinstance(seed,str):
seed = eval(seed)
self.cryodata.divide_dataset(minibatch_size,testset_size,partition,num_partitions,seed)
self.cryodata.set_datasign(self.params.get('datasign','auto'))
if self.params.get('normalize_data',True):
self.cryodata.normalize_dataset()
self.voxel_size = self.cryodata.pixel_size
# Iterations setup -------------------------------------------------
self.iteration = 0
self.tic_epoch = None
self.num_data_evals = 0
self.eval_params()
outdir = self.cparams.get('outdir',None)
if outdir is None:
if self.cparams.get('num_partitions',1) > 1:
outdir = 'partition{0}'.format(self.cparams['partition'])
else:
outdir = ''
self.outbase = opj(self.expbase,outdir)
if not os.path.isdir(self.outbase):
os.makedirs(self.outbase)
# Output setup -----------------------------------------------------
self.ostream = OutputStream(opj(self.outbase,'stdout'))
self.ostream(80*"=")
self.ostream("Experiment: " + expbase + \
" Kernel: " + self.params['kernel'])
self.ostream("Started on " + socket.gethostname() + \
" At: " + time.strftime('%B %d %Y: %I:%M:%S %p'))
self.ostream("Git SHA1: " + gitutil.git_get_SHA1())
self.ostream(80*"=")
gitutil.git_info_dump(opj(self.outbase, 'gitinfo'))
self.startdatetime = datetime.now()
# for diagnostics and parameters
self.diagout = Output(opj(self.outbase, 'diag'),runningout=False)
# for stats (per image etc)
self.statout = Output(opj(self.outbase, 'stat'),runningout=True)
# for likelihoods of individual images
self.likeout = Output(opj(self.outbase, 'like'),runningout=False)
self.img_likes = n.empty(self.cryodata.N_D)
self.img_likes[:] = n.inf
# optimization state vars ------------------------------------------
init_model = self.cparams.get('init_model',None)
if init_model is not None:
filename = init_model
if filename.upper().endswith('.MRC'):
M = readMRC(filename)
else:
with open(filename) as fp:
M = cPickle.load(fp)
if type(M)==list:
M = M[-1]['M']
if M.shape != 3*(self.cryodata.N,):
M = cryoem.resize_ndarray(M,3*(self.cryodata.N,),axes=(0,1,2))
else:
init_seed = self.cparams.get('init_random_seed',0) + self.cparams.get('partition',0)
print "Randomly generating initial density (init_random_seed = {0})...".format(init_seed), ; sys.stdout.flush()
tic = time.time()
M = cryoem.generate_phantom_density(self.cryodata.N, 0.95*self.cryodata.N/2.0, \
5*self.cryodata.N/128.0, 30, seed=init_seed)
print "done in {0}s".format(time.time() - tic)
tic = time.time()
print "Windowing and aligning initial density...", ; sys.stdout.flush()
# window the initial density
wfunc = self.cparams.get('init_window','circle')
cryoem.window(M,wfunc)
# Center and orient the initial density
cryoem.align_density(M)
print "done in {0:.2f}s".format(time.time() - tic)
# apply the symmetry operator
init_sym = get_symmetryop(self.cparams.get('init_symmetry',self.cparams.get('symmetry',None)))
if init_sym is not None:
tic = time.time()
print "Applying symmetry operator...", ; sys.stdout.flush()
M = init_sym.apply(M)
print "done in {0:.2f}s".format(time.time() - tic)
tic = time.time()
print "Scaling initial model...", ; sys.stdout.flush()
modelscale = self.cparams.get('modelscale','auto')
mleDC, _, mleDC_est_std = self.cryodata.get_dc_estimate()
if modelscale == 'auto':
# Err on the side of a weaker prior by using a larger value for modelscale
modelscale = (n.abs(mleDC) + 2*mleDC_est_std)/self.cryodata.N
print "estimated modelscale = {0:.3g}...".format(modelscale), ; sys.stdout.flush()
self.params['modelscale'] = modelscale
self.cparams['modelscale'] = modelscale
M *= modelscale/M.sum()
print "done in {0:.2f}s".format(time.time() - tic)
if mleDC_est_std/n.abs(mleDC) > 0.05:
print " WARNING: the DC component estimate has a high relative variance, it may be inaccurate!"
if ((modelscale*self.cryodata.N - n.abs(mleDC)) / mleDC_est_std) > 3:
print " WARNING: the selected modelscale value is more than 3 std devs different than the estimated one. Be sure this is correct."
self.M = n.require(M,dtype=density.real_t)
self.fM = density.real_to_fspace(M)
self.dM = density.zeros_like(self.M)
self.step = eval(self.cparams['optim_algo'])
self.step.setup(self.cparams, self.diagout, self.statout, self.ostream)
# Objective function setup --------------------------------------------
param_type = self.cparams.get('parameterization','real')
cplx_param = param_type in ['complex','complex_coeff','complex_herm_coeff']
self.like_func = eval_objective(self.cparams['likelihood'])
self.prior_func = eval_objective(self.cparams['prior'])
if self.cparams.get('penalty',None) is not None:
self.penalty_func = eval_objective(self.cparams['penalty'])
prior_func = SumObjectives(self.prior_func.fspace, \
[self.penalty_func,self.prior_func], None)
else:
prior_func = self.prior_func
self.obj = SumObjectives(cplx_param,
[self.like_func,prior_func], [None,None])
self.obj.setup(self.cparams, self.diagout, self.statout, self.ostream)
self.obj.set_dataset(self.cryodata)
self.obj_wrapper = ObjectiveWrapper(param_type)
self.last_save = time.time()
self.logpost_history = FiniteRunningSum()
self.like_history = FiniteRunningSum()
# Importance Samplers -------------------------------------------------
self.is_sym = get_symmetryop(self.cparams.get('is_symmetry',self.cparams.get('symmetry',None)))
self.sampler_R = FixedFisherImportanceSampler('_R',self.is_sym)
self.sampler_I = FixedFisherImportanceSampler('_I')
self.sampler_S = FixedGaussianImportanceSampler('_S')
self.like_func.set_samplers(sampler_R=self.sampler_R,sampler_I=self.sampler_I,sampler_S=self.sampler_S)
def updatevis(self, levels=[0.2,0.5,0.8]):
if self.M is None or self.diag is None or self.stat is None:
return
cdiag = self.diag
cparams = cdiag['params']
sym = get_symmetryop(cparams.get('symmetry',None))
quad_sym = sym if cparams.get('perfect_symmetry',True) else None
resolution = cparams['voxel_size']
name = cparams['name']
maxfreq = cparams['max_frequency']
N = self.M.shape[0]
rad_cutoff = cparams.get('rad_cutoff', 1.0)
rad = min(rad_cutoff,maxfreq*2.0*resolution)
# Show objective function
self.show_objective_plot(self.get_figure('stats'))
# Show information about noise and error
self.show_error_plot(self.get_figure('error'))
self.show_noise_plot(self.get_figure('noise'))
# Plot the envelope function if we have the info
if 'envelope_mle' in cdiag:
self.show_envelope_plot(self.get_figure('envelope'))
else:
self.close_figure('envelope')
if sym is None:
assert quad_sym is None
alignedM,R = cryoem.align_density(self.M)
if self.show_grad:
aligneddM = cryoem.rotate_density(self.dM,R)
else:
aligneddM = None
else:
alignedM, aligneddM = self.M, self.dM
R = np.identity(3)
self.alignedM,self.aligneddM,self.alignedR = alignedM,aligneddM,R
self.fM = density.real_to_fspace(self.M)
self.figMslices.set_data(alignedM)
glbl_phi_R = np.array([cdiag['global_phi_R']]).ravel()
if len(glbl_phi_R) == 1:
glbl_phi_R = None
glbl_phi_I = cdiag['global_phi_I']
if 'global_phi_S' in cdiag:
glbl_phi_S = cdiag['global_phi_S']
else:
glbl_phi_S = None
# Get direction quadrature
quad_R = quadrature.quad_schemes[('dir',cparams.get('quad_type_R','sk97'))]
quad_degree_R = cparams.get('quad_degree_R','auto')
if quad_degree_R == 'auto':
usFactor_R = cparams.get('quad_undersample_R',
cparams.get('quad_undersample',1.0))
quad_degree_R,_ = quad_R.compute_degree(N,rad,usFactor_R)
origlebDirs,_ = quad_R.get_quad_points(quad_degree_R,quad_sym)
lebDirs = np.dot(origlebDirs,R)
vmax_R = 5.0/len(glbl_phi_R)
# Get shift quadrature
if 'global_phi_S' in cdiag:
quad_S = quadrature.quad_schemes[('shift',cparams.get('quad_type_S','hermite'))]
quad_degree_S = cparams.get('quad_degree_S','auto')
if quad_degree_S == 'auto':
usFactor_S = cparams.get('quad_undersample_S',
cparams.get('quad_undersample',1.0))
quad_degree_S = quad_S.get_degree(N,rad,
cparams['quad_shiftsigma']/resolution,
cparams['quad_shiftextent']/resolution,
usFactor_S)
pts_S,_ = quad_S.get_quad_points(quad_degree_S,
cparams['quad_shiftsigma']/resolution,
cparams['quad_shiftextent']/resolution,
cparams.get('quad_shifttrunc','circ'))
vmax_S = 5.0/len(glbl_phi_S)
else:
pts_S = np.zeros_like([0])
vmax_S = None
# Density visualization
mlab.figure(self.fig1)
mlab.clf()
self.curr_contours = plot_density(alignedM, self.contours, levels)
# dispPhiR = glbl_phi_R
# dispDirs = lebDirs
# plot_directions(alignedM.shape[0]*dispDirs + alignedM.shape[0]/2.0,
# dispPhiR,
# 0, vmax_R)
mlab.view(focalpoint=[alignedM.shape[0]/2.0,alignedM.shape[0]/2.0,alignedM.shape[0]/2.0],distance=1.5*alignedM.shape[0])
if glbl_phi_R is not None:
plt.figure(self.get_figure('global_is_dists').number)
plt.clf()
plot_importance_dists(name,lebDirs,pts_S*resolution,glbl_phi_R,glbl_phi_I,glbl_phi_S,vmax_R,vmax_S)
if self.show_grad:
# Statistics of dM
self.figdMslices.set_data(aligneddM)
plt.figure(self.get_figure('step_stats').number)
plt.clf()
plt.suptitle(name + ' Step Statistics')
plt.subplot(1,2,1)
plt.hist(self.dM.reshape((-1,)),bins=0.5*self.dM.shape[0],log=True)
plt.title('Voxel Histogram')
(fs,raps) = rot_power_spectra(self.dM,resolution=resolution)
plt.subplot(1,2,2)
plt.plot(fs/(N/2.0)/(2.0*resolution),raps,label='RAPS')
plt.plot((rad/(2.0*resolution))*np.ones((2,)),
np.array([raps[raps > 0].min(),raps.max()]))
plt.yscale('log')
plt.title('RAPS Step')
if not self.extra_plots:
self.close_figure('density_stats')
return
# Statistics of M
self.show_density_plot(self.get_figure('density_stats'))
def updatevis(self, levels=[0.2,0.5,0.8]):
if self.M is None or self.diag is None or self.stat is None:
return
cdiag = self.diag
cparams = cdiag['params']
sym = get_symmetryop(cparams.get('symmetry',None))
quad_sym = sym if cparams.get('perfect_symmetry',True) else None
resolution = cparams['voxel_size']
name = cparams['name']
maxfreq = cparams['max_frequency']
N = self.M.shape[0]
rad_cutoff = cparams.get('rad_cutoff', 1.0)
rad = min(rad_cutoff,maxfreq*2.0*resolution)
# Show objective function
self.show_objective_plot(self.get_figure('stats'))
# Show information about noise and error
self.show_error_plot(self.get_figure('error'))
self.show_noise_plot(self.get_figure('noise'))
# Plot the envelope function if we have the info
if 'envelope_mle' in cdiag:
self.show_envelope_plot(self.get_figure('envelope'))
else:
self.close_figure('envelope')
if sym is None:
assert quad_sym is None
alignedM,R = c.align_density(self.M)
if self.show_grad:
aligneddM = c.rotate_density(self.dM,R)
else:
aligneddM = None
else:
alignedM, aligneddM = self.M, self.dM
R = n.identity(3)
self.alignedM,self.aligneddM,self.alignedR = alignedM,aligneddM,R
self.fM = density.real_to_fspace(self.M)
self.figMslices.set_data(alignedM)
glbl_phi_R = n.array([cdiag['global_phi_R']]).ravel()
if len(glbl_phi_R) == 1:
glbl_phi_R = None
glbl_phi_I = cdiag['global_phi_I']
glbl_phi_S = cdiag['global_phi_S']
# Get direction quadrature
quad_R = quadrature.quad_schemes[('dir',cparams.get('quad_type_R','sk97'))]
quad_degree_R = cparams.get('quad_degree_R','auto')
if quad_degree_R == 'auto':
usFactor_R = cparams.get('quad_undersample_R',
cparams.get('quad_undersample',1.0))
quad_degree_R,_ = quad_R.compute_degree(N,rad,usFactor_R)
origlebDirs,_ = quad_R.get_quad_points(quad_degree_R,quad_sym)
lebDirs = n.dot(origlebDirs,R)
# Get shift quadrature
quad_S = quadrature.quad_schemes[('shift',cparams.get('quad_type_S','hermite'))]
quad_degree_S = cparams.get('quad_degree_S','auto')
if quad_degree_S == 'auto':
usFactor_S = cparams.get('quad_undersample_S',
cparams.get('quad_undersample',1.0))
quad_degree_S = quad_S.get_degree(N,rad,
cparams['quad_shiftsigma']/resolution,
cparams['quad_shiftextent']/resolution,
usFactor_S)
pts_S,_ = quad_S.get_quad_points(quad_degree_S,
cparams['quad_shiftsigma']/resolution,
cparams['quad_shiftextent']/resolution,
cparams.get('quad_shifttrunc','circ'))
vmax_R = 5.0/len(glbl_phi_R)
vmax_S = 5.0/len(glbl_phi_S)
# Density visualization
mlab.figure(self.fig1)
mlab.clf()
self.curr_contours = plot_density(alignedM, self.contours, levels)
# dispPhiR = glbl_phi_R
# dispDirs = lebDirs
# plot_directions(alignedM.shape[0]*dispDirs + alignedM.shape[0]/2.0,
# dispPhiR,
# 0, vmax_R)
mlab.view(focalpoint=[alignedM.shape[0]/2.0,alignedM.shape[0]/2.0,alignedM.shape[0]/2.0],distance=1.5*alignedM.shape[0])
if glbl_phi_R is not None:
plt.figure(self.get_figure('global_is_dists').number)
plt.clf()
plot_importance_dists(name,lebDirs,pts_S*resolution,glbl_phi_R,glbl_phi_I,glbl_phi_S,vmax_R,vmax_S)
if self.show_grad:
# Statistics of dM
self.figdMslices.set_data(aligneddM)
plt.figure(self.get_figure('step_stats').number)
plt.clf()
plt.suptitle(name + ' Step Statistics')
plt.subplot(1,2,1)
plt.hist(self.dM.reshape((-1,)),bins=0.5*self.dM.shape[0],log=True)
plt.title('Voxel Histogram')
(fs,raps) = rot_power_spectra(self.dM,resolution=resolution)
plt.subplot(1,2,2)
plt.plot(fs/(N/2.0)/(2.0*resolution),raps,label='RAPS')
plt.plot((rad/(2.0*resolution))*n.ones((2,)),
n.array([raps[raps > 0].min(),raps.max()]))
plt.yscale('log')
plt.title('RAPS Step')
if not self.extra_plots:
self.close_figure('density_stats')
return
# Statistics of M
self.show_density_plot(self.get_figure('density_stats'))