def create_output_func(dataset, x, y, drop, input, output_layer, batch_size):
name = 'output'
#Compability with create_theano_func
data = Params({'set': {'output': dataset}})
output = (output_layer.output, y)
return create_theano_func(name, data, x, y, drop, input, output,
batch_size)
def setup(self):
"""
Builds vec_cn modules to test if they
"""
params = Params("source/tests/learned_cn/params.json")
A = torch.tensor([[0., 1., 0.], [1., 0., 1.], [0., 1., 0.]])
self.vec_cn = LCIModule(params, A.numpy())
def __init__(self, expbase, cmdparams=None):
"""cryodata is a CryoData instance.
expbase is a path to the base of folder where this experiment's files
will be stored. The folder above expbase will also be searched
for .params files. These will be loaded first."""
BackgroundWorker.__init__(self)
# Create a background thread which handles IO
self.io_queue = Queue()
self.io_thread = Thread(target=self.ioworker)
self.io_thread.daemon = True
self.io_thread.start()
# General setup ----------------------------------------------------
self.expbase = expbase
self.outbase = None
# Paramter setup ---------------------------------------------------
# search above expbase for params files
_,_,filenames = os.walk(opj(expbase,'../')).next()
self.paramfiles = [opj(opj(expbase,'../'), fname) \
for fname in filenames if fname.endswith('.params')]
# search expbase for params files
_,_,filenames = os.walk(opj(expbase)).next()
self.paramfiles += [opj(expbase,fname) \
for fname in filenames if fname.endswith('.params')]
if 'local.params' in filenames:
self.paramfiles += [opj(expbase,'local.params')]
# load parameter files
self.params = Params(self.paramfiles)
self.cparams = None
if cmdparams is not None:
# Set parameter specified on the command line
for k,v in cmdparams.iteritems():
self.params[k] = v
# Dataset setup -------------------------------------------------------
self.imgpath = self.params['inpath']
psize = self.params['resolution']
if not isinstance(self.imgpath,list):
imgstk = MRCImageStack(self.imgpath,psize)
else:
imgstk = CombinedImageStack([MRCImageStack(cimgpath,psize) for cimgpath in self.imgpath])
if self.params.get('float_images',True):
imgstk.float_images()
self.ctfpath = self.params['ctfpath']
mscope_params = self.params['microscope_params']
if not isinstance(self.ctfpath,list):
ctfstk = CTFStack(self.ctfpath,mscope_params)
else:
ctfstk = CombinedCTFStack([CTFStack(cctfpath,mscope_params) for cctfpath in self.ctfpath])
self.cryodata = CryoDataset(imgstk,ctfstk)
self.cryodata.compute_noise_statistics()
if self.params.get('window_images',True):
imgstk.window_images()
minibatch_size = self.params['minisize']
testset_size = self.params['test_imgs']
partition = self.params.get('partition',0)
num_partitions = self.params.get('num_partitions',1)
seed = self.params['random_seed']
if isinstance(partition,str):
partition = eval(partition)
if isinstance(num_partitions,str):
num_partitions = eval(num_partitions)
if isinstance(seed,str):
seed = eval(seed)
self.cryodata.divide_dataset(minibatch_size,testset_size,partition,num_partitions,seed)
self.cryodata.set_datasign(self.params.get('datasign','auto'))
if self.params.get('normalize_data',True):
self.cryodata.normalize_dataset()
self.voxel_size = self.cryodata.pixel_size
# Iterations setup -------------------------------------------------
self.iteration = 0
self.tic_epoch = None
self.num_data_evals = 0
self.eval_params()
outdir = self.cparams.get('outdir',None)
if outdir is None:
if self.cparams.get('num_partitions',1) > 1:
outdir = 'partition{0}'.format(self.cparams['partition'])
else:
outdir = ''
self.outbase = opj(self.expbase,outdir)
if not os.path.isdir(self.outbase):
os.makedirs(self.outbase)
# Output setup -----------------------------------------------------
self.ostream = OutputStream(opj(self.outbase,'stdout'))
self.ostream(80*"=")
self.ostream("Experiment: " + expbase + \
" Kernel: " + self.params['kernel'])
self.ostream("Started on " + socket.gethostname() + \
" At: " + time.strftime('%B %d %Y: %I:%M:%S %p'))
self.ostream("Git SHA1: " + gitutil.git_get_SHA1())
self.ostream(80*"=")
gitutil.git_info_dump(opj(self.outbase, 'gitinfo'))
self.startdatetime = datetime.now()
# for diagnostics and parameters
self.diagout = Output(opj(self.outbase, 'diag'),runningout=False)
# for stats (per image etc)
self.statout = Output(opj(self.outbase, 'stat'),runningout=True)
# for likelihoods of individual images
self.likeout = Output(opj(self.outbase, 'like'),runningout=False)
self.img_likes = n.empty(self.cryodata.N_D)
self.img_likes[:] = n.inf
# optimization state vars ------------------------------------------
init_model = self.cparams.get('init_model',None)
if init_model is not None:
filename = init_model
if filename.upper().endswith('.MRC'):
M = readMRC(filename)
else:
with open(filename) as fp:
M = cPickle.load(fp)
if type(M)==list:
M = M[-1]['M']
if M.shape != 3*(self.cryodata.N,):
M = cryoem.resize_ndarray(M,3*(self.cryodata.N,),axes=(0,1,2))
else:
init_seed = self.cparams.get('init_random_seed',0) + self.cparams.get('partition',0)
print "Randomly generating initial density (init_random_seed = {0})...".format(init_seed), ; sys.stdout.flush()
tic = time.time()
M = cryoem.generate_phantom_density(self.cryodata.N, 0.95*self.cryodata.N/2.0, \
5*self.cryodata.N/128.0, 30, seed=init_seed)
print "done in {0}s".format(time.time() - tic)
tic = time.time()
print "Windowing and aligning initial density...", ; sys.stdout.flush()
# window the initial density
wfunc = self.cparams.get('init_window','circle')
cryoem.window(M,wfunc)
# Center and orient the initial density
cryoem.align_density(M)
print "done in {0:.2f}s".format(time.time() - tic)
# apply the symmetry operator
init_sym = get_symmetryop(self.cparams.get('init_symmetry',self.cparams.get('symmetry',None)))
if init_sym is not None:
tic = time.time()
print "Applying symmetry operator...", ; sys.stdout.flush()
M = init_sym.apply(M)
print "done in {0:.2f}s".format(time.time() - tic)
tic = time.time()
print "Scaling initial model...", ; sys.stdout.flush()
modelscale = self.cparams.get('modelscale','auto')
mleDC, _, mleDC_est_std = self.cryodata.get_dc_estimate()
if modelscale == 'auto':
# Err on the side of a weaker prior by using a larger value for modelscale
modelscale = (n.abs(mleDC) + 2*mleDC_est_std)/self.cryodata.N
print "estimated modelscale = {0:.3g}...".format(modelscale), ; sys.stdout.flush()
self.params['modelscale'] = modelscale
self.cparams['modelscale'] = modelscale
M *= modelscale/M.sum()
print "done in {0:.2f}s".format(time.time() - tic)
if mleDC_est_std/n.abs(mleDC) > 0.05:
print " WARNING: the DC component estimate has a high relative variance, it may be inaccurate!"
if ((modelscale*self.cryodata.N - n.abs(mleDC)) / mleDC_est_std) > 3:
print " WARNING: the selected modelscale value is more than 3 std devs different than the estimated one. Be sure this is correct."
self.M = n.require(M,dtype=density.real_t)
self.fM = density.real_to_fspace(M)
self.dM = density.zeros_like(self.M)
self.step = eval(self.cparams['optim_algo'])
self.step.setup(self.cparams, self.diagout, self.statout, self.ostream)
# Objective function setup --------------------------------------------
param_type = self.cparams.get('parameterization','real')
cplx_param = param_type in ['complex','complex_coeff','complex_herm_coeff']
self.like_func = eval_objective(self.cparams['likelihood'])
self.prior_func = eval_objective(self.cparams['prior'])
if self.cparams.get('penalty',None) is not None:
self.penalty_func = eval_objective(self.cparams['penalty'])
prior_func = SumObjectives(self.prior_func.fspace, \
[self.penalty_func,self.prior_func], None)
else:
prior_func = self.prior_func
self.obj = SumObjectives(cplx_param,
[self.like_func,prior_func], [None,None])
self.obj.setup(self.cparams, self.diagout, self.statout, self.ostream)
self.obj.set_dataset(self.cryodata)
self.obj_wrapper = ObjectiveWrapper(param_type)
self.last_save = time.time()
self.logpost_history = FiniteRunningSum()
self.like_history = FiniteRunningSum()
# Importance Samplers -------------------------------------------------
self.is_sym = get_symmetryop(self.cparams.get('is_symmetry',self.cparams.get('symmetry',None)))
self.sampler_R = FixedFisherImportanceSampler('_R',self.is_sym)
self.sampler_I = FixedFisherImportanceSampler('_I')
self.sampler_S = FixedGaussianImportanceSampler('_S')
self.like_func.set_samplers(sampler_R=self.sampler_R,sampler_I=self.sampler_I,sampler_S=self.sampler_S)
class CryoOptimizer(BackgroundWorker):
def outputbatchinfo(self,batch,res,logP,prefix,name):
diag = {}
stat = {}
like = {}
N_M = batch['N_M']
cepoch = self.cryodata.get_epoch(frac=True)
epoch = self.cryodata.get_epoch()
num_data = self.cryodata.N_D_Train
sigma = n.sqrt(n.mean(res['Evar_like']))
sigma_prior = n.sqrt(n.mean(res['Evar_prior']))
self.ostream(' {0} Batch:'.format(name))
for suff in ['R','I','S']:
diag[prefix+'_CV2_'+suff] = res['CV2_'+suff]
diag[prefix+'_idxs'] = batch['img_idxs']
diag[prefix+'_sigma2_est'] = res['sigma2_est']
diag[prefix+'_correlation'] = res['correlation']
diag[prefix+'_power'] = res['power']
# self.ostream(" RMS Error: %g" % (sigma/n.sqrt(self.cryodata.noise_var)))
self.ostream(" RMS Error: %g, Signal: %g" % (sigma/n.sqrt(self.cryodata.noise_var), \
sigma_prior/n.sqrt(self.cryodata.noise_var)))
self.ostream(" Effective # of R / I / S: %.2f / %.2f / %.2f " %\
(n.mean(res['CV2_R']), n.mean(res['CV2_I']),n.mean(res['CV2_S'])))
# Importance Sampling Statistics
is_speedups = []
for suff in ['R','I','S','Total']:
if self.cparams.get('is_on_'+suff,False) or (suff == 'Total' and len(is_speedups) > 0):
spdup = N_M/res['N_' + suff + '_sampled_total']
is_speedups.append((suff,spdup,n.mean(res['N_'+suff+'_sampled']),res['N_'+suff]))
stat[prefix+'_is_speedup_'+suff] = [spdup]
else:
stat[prefix+'_is_speedup_'+suff] = [1.0]
if len(is_speedups) > 0:
lblstr = is_speedups[0][0]
numstr = '%.2f (%d of %d)' % is_speedups[0][1:]
for i in range(1,len(is_speedups)):
lblstr += ' / ' + is_speedups[i][0]
numstr += ' / %.2f (%d of %d)' % is_speedups[i][1:]
self.ostream(" IS Speedup {0}: {1}".format(lblstr,numstr))
stat[prefix+'_sigma'] = [sigma]
stat[prefix+'_logp'] = [logP]
stat[prefix+'_like'] = [res['L']]
stat[prefix+'_num_data'] = [num_data]
stat[prefix+'_num_data_evals'] = [self.num_data_evals]
stat[prefix+'_iteration'] = [self.iteration]
stat[prefix+'_epoch'] = [epoch]
stat[prefix+'_cepoch'] = [cepoch],
stat[prefix+'_time'] = [time.time()]
for k,v in res['like_timing'].iteritems():
stat[prefix+'_like_timing_'+k] = [v]
Idxs = batch['img_idxs']
self.img_likes[Idxs] = res['like']
like['img_likes'] = self.img_likes
like['train_idxs'] = self.cryodata.train_idxs
like['test_idxs'] = self.cryodata.test_idxs
keepidxs = self.cryodata.train_idxs if prefix == 'train' else self.cryodata.test_idxs
keeplikes = self.img_likes[keepidxs]
keeplikes = keeplikes[n.isfinite(keeplikes)]
quants = n.percentile(keeplikes, range(0,101))
stat[prefix+'_full_like_quantiles'] = [quants]
quants = n.percentile(res['like'], range(0,101))
stat[prefix+'_mini_like_quantiles'] = [quants]
stat[prefix+'_num_like_quantiles'] = [len(keeplikes)]
self.diagout.output(**diag)
self.statout.output(**stat)
self.likeout.output(**like)
def ioworker(self):
while True:
iotype,fname,data = self.io_queue.get()
try:
if iotype == 'mrc':
writeMRC(fname,*data)
elif iotype == 'pkl':
with open(fname, 'wb') as f:
cPickle.dump(data, f, protocol=-1)
elif iotype == 'cp':
copyfile(fname,data)
except:
print "ERROR DUMPING {0}: {1}".format(fname, sys.exc_info()[0])
self.io_queue.task_done()
def __init__(self, expbase, cmdparams=None):
"""cryodata is a CryoData instance.
expbase is a path to the base of folder where this experiment's files
will be stored. The folder above expbase will also be searched
for .params files. These will be loaded first."""
BackgroundWorker.__init__(self)
# Create a background thread which handles IO
self.io_queue = Queue()
self.io_thread = Thread(target=self.ioworker)
self.io_thread.daemon = True
self.io_thread.start()
# General setup ----------------------------------------------------
self.expbase = expbase
self.outbase = None
# Paramter setup ---------------------------------------------------
# search above expbase for params files
_,_,filenames = os.walk(opj(expbase,'../')).next()
self.paramfiles = [opj(opj(expbase,'../'), fname) \
for fname in filenames if fname.endswith('.params')]
# search expbase for params files
_,_,filenames = os.walk(opj(expbase)).next()
self.paramfiles += [opj(expbase,fname) \
for fname in filenames if fname.endswith('.params')]
if 'local.params' in filenames:
self.paramfiles += [opj(expbase,'local.params')]
# load parameter files
self.params = Params(self.paramfiles)
self.cparams = None
if cmdparams is not None:
# Set parameter specified on the command line
for k,v in cmdparams.iteritems():
self.params[k] = v
# Dataset setup -------------------------------------------------------
self.imgpath = self.params['inpath']
psize = self.params['resolution']
if not isinstance(self.imgpath,list):
imgstk = MRCImageStack(self.imgpath,psize)
else:
imgstk = CombinedImageStack([MRCImageStack(cimgpath,psize) for cimgpath in self.imgpath])
if self.params.get('float_images',True):
imgstk.float_images()
self.ctfpath = self.params['ctfpath']
mscope_params = self.params['microscope_params']
if not isinstance(self.ctfpath,list):
ctfstk = CTFStack(self.ctfpath,mscope_params)
else:
ctfstk = CombinedCTFStack([CTFStack(cctfpath,mscope_params) for cctfpath in self.ctfpath])
self.cryodata = CryoDataset(imgstk,ctfstk)
self.cryodata.compute_noise_statistics()
if self.params.get('window_images',True):
imgstk.window_images()
minibatch_size = self.params['minisize']
testset_size = self.params['test_imgs']
partition = self.params.get('partition',0)
num_partitions = self.params.get('num_partitions',1)
seed = self.params['random_seed']
if isinstance(partition,str):
partition = eval(partition)
if isinstance(num_partitions,str):
num_partitions = eval(num_partitions)
if isinstance(seed,str):
seed = eval(seed)
self.cryodata.divide_dataset(minibatch_size,testset_size,partition,num_partitions,seed)
self.cryodata.set_datasign(self.params.get('datasign','auto'))
if self.params.get('normalize_data',True):
self.cryodata.normalize_dataset()
self.voxel_size = self.cryodata.pixel_size
# Iterations setup -------------------------------------------------
self.iteration = 0
self.tic_epoch = None
self.num_data_evals = 0
self.eval_params()
outdir = self.cparams.get('outdir',None)
if outdir is None:
if self.cparams.get('num_partitions',1) > 1:
outdir = 'partition{0}'.format(self.cparams['partition'])
else:
outdir = ''
self.outbase = opj(self.expbase,outdir)
if not os.path.isdir(self.outbase):
os.makedirs(self.outbase)
# Output setup -----------------------------------------------------
self.ostream = OutputStream(opj(self.outbase,'stdout'))
self.ostream(80*"=")
self.ostream("Experiment: " + expbase + \
" Kernel: " + self.params['kernel'])
self.ostream("Started on " + socket.gethostname() + \
" At: " + time.strftime('%B %d %Y: %I:%M:%S %p'))
self.ostream("Git SHA1: " + gitutil.git_get_SHA1())
self.ostream(80*"=")
gitutil.git_info_dump(opj(self.outbase, 'gitinfo'))
self.startdatetime = datetime.now()
# for diagnostics and parameters
self.diagout = Output(opj(self.outbase, 'diag'),runningout=False)
# for stats (per image etc)
self.statout = Output(opj(self.outbase, 'stat'),runningout=True)
# for likelihoods of individual images
self.likeout = Output(opj(self.outbase, 'like'),runningout=False)
self.img_likes = n.empty(self.cryodata.N_D)
self.img_likes[:] = n.inf
# optimization state vars ------------------------------------------
init_model = self.cparams.get('init_model',None)
if init_model is not None:
filename = init_model
if filename.upper().endswith('.MRC'):
M = readMRC(filename)
else:
with open(filename) as fp:
M = cPickle.load(fp)
if type(M)==list:
M = M[-1]['M']
if M.shape != 3*(self.cryodata.N,):
M = cryoem.resize_ndarray(M,3*(self.cryodata.N,),axes=(0,1,2))
else:
init_seed = self.cparams.get('init_random_seed',0) + self.cparams.get('partition',0)
print "Randomly generating initial density (init_random_seed = {0})...".format(init_seed), ; sys.stdout.flush()
tic = time.time()
M = cryoem.generate_phantom_density(self.cryodata.N, 0.95*self.cryodata.N/2.0, \
5*self.cryodata.N/128.0, 30, seed=init_seed)
print "done in {0}s".format(time.time() - tic)
tic = time.time()
print "Windowing and aligning initial density...", ; sys.stdout.flush()
# window the initial density
wfunc = self.cparams.get('init_window','circle')
cryoem.window(M,wfunc)
# Center and orient the initial density
cryoem.align_density(M)
print "done in {0:.2f}s".format(time.time() - tic)
# apply the symmetry operator
init_sym = get_symmetryop(self.cparams.get('init_symmetry',self.cparams.get('symmetry',None)))
if init_sym is not None:
tic = time.time()
print "Applying symmetry operator...", ; sys.stdout.flush()
M = init_sym.apply(M)
print "done in {0:.2f}s".format(time.time() - tic)
tic = time.time()
print "Scaling initial model...", ; sys.stdout.flush()
modelscale = self.cparams.get('modelscale','auto')
mleDC, _, mleDC_est_std = self.cryodata.get_dc_estimate()
if modelscale == 'auto':
# Err on the side of a weaker prior by using a larger value for modelscale
modelscale = (n.abs(mleDC) + 2*mleDC_est_std)/self.cryodata.N
print "estimated modelscale = {0:.3g}...".format(modelscale), ; sys.stdout.flush()
self.params['modelscale'] = modelscale
self.cparams['modelscale'] = modelscale
M *= modelscale/M.sum()
print "done in {0:.2f}s".format(time.time() - tic)
if mleDC_est_std/n.abs(mleDC) > 0.05:
print " WARNING: the DC component estimate has a high relative variance, it may be inaccurate!"
if ((modelscale*self.cryodata.N - n.abs(mleDC)) / mleDC_est_std) > 3:
print " WARNING: the selected modelscale value is more than 3 std devs different than the estimated one. Be sure this is correct."
self.M = n.require(M,dtype=density.real_t)
self.fM = density.real_to_fspace(M)
self.dM = density.zeros_like(self.M)
self.step = eval(self.cparams['optim_algo'])
self.step.setup(self.cparams, self.diagout, self.statout, self.ostream)
# Objective function setup --------------------------------------------
param_type = self.cparams.get('parameterization','real')
cplx_param = param_type in ['complex','complex_coeff','complex_herm_coeff']
self.like_func = eval_objective(self.cparams['likelihood'])
self.prior_func = eval_objective(self.cparams['prior'])
if self.cparams.get('penalty',None) is not None:
self.penalty_func = eval_objective(self.cparams['penalty'])
prior_func = SumObjectives(self.prior_func.fspace, \
[self.penalty_func,self.prior_func], None)
else:
prior_func = self.prior_func
self.obj = SumObjectives(cplx_param,
[self.like_func,prior_func], [None,None])
self.obj.setup(self.cparams, self.diagout, self.statout, self.ostream)
self.obj.set_dataset(self.cryodata)
self.obj_wrapper = ObjectiveWrapper(param_type)
self.last_save = time.time()
self.logpost_history = FiniteRunningSum()
self.like_history = FiniteRunningSum()
# Importance Samplers -------------------------------------------------
self.is_sym = get_symmetryop(self.cparams.get('is_symmetry',self.cparams.get('symmetry',None)))
self.sampler_R = FixedFisherImportanceSampler('_R',self.is_sym)
self.sampler_I = FixedFisherImportanceSampler('_I')
self.sampler_S = FixedGaussianImportanceSampler('_S')
self.like_func.set_samplers(sampler_R=self.sampler_R,sampler_I=self.sampler_I,sampler_S=self.sampler_S)
def eval_params(self):
# cvars are state variables that can be used in parameter expressions
cvars = {}
cvars['cepoch'] = self.cryodata.get_epoch(frac=True)
cvars['epoch'] = self.cryodata.get_epoch()
cvars['iteration'] = self.iteration
cvars['num_data'] = self.cryodata.N_D_Train
cvars['num_batches'] = self.cryodata.N_batches
cvars['noise_std'] = n.sqrt(self.cryodata.noise_var)
cvars['data_std'] = n.sqrt(self.cryodata.data_var)
cvars['voxel_size'] = self.voxel_size
cvars['pixel_size'] = self.cryodata.pixel_size
cvars['prev_max_frequency'] = self.cparams['max_frequency'] if self.cparams is not None else None
# prelist fields are parameters that can be used in evaluating other parameter
# expressions, they can only depend on values defined in cvars
prelist = ['max_frequency']
skipfields = set(['inpath','ctfpath'])
cvars = self.params.partial_evaluate(prelist,**cvars)
if self.cparams is None:
self.max_frequency_changes = 0
else:
self.max_frequency_changes += cvars['max_frequency'] != cvars['prev_max_frequency']
cvars['num_max_frequency_changes'] = self.max_frequency_changes
cvars['max_frequency_changed'] = cvars['max_frequency'] != cvars['prev_max_frequency']
self.cparams = self.params.evaluate(skipfields,**cvars)
self.cparams['exp_path'] = self.expbase
self.cparams['out_path'] = self.outbase
if 'name' not in self.cparams:
self.cparams['name'] = '{0} - {1} - {2} ({3})'.format(self.cparams['dataset_name'], self.cparams['prior_name'], self.cparams['optimizer_name'], self.cparams['kernel'])
def run(self):
while self.dowork(): pass
print "Waiting for IO queue to clear...", ; sys.stdout.flush()
self.io_queue.join()
print "done." ; sys.stdout.flush()
def begin(self):
BackgroundWorker.begin(self)
def end(self):
BackgroundWorker.end(self)
def dowork(self):
"""Do one atom of work. I.E. Execute one minibatch"""
timing = {}
# Time each minibatch
tic_mini = time.time()
self.iteration += 1
# Fetch the current batches
trainbatch = self.cryodata.get_next_minibatch(self.cparams.get('shuffle_minibatches',True))
# Get the current epoch
cepoch = self.cryodata.get_epoch(frac=True)
epoch = self.cryodata.get_epoch()
num_data = self.cryodata.N_D_Train
# Evaluate the parameters
self.eval_params()
timing['setup'] = time.time() - tic_mini
# Do hyperparameter learning
if self.cparams.get('learn_params',False):
tic_learn = time.time()
if self.cparams.get('learn_prior_params',True):
tic_learn_prior = time.time()
self.prior_func.learn_params(self.params, self.cparams, M=self.M, fM=self.fM)
timing['learn_prior'] = time.time() - tic_learn_prior
if self.cparams.get('learn_likelihood_params',True):
tic_learn_like = time.time()
self.like_func.learn_params(self.params, self.cparams, M=self.M, fM=self.fM)
timing['learn_like'] = time.time() - tic_learn_like
if self.cparams.get('learn_prior_params',True) or self.cparams.get('learn_likelihood_params',True):
timing['learn_total'] = time.time() - tic_learn
# Time each epoch
if self.tic_epoch == None:
self.ostream("Epoch: %d" % epoch)
self.tic_epoch = (tic_mini,epoch)
elif self.tic_epoch[1] != epoch:
self.ostream("Epoch Total - %.6f seconds " % \
(tic_mini - self.tic_epoch[0]))
self.tic_epoch = (tic_mini,epoch)
sym = get_symmetryop(self.cparams.get('symmetry',None))
if sym is not None:
self.obj.ws[1] = 1.0/sym.get_order()
tic_mstats = time.time()
self.ostream(self.cparams['name']," Iteration:", self.iteration,\
" Epoch:", epoch, " Host:", socket.gethostname())
# Compute density statistics
N = self.cryodata.N
M_sum = self.M.sum(dtype=n.float64)
M_zeros = (self.M == 0).sum()
M_mean = M_sum/N**3
M_max = self.M.max()
M_min = self.M.min()
# self.ostream(" Density (min/max/avg/sum/zeros): " +
# "%.2e / %.2e / %.2e / %.2e / %g " %
# (M_min, M_max, M_mean, M_sum, M_zeros))
self.statout.output(total_density=[M_sum],
avg_density=[M_mean],
nonzero_density=[M_zeros],
max_density=[M_max],
min_density=[M_min])
timing['density_stats'] = time.time() - tic_mstats
# evaluate test batch if requested
if self.iteration <= 1 or self.cparams.get('evaluate_test_set',self.iteration%5):
tic_test = time.time()
testbatch = self.cryodata.get_testbatch()
self.obj.set_data(self.cparams,testbatch)
testLogP, res_test = self.obj.eval(M=self.M, fM=self.fM,
compute_gradient=False)
self.outputbatchinfo(testbatch, res_test, testLogP, 'test', 'Test')
timing['test_batch'] = time.time() - tic_test
else:
testLogP, res_test = None, None
# setup the wrapper for the objective function
tic_objsetup = time.time()
self.obj.set_data(self.cparams,trainbatch)
self.obj_wrapper.set_objective(self.obj)
x0 = self.obj_wrapper.set_density(self.M,self.fM)
evalobj = self.obj_wrapper.eval_obj
timing['obj_setup'] = time.time() - tic_objsetup
# Get step size
self.num_data_evals += trainbatch['N_M'] # at least one gradient
tic_objstep = time.time()
trainLogP, dlogP, v, res_train, extra_num_data = self.step.do_step(x0,
self.cparams,
self.cryodata,
evalobj,
batch=trainbatch)
# Apply the step
x = x0 + v
timing['step'] = time.time() - tic_objstep
# Convert from parameters to value
tic_stepfinalize = time.time()
prevM = n.copy(self.M)
self.M, self.fM = self.obj_wrapper.convert_parameter(x,comp_real=True)
apply_sym = sym is not None and self.cparams.get('perfect_symmetry',True) and self.cparams.get('apply_symmetry',True)
if apply_sym:
self.M = sym.apply(self.M)
# Truncate the density to bounds if they exist
if self.cparams['density_lb'] is not None:
n.maximum(self.M,self.cparams['density_lb']*self.cparams['modelscale'],out=self.M)
if self.cparams['density_ub'] is not None:
n.minimum(self.M,self.cparams['density_ub']*self.cparams['modelscale'],out=self.M)
# Compute net change
self.dM = prevM - self.M
# Convert to fourier space (may not be required)
if self.fM is None or apply_sym \
or self.cparams['density_lb'] != None \
or self.cparams['density_ub'] != None:
self.fM = density.real_to_fspace(self.M)
timing['step_finalize'] = time.time() - tic_stepfinalize
# Compute step statistics
tic_stepstats = time.time()
step_size = n.linalg.norm(self.dM)
grad_size = n.linalg.norm(dlogP)
M_norm = n.linalg.norm(self.M)
self.num_data_evals += extra_num_data
inc_ratio = step_size / M_norm
self.statout.output(step_size=[step_size],
inc_ratio=[inc_ratio],
grad_size=[grad_size],
norm_density=[M_norm])
timing['step_stats'] = time.time() - tic_stepstats
# Update import sampling distributions
tic_isupdate = time.time()
self.sampler_R.perform_update()
self.sampler_I.perform_update()
self.sampler_S.perform_update()
self.diagout.output(global_phi_R=self.sampler_R.get_global_dist())
self.diagout.output(global_phi_I=self.sampler_I.get_global_dist())
self.diagout.output(global_phi_S=self.sampler_S.get_global_dist())
timing['is_update'] = time.time() - tic_isupdate
# Output basic diagnostics
tic_diagnostics = time.time()
self.diagout.output(iteration=self.iteration, epoch=epoch, cepoch=cepoch)
if self.logpost_history.N_sum != self.cryodata.N_batches:
self.logpost_history.setup(trainLogP,self.cryodata.N_batches)
self.logpost_history.set_value(trainbatch['id'],trainLogP)
if self.like_history.N_sum != self.cryodata.N_batches:
self.like_history.setup(res_train['L'],self.cryodata.N_batches)
self.like_history.set_value(trainbatch['id'],res_train['L'])
self.outputbatchinfo(trainbatch, res_train, trainLogP, 'train', 'Train')
# Dump parameters here to catch the defaults used in evaluation
self.diagout.output(params=self.cparams,
envelope_mle=self.like_func.get_envelope_mle(),
sigma2_mle=self.like_func.get_sigma2_mle(),
hostname=socket.gethostname())
self.statout.output(num_data=[num_data],
num_data_evals=[self.num_data_evals],
iteration=[self.iteration],
epoch=[epoch],
cepoch=[cepoch],
logp=[self.logpost_history.get_mean()],
like=[self.like_history.get_mean()],
sigma=[self.like_func.get_rmse()],
time=[time.time()])
timing['diagnostics'] = time.time() - tic_diagnostics
checkpoint_it = self.iteration % self.cparams.get('checkpoint_frequency',50) == 0
save_it = checkpoint_it or self.cparams['save_iteration'] or \
time.time() - self.last_save > self.cparams.get('save_time',n.inf)
if save_it:
tic_save = time.time()
self.last_save = tic_save
if self.io_queue.qsize():
print "Warning: IO queue has become backlogged with {0} remaining, waiting for it to clear".format(self.io_queue.qsize())
self.io_queue.join()
self.io_queue.put(( 'pkl', self.statout.fname, copy(self.statout.outdict) ))
self.io_queue.put(( 'pkl', self.diagout.fname, deepcopy(self.diagout.outdict) ))
self.io_queue.put(( 'pkl', self.likeout.fname, deepcopy(self.likeout.outdict) ))
self.io_queue.put(( 'mrc', opj(self.outbase,'model.mrc'), \
(n.require(self.M,dtype=density.real_t),self.voxel_size) ))
self.io_queue.put(( 'mrc', opj(self.outbase,'dmodel.mrc'), \
(n.require(self.dM,dtype=density.real_t),self.voxel_size) ))
if checkpoint_it:
self.io_queue.put(( 'cp', self.diagout.fname, self.diagout.fname+'-{0:06}'.format(self.iteration) ))
self.io_queue.put(( 'cp', self.likeout.fname, self.likeout.fname+'-{0:06}'.format(self.iteration) ))
self.io_queue.put(( 'cp', opj(self.outbase,'model.mrc'), opj(self.outbase,'model-{0:06}.mrc'.format(self.iteration)) ))
timing['save'] = time.time() - tic_save
time_total = time.time() - tic_mini
self.ostream(" Minibatch Total - %.2f seconds Total Runtime - %s" %
(time_total, format_timedelta(datetime.now() - self.startdatetime) ))
return self.iteration < self.cparams.get('max_iterations',n.inf) and \
cepoch < self.cparams.get('max_epochs',n.inf)
from scipy.stats import rankdata
from util import Params, set_logger, prepare_sns
parser = argparse.ArgumentParser()
parser.add_argument('--experiment_dir',
default='experiments/base_model',
help="Directory containing params.json")
if __name__ == '__main__':
# Load the parameters from the experiment params.json file in model_dir
args = parser.parse_args()
json_path = os.path.join(args.experiment_dir, 'params.json')
assert os.path.isfile(
json_path), "No json configuration file found at {}".format(json_path)
params = Params(json_path)
params.update(json_path)
# Set the logger
set_logger(os.path.join(args.experiment_dir, 'experiment.log'),
level=logging.INFO,
console=True)
# Log Title
logging.info("DPP-Diff Generator")
logging.info("Sabri Eyuboglu -- SNAP Group")
logging.info("======================================")
prepare_sns(sns, params)
diseases_dict = load_diseases(params.diseases_path, params.disease_subset)
Hh = np.zeros(n, dtype=np.cdouble)
for i in range(n):
for j in range(n):
# Jh[i]=Jh[i] + (ds/2.0/np.pi)*np.exp(1j*s[j]*sigma[i])*amp[j]
Hh[i] = Hh[i] + (ds / 2.0 / np.pi) * np.exp(1j * s[j] * sigma[i]) * \
amp[j] * (-np.abs(s[j]) / (3.0 * phix[i])) * rat[j]
print('Hh reals: ', np.real(Hh))
return np.abs(np.real(Hh))
if __name__ == '__main__':
# Set up line and get data from bin files
lya = Line(1215.6701, 0.4164, 6.265e8)
p = Params(line=lya, temp=1e4, tau0=1e7, num_dens=1701290465.5139434,
energy=1., R=1e11, sigma_source=0., n_points=1e4)
mu, x, time = read_bin('../data/1m_tau0_10000000.0_xinit_0.0_temp_10000.0_probabs_0.0/')
norm = 4.0 * np.pi * p.R**2. * p.delta * 4.0 * np.pi / p.energy
print('norm: ', norm)
# Set up matplotlib figure
fig, ax = plt.subplots(1, 1, figsize=(12, 9))
# Histogram data from Monte Carlo x
xbins, n, err = monte_carlo(x)
# xbins2, n2, err2 = monte_carlo(x, bins=256)
ax.errorbar(xbins, n, yerr=err, fmt='.', c='m', ms=3, label='Monte Carlo')
# ax.errorbar(xbins2, n2, yerr=err2, fmt='.', c='c', ms=3, label='Monte Carlo 2', alpha=0.25)
# H_0 from analytic solution
import secret
'''
This file contains the configurable hyperparameters of the CNN. For instance the backprogagation method,
the number of convolutional layers and their architecutre, curriculum learning behavior etc.
'''
#Create secret python file and variable token
token = secret.token
verbose = True
number_of_epochs = 100
dataset_path = '/home/olav/Pictures/Mass_roads_alpha'
pr_path = '/home/olav/Pictures/Mass_roads_alpha'
filename_params = Params({
"results" : "./results",
"network_save_name" : "./results/params.pkl",
"curriculum_teacher" : "/home/olav/Documents/Results/E7_inexperienced_teacher/teacher/params.pkl",
"curriculum_location" : "/media/olav/Data storage/dataset/Mass_inexperienced_100-2-5-stages"
})
visual_params = Params({
"endpoint" : "http://178.62.232.71/",
"gui_enabled" : True
})
optimization_params = Params({
"backpropagation" : "sgd_nesterov",
"batch_size" : 64,
"l2_reg" : 0.0001,
"momentum" : 0.93,
"initial_patience" : 500000,
(obj.p.delta * obj.kappa_n / obj.p.k)**2. * y1 -
3. * obj.omega * obj.p.delta**2. * obj.p.phi(float(sigma)) / obj.p.k /
c * x1, # dy2_dsigma
x2, # dx1_dsigma = x2
y2 # dy1_dsigma = y2
])
if __name__ == '__main__':
# Create params object
lya = Line(1215.6701, 0.4164, 6.265e8)
p = Params(line=lya,
temp=1e4,
tau0=1e7,
num_dens=1e6,
energy=1.,
sigma_source=0.,
n_points=1e5,
R=1e11)
# Comparison of characteristic time and characteristic frequency
tc = p.R / c * p.tau0 # Characteristic timescale
omega_c = c / p.R * (p.a * p.tau0)**(-1. / 3.)
print('1/tc=', 1. / tc)
print('omega_c=', omega_c)
print('R/c = ', p.R / c)
# Plot some fourier coefficients
# fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 6))
