class DiscontinuityType(object):
name = ''
pnames = []
npar = len(pnames)
penalty = 0.
def __init__(self, discontinuity):
self.discontinuity = self.d = discontinuity
self.best_fit_pv = None
self.best_fit_model = None
self.bic = None
self._optimization_result = None
# def __str__(self):
# return '{:s} {:4.1f} {:4.1f}'.format(self.name, self.position, self.amplitude)
# def __repr__(self):
# return '{:s}({:4.1f}, {:4.1f})'.format(self.name, self.position, self.amplitude)
def nlnlike_wn(self, pv):
if not self.is_inside_bounds(pv):
return inf
else:
return -lnlikelihood(self.d.flux, self.model(pv), self.d.wn_estimate)
def nlnlike_gp(self, pv):
if not self.is_inside_bounds(pv):
return inf
else:
return -self.d.gp.lnlikelihood(self.d.cadence, self.d.flux-self.model(pv), freeze_k=True)
def fit(self, use_de=False, de_npop=30, de_niter=100, method='Powell'):
jamp, jpos, fstd = self.d.amplitude, self.d.position, self.d.flux.std()
nlnlike = self.nlnlike_gp if self.d.use_gp else self.nlnlike_wn
if use_de:
self._de = DiffEvol(nlnlike, self._de_bounds(jamp, jpos, fstd), npop=de_npop)
self._de.optimize(de_niter)
pv0 = self._de.minimum_location
else:
pv0 = self._pv0(jamp, jpos, fstd)
self._optimization_result = r = minimize(nlnlike, pv0, method=method)
self.best_fit_pv = r.x
self.best_fit_model = self.model(r.x)
xx = r.x.copy()
xx[-2:] = 0
self.best_fit_model_wo_baseline = self.model(xx)
self.bic = self.c_bic(r.fun)
return self.bic
def model(self, pv, cad=None):
raise NotImplementedError
def c_bic(self, nln):
return 2*nln + self.npar*log(self.d.npt) + self.penalty
class DiscontinuityType(object):
name = ''
pnames = []
npar = len(pnames)
penalty = 0.
def __init__(self, discontinuity):
self.discontinuity = self.d = discontinuity
self.best_fit_pv = None
self.best_fit_model = None
self.bic = None
self._optimization_result = None
# def __str__(self):
# return '{:s} {:4.1f} {:4.1f}'.format(self.name, self.position, self.amplitude)
# def __repr__(self):
# return '{:s}({:4.1f}, {:4.1f})'.format(self.name, self.position, self.amplitude)
def nlnlike_wn(self, pv):
if not self.is_inside_bounds(pv):
return inf
else:
return -lnlikelihood(self.d.flux, self.model(pv),
self.d.wn_estimate)
def nlnlike_gp(self, pv):
if not self.is_inside_bounds(pv):
return inf
else:
return -self.d.gp.lnlikelihood(
self.d.cadence, self.d.flux - self.model(pv), freeze_k=True)
def fit(self, use_de=False, de_npop=30, de_niter=100, method='Powell'):
jamp, jpos, fstd = self.d.amplitude, self.d.position, self.d.flux.std()
nlnlike = self.nlnlike_gp if self.d.use_gp else self.nlnlike_wn
if use_de:
self._de = DiffEvol(nlnlike,
self._de_bounds(jamp, jpos, fstd),
npop=de_npop)
self._de.optimize(de_niter)
pv0 = self._de.minimum_location
else:
pv0 = self._pv0(jamp, jpos, fstd)
self._optimization_result = r = minimize(nlnlike, pv0, method=method)
self.best_fit_pv = r.x
self.best_fit_model = self.model(r.x)
xx = r.x.copy()
xx[-2:] = 0
self.best_fit_model_wo_baseline = self.model(xx)
self.bic = self.c_bic(r.fun)
return self.bic
def model(self, pv, cad=None):
raise NotImplementedError
def c_bic(self, nln):
return 2 * nln + self.npar * log(self.d.npt) + self.penalty
def fit(self, use_de=False, de_npop=30, de_niter=100, method='Powell'):
jamp, jpos, fstd = self.d.amplitude, self.d.position, self.d.flux.std()
nlnlike = self.nlnlike_gp if self.d.use_gp else self.nlnlike_wn
if use_de:
self._de = DiffEvol(nlnlike, self._de_bounds(jamp, jpos, fstd), npop=de_npop)
self._de.optimize(de_niter)
pv0 = self._de.minimum_location
else:
pv0 = self._pv0(jamp, jpos, fstd)
self._optimization_result = r = minimize(nlnlike, pv0, method=method)
self.best_fit_pv = r.x
self.best_fit_model = self.model(r.x)
xx = r.x.copy()
xx[-2:] = 0
self.best_fit_model_wo_baseline = self.model(xx)
self.bic = self.c_bic(r.fun)
return self.bic
def fit(self, use_de=False, de_npop=30, de_niter=100, method='Powell'):
jamp, jpos, fstd = self.d.amplitude, self.d.position, self.d.flux.std()
nlnlike = self.nlnlike_gp if self.d.use_gp else self.nlnlike_wn
if use_de:
self._de = DiffEvol(nlnlike, self._de_bounds(jamp, jpos, fstd), npop=de_npop)
self._de.optimize(de_niter)
pv0 = self._de.minimum_location
else:
pv0 = self._pv0(jamp, jpos, fstd)
self._optimization_result = r = minimize(nlnlike, pv0, method=method)
self.best_fit_pv = r.x
self.best_fit_model = self.model(r.x)
xx = r.x.copy()
xx[-2:] = 0
self.best_fit_model_wo_baseline = self.model(xx)
self.bic = self.c_bic(r.fun)
return self.bic
def pyorbit_emcee(config_in, input_datasets=None, return_output=None):
try:
import emcee
except:
print("ERROR: emcee not installed, this will not work")
quit()
os.environ["OMP_NUM_THREADS"] = "1"
optimize_dir_output = './' + config_in['output'] + '/optimize/'
pyde_dir_output = './' + config_in['output'] + '/pyde/'
emcee_dir_output = './' + config_in['output'] + '/emcee/'
reloaded_optimize = False
reloaded_pyde = False
reloaded_emcee = False
try:
mc, population, starting_point, theta_dict = pyde_load_from_cpickle(
pyde_dir_output, prefix='')
reloaded_pyde = True
except FileNotFoundError:
pass
try:
mc, starting_point, population, prob, state, sampler_chain, \
sampler_lnprobability, _, theta_dict, sampler = \
emcee_load_from_cpickle(emcee_dir_output)
reloaded_emcee = True
except FileNotFoundError:
pass
try:
starting_point, previous_boundaries, theta_dict = starting_point_load_from_cpickle(
optimize_dir_output)
reloaded_optimize = True
except FileNotFoundError:
pass
print()
print('reloaded_optimize: ', reloaded_optimize)
print('reloaded_pyde: ', reloaded_pyde)
print('reloaded_emcee: ', reloaded_emcee)
print()
if reloaded_pyde or reloaded_emcee:
previous_boundaries = mc.bounds
if reloaded_emcee:
print('Requested steps:', mc.emcee_parameters['nsteps'])
mc.emcee_parameters['completed_nsteps'] = \
int(sampler_chain.shape[1] * mc.emcee_parameters['thin'])
print('Completed:', mc.emcee_parameters['completed_nsteps'])
pars_input(config_in, mc, input_datasets, reload_emcee=True)
print('Total:', mc.emcee_parameters['nsteps'])
""" There's no need to do anything"""
flatchain = emcee_flatchain(sampler_chain,
mc.emcee_parameters['nburn'],
mc.emcee_parameters['thin'])
mc.model_setup()
mc.initialize_logchi2()
results_analysis.print_integrated_ACF(sampler_chain, theta_dict,
mc.emcee_parameters['thin'])
""" In case the current startin point comes from a previous analysis """
mc.emcee_dir_output = emcee_dir_output
if mc.emcee_parameters['nsteps'] <= mc.emcee_parameters[
'completed_nsteps']:
print('Reference Time Tref: ', mc.Tref)
print()
print('Dimensions = ', mc.ndim)
print('Nwalkers = ', mc.emcee_parameters['nwalkers'])
print('Steps = ', mc.emcee_parameters['nsteps'])
print()
print('Original starting point of emcee:')
print()
results_analysis.results_resumen(mc,
starting_point,
compute_lnprob=True,
is_starting_point=True)
print('emcee results:')
print()
results_analysis.results_resumen(mc, flatchain)
print()
print('emcee completed')
print()
if return_output:
return mc, sampler_chain, sampler_lnprobability
else:
return
elif not sampler:
print(
'Sampler file is missing - only analysis performed with PyORBIT >8.1 cn be resumed'
)
if return_output:
return mc, sampler_chain, sampler_lnprobability
else:
return
if reloaded_emcee:
sampled = mc.emcee_parameters['completed_nsteps']
nsteps_todo = mc.emcee_parameters['nsteps'] \
- mc.emcee_parameters['completed_nsteps']
print('Resuming from a previous run:')
print('Performed steps = ', mc.emcee_parameters['completed_nsteps'])
print('Final # of steps = ', mc.emcee_parameters['nsteps'])
print('Steps to be performed = ', nsteps_todo)
print()
else:
mc = ModelContainerEmcee()
pars_input(config_in, mc, input_datasets)
if mc.pyde_parameters['shutdown_jitter'] or mc.emcee_parameters[
'shutdown_jitter']:
for dataset_name, dataset in mc.dataset_dict.items():
dataset.shutdown_jitter()
# keep track of which version has been used to perform emcee computations
mc.emcee_parameters['version'] = emcee.__version__[0]
mc.model_setup()
mc.create_variables_bounds()
mc.initialize_logchi2()
results_analysis.results_resumen(mc, None, skip_theta=True)
mc.pyde_dir_output = pyde_dir_output
mc.emcee_dir_output = emcee_dir_output
mc.emcee_parameters['nwalkers'] = mc.ndim * \
mc.emcee_parameters['npop_mult']
if mc.emcee_parameters['nwalkers'] % 2 == 1:
mc.emcee_parameters['nwalkers'] += 1
if not os.path.exists(mc.emcee_dir_output):
os.makedirs(mc.emcee_dir_output)
state = None
sampled = 0
nsteps_todo = mc.emcee_parameters['nsteps']
print('Include priors: ', mc.include_priors)
print()
print('Reference Time Tref: ', mc.Tref)
print()
print('Dimensions = ', mc.ndim)
print('Nwalkers = ', mc.emcee_parameters['nwalkers'])
print()
if mc.emcee_parameters['version'] == '2':
mc.emcee_parameters['use_threading_pool'] = False
if not mc.pyde_parameters.get('use_threading_pool', False):
mc.pyde_parameters['use_threading_pool'] = False
if not mc.emcee_parameters.get('use_threading_pool', False):
mc.emcee_parameters['use_threading_pool'] = False
if reloaded_emcee:
sys.stdout.flush()
pass
elif reloaded_pyde:
theta_dict_legacy = theta_dict.copy()
population_legacy = population.copy()
theta_dict = results_analysis.get_theta_dictionary(mc)
population = np.zeros([mc.emcee_parameters['nwalkers'], mc.ndim],
dtype=np.double)
for theta_name, theta_i in theta_dict.items():
population[:, theta_i] = population_legacy[:, theta_dict_legacy[
theta_name]]
mc.bounds[theta_i] = previous_boundaries[
theta_dict_legacy[theta_name]]
starting_point = np.median(population, axis=0)
# print(starting_point)
# print(population)
print('Using previous population as starting point. ')
print()
sys.stdout.flush()
elif mc.starting_point_flag or reloaded_optimize:
if reloaded_optimize:
print(
'Using the output from a previous optimize run as starting point'
)
theta_dict_legacy = theta_dict.copy()
starting_point_legacy = starting_point.copy()
theta_dict = results_analysis.get_theta_dictionary(mc)
for theta_name, theta_i in theta_dict.items():
starting_point[theta_i] = starting_point_legacy[
theta_dict_legacy[theta_name]]
else:
print('Using user-defined starting point from YAML file')
mc.create_starting_point()
starting_point = mc.starting_point
population = np.zeros([mc.emcee_parameters['nwalkers'], mc.ndim],
dtype=np.double)
for ii in range(0, mc.emcee_parameters['nwalkers']):
population[ii, :] = np.random.normal(starting_point, 0.0000001)
print(
'to create a synthetic population extremely close to the starting values.'
)
print()
sys.stdout.flush()
else:
""" None of the previous cases has been satisfied, we have to run PyDE """
try:
from pyde.de import DiffEvol
except ImportError:
print(
'ERROR! PyDE is not installed, run first with optimize instead of emcee'
)
quit()
if not os.path.exists(mc.pyde_dir_output):
os.makedirs(mc.pyde_dir_output)
print('Using threading pool for PyDE:',
mc.pyde_parameters['use_threading_pool'])
print('PyDE running')
sys.stdout.flush()
if mc.pyde_parameters['use_threading_pool']:
with multiprocessing.Pool() as pool:
de = DiffEvol(mc,
mc.bounds,
mc.emcee_parameters['nwalkers'],
maximize=True,
pool=pool)
de.optimize(int(mc.pyde_parameters['ngen']))
else:
de = DiffEvol(mc,
mc.bounds,
mc.emcee_parameters['nwalkers'],
maximize=True)
de.optimize(int(mc.pyde_parameters['ngen']))
population = de.population
starting_point = np.median(population, axis=0)
theta_dict = results_analysis.get_theta_dictionary(mc)
""" bounds redefinition and fix for PyDE anomalous results """
if mc.recenter_bounds_flag:
pyde_save_to_pickle(mc,
population,
starting_point,
theta_dict,
prefix='orig')
mc.recenter_bounds(starting_point)
population = mc.fix_population(starting_point, population)
starting_point = np.median(population, axis=0)
print('Boundaries redefined after PyDE output')
pyde_save_to_pickle(mc, population, starting_point, theta_dict)
print('PyDE completed')
print()
sys.stdout.flush()
if reloaded_emcee:
print('Original starting point of emcee:')
print()
results_analysis.results_resumen(mc,
starting_point,
compute_lnprob=True,
is_starting_point=True)
sys.stdout.flush()
print()
print('*************************************************************')
print()
print('Running emcee')
print()
print('emcee version: ', emcee.__version__)
if mc.emcee_parameters['version'] == '2':
print('WARNING: upgrading to version 3 is strongly advised')
print('Using threading pool for emcee:',
mc.emcee_parameters.get('use_threading_pool', False))
print()
if reloaded_emcee:
print('Using reloaded sampler')
print()
else:
sampler = emcee.EnsembleSampler(mc.emcee_parameters['nwalkers'],
mc.ndim, mc)
if mc.emcee_parameters['nsave'] > 0:
print('Saving temporary steps')
print()
niter = int(np.ceil(nsteps_todo / mc.emcee_parameters['nsave']))
for i in range(0, niter):
if mc.emcee_parameters['use_threading_pool']:
with multiprocessing.Pool() as pool:
sampler.pool = pool
population, prob, state = sampler.run_mcmc(
population,
mc.emcee_parameters['nsave'],
thin=mc.emcee_parameters['thin'],
rstate0=state,
progress=True)
else:
population, prob, state = sampler.run_mcmc(
population,
mc.emcee_parameters['nsave'],
thin=mc.emcee_parameters['thin'],
rstate0=state,
progress=True)
sampled += mc.emcee_parameters['nsave']
theta_dict = results_analysis.get_theta_dictionary(mc)
emcee_save_to_cpickle(mc,
starting_point,
population,
prob,
state,
sampler,
theta_dict,
samples=sampled)
flatchain = emcee_flatchain(sampler.chain,
mc.emcee_parameters['nburn'],
mc.emcee_parameters['thin'])
results_analysis.print_integrated_ACF(sampler.chain, theta_dict,
mc.emcee_parameters['thin'])
results_analysis.results_resumen(mc, flatchain)
print()
print(sampled, ' steps completed, average lnprob:, ',
np.median(prob))
print()
sys.stdout.flush()
# It turns out that reloading the sampler from the file will
# result in faster parallelization...
state, sampler = emcee_simpler_load_from_cpickle(emcee_dir_output)
else:
if mc.emcee_parameters['use_threading_pool']:
with multiprocessing.Pool() as pool:
sampler.pool = pool
population, prob, state = sampler.run_mcmc(
population,
nsteps_todo,
thin=mc.emcee_parameters['thin'],
rstate0=state,
progress=True)
else:
population, prob, state = sampler.run_mcmc(
population,
nsteps_todo,
thin=mc.emcee_parameters['thin'],
rstate0=state,
progress=True)
sampled += nsteps_todo
theta_dict = results_analysis.get_theta_dictionary(mc)
emcee_save_to_cpickle(mc,
starting_point,
population,
prob,
state,
sampler,
theta_dict,
samples=sampled)
flatchain = emcee_flatchain(sampler.chain,
mc.emcee_parameters['nburn'],
mc.emcee_parameters['thin'])
results_analysis.print_integrated_ACF(sampler.chain, theta_dict,
mc.emcee_parameters['thin'])
results_analysis.results_resumen(mc, flatchain)
print()
print('emcee completed')
print()
""" A dummy file is created to let the cpulimit script to proceed with the next step"""
emcee_create_dummy_file(mc)
if return_output:
return mc, sampler.chain, sampler.lnprobability
def pyorbit_emcee(config_in, input_datasets=None, return_output=None):
try:
import emcee
except:
print("ERROR: emcee not installed, this will not work")
quit()
os.environ["OMP_NUM_THREADS"] = "1"
optimize_dir_output = './' + config_in['output'] + '/optimize/'
pyde_dir_output = './' + config_in['output'] + '/pyde/'
emcee_dir_output = './' + config_in['output'] + '/emcee/'
reloaded_optimize = False
reloaded_pyde = False
reloaded_emcee_multirun = False
reloaded_emcee = False
try:
mc, population, starting_point, theta_dict = pyde_load_from_cpickle(
pyde_dir_output, prefix='')
reloaded_pyde = True
except:
pass
try:
mc, starting_point, population, _, _, sampler_chain, _, _, theta_dict, _ = \
emcee_load_from_cpickle(emcee_dir_output, prefix='MR')
reloaded_emcee_multirun = True
except:
pass
try:
mc, starting_point, population, _, _, sampler_chain, sampler_lnprobability, _, theta_dict, _ = \
emcee_load_from_cpickle(emcee_dir_output)
reloaded_emcee = True
except:
pass
try:
starting_point, previous_boundaries, theta_dict = starting_point_load_from_cpickle(
optimize_dir_output)
reloaded_optimize = True
except:
pass
print()
print('reloaded_optimize: ', reloaded_pyde)
print('reloaded_pyde: ', reloaded_pyde)
print('reloaded_emcee_multirun: ', reloaded_emcee_multirun)
print('reloaded_emcee: ', reloaded_emcee)
if reloaded_emcee:
""" There's no need to do anything"""
flatchain = emcee_flatchain(sampler_chain,
mc.emcee_parameters['nburn'],
mc.emcee_parameters['thin'])
mc.model_setup()
mc.initialize_logchi2()
results_analysis.print_integrated_ACF(sampler_chain, theta_dict,
mc.emcee_parameters['thin'])
results_analysis.results_resumen(mc, flatchain)
if return_output:
return mc, sampler_chain, sampler_lnprobability
else:
return
reloaded_mc = reloaded_pyde or reloaded_emcee_multirun
if reloaded_mc:
previous_boundaries = mc.bounds
mc = ModelContainerEmcee()
pars_input(config_in, mc, input_datasets)
if mc.pyde_parameters['shutdown_jitter'] or mc.emcee_parameters[
'shutdown_jitter']:
for dataset_name, dataset in mc.dataset_dict.items():
dataset.shutdown_jitter()
# keep track of which version has been used to perform emcee computations
mc.emcee_parameters['version'] = emcee.__version__[0]
mc.model_setup()
mc.create_variables_bounds()
mc.initialize_logchi2()
results_analysis.results_resumen(mc, None, skip_theta=True)
mc.pyde_dir_output = pyde_dir_output
mc.emcee_dir_output = emcee_dir_output
mc.emcee_parameters['nwalkers'] = mc.ndim * \
mc.emcee_parameters['npop_mult']
if mc.emcee_parameters['nwalkers'] % 2 == 1:
mc.emcee_parameters['nwalkers'] += 1
if not os.path.exists(mc.emcee_dir_output):
os.makedirs(mc.emcee_dir_output)
print()
print('emcee version: ', emcee.__version__)
if mc.emcee_parameters['version'] == '2':
print('WARNING: upgrading to version 3 is strongly advised')
print()
print('Include priors: ', mc.include_priors)
print()
print('Reference Time Tref: ', mc.Tref)
print()
print('Dimensions = ', mc.ndim)
print('Nwalkers = ', mc.emcee_parameters['nwalkers'])
if not getattr(mc, 'use_threading_pool', False):
mc.use_threading_pool = False
print()
print('Using threading pool:', mc.use_threading_pool)
print()
print('*************************************************************')
print()
if reloaded_mc:
theta_dict_legacy = theta_dict.copy()
population_legacy = population.copy()
theta_dict = results_analysis.get_theta_dictionary(mc)
population = np.zeros([mc.emcee_parameters['nwalkers'], mc.ndim],
dtype=np.double)
for theta_name, theta_i in theta_dict.items():
population[:, theta_i] = population_legacy[:, theta_dict_legacy[
theta_name]]
mc.bounds[theta_i] = previous_boundaries[
theta_dict_legacy[theta_name]]
starting_point = np.median(population, axis=0)
# print(starting_point)
# print(population)
print('Using previous population as starting point. ')
sys.stdout.flush()
print()
else:
if mc.starting_point_flag or reloaded_optimize:
if reloaded_optimize:
print(
'Using the output from a previous optimize run as starting point'
)
theta_dict_legacy = theta_dict.copy()
starting_point_legacy = starting_point.copy()
theta_dict = results_analysis.get_theta_dictionary(mc)
for theta_name, theta_i in theta_dict.items():
starting_point[theta_i] = starting_point_legacy[
theta_dict_legacy[theta_name]]
else:
print('Using user-defined starting point from YAML file')
mc.create_starting_point()
starting_point = mc.starting_point
population = np.zeros([mc.emcee_parameters['nwalkers'], mc.ndim],
dtype=np.double)
for ii in range(0, mc.emcee_parameters['nwalkers']):
population[ii, :] = np.random.normal(starting_point, 0.0000001)
print(
'to create a synthetic population extremely close to the starting values.'
)
sys.stdout.flush()
else:
try:
from pyde.de import DiffEvol
except ImportError:
print(
'ERROR! PyDE is not installed, run first with optimize instead of emcee'
)
quit()
if not os.path.exists(mc.pyde_dir_output):
os.makedirs(mc.pyde_dir_output)
print('PyDE running')
sys.stdout.flush()
de = DiffEvol(mc,
mc.bounds,
mc.emcee_parameters['nwalkers'],
maximize=True)
de.optimize(int(mc.pyde_parameters['ngen']))
population = de.population
starting_point = np.median(population, axis=0)
theta_dict = results_analysis.get_theta_dictionary(mc)
""" bounds redefinition and fix for PyDE anomalous results """
if mc.recenter_bounds_flag:
pyde_save_to_pickle(mc,
population,
starting_point,
theta_dict,
prefix='orig')
mc.recenter_bounds(starting_point)
population = mc.fix_population(starting_point, population)
starting_point = np.median(population, axis=0)
print('Boundaries redefined after PyDE output')
pyde_save_to_pickle(mc, population, starting_point, theta_dict)
print('PyDE completed')
sys.stdout.flush()
results_analysis.results_resumen(mc,
starting_point,
compute_lnprob=True,
is_starting_point=True)
if mc.use_threading_pool:
if mc.emcee_parameters['version'] == '2':
threads_pool = emcee.interruptible_pool.InterruptiblePool(
mc.emcee_parameters['nwalkers'])
else:
from multiprocessing.pool import Pool as InterruptiblePool
threads_pool = InterruptiblePool(mc.emcee_parameters['nwalkers'])
if mc.emcee_parameters['multirun'] and not reloaded_emcee_multirun:
for ii in range(0, mc.emcee_parameters['multirun_iter']):
print('emcee exploratory run #', ii, ' of ',
mc.emcee_parameters['multirun_iter'])
# sampler = emcee.EnsembleSampler(mc.emcee_parameters['nwalkers'], mc.ndim, mc,
# threads=mc.emcee_parameters['nwalkers'])
if mc.use_threading_pool:
sampler = emcee.EnsembleSampler(
mc.emcee_parameters['nwalkers'],
mc.ndim,
mc,
pool=threads_pool)
else:
sampler = emcee.EnsembleSampler(
mc.emcee_parameters['nwalkers'], mc.ndim, mc)
population, prob, state = sampler.run_mcmc(
population, mc.emcee_parameters['multirun'])
flatchain = emcee_flatchain(sampler.chain,
mc.emcee_parameters['nburn'],
mc.emcee_parameters['thin'])
results_analysis.results_resumen(mc, flatchain)
max_ind = np.argmax(prob)
starting_point = population[max_ind, :]
population = np.asarray([
starting_point + 1e-4 * np.random.randn(mc.ndim)
for i in range(mc.emcee_parameters['nwalkers'])
])
sampler.reset()
theta_dict = results_analysis.get_theta_dictionary(mc)
emcee_save_to_cpickle(mc,
starting_point,
population,
prob,
state,
sampler,
theta_dict,
prefix='MR_' + repr(ii))
emcee_save_to_cpickle(mc,
starting_point,
population,
prob,
state,
sampler,
theta_dict,
prefix='MR')
flatchain = emcee_flatchain(sampler.chain,
mc.emcee_parameters['nburn'],
mc.emcee_parameters['thin'])
results_analysis.print_integrated_ACF(sampler.chain, theta_dict,
mc.emcee_parameters['thin'])
results_analysis.results_resumen(mc, flatchain)
print('emcee exploratory runs completed')
sys.stdout.flush()
print()
print('Running emcee')
state = None
if mc.use_threading_pool:
sampler = emcee.EnsembleSampler(mc.emcee_parameters['nwalkers'],
mc.ndim,
mc,
pool=threads_pool)
else:
sampler = emcee.EnsembleSampler(mc.emcee_parameters['nwalkers'],
mc.ndim, mc)
if mc.emcee_parameters['nsave'] > 0:
print()
print(' Saving temporary steps')
niter = int(mc.emcee_parameters['nsteps'] /
mc.emcee_parameters['nsave'])
sampled = 0
for i in range(0, niter):
population, prob, state = sampler.run_mcmc(
population,
mc.emcee_parameters['nsave'],
thin=mc.emcee_parameters['thin'],
rstate0=state)
sampled += mc.emcee_parameters['nsave']
theta_dict = results_analysis.get_theta_dictionary(mc)
emcee_save_to_cpickle(mc,
starting_point,
population,
prob,
state,
sampler,
theta_dict,
samples=sampled)
flatchain = emcee_flatchain(sampler.chain,
mc.emcee_parameters['nburn'],
mc.emcee_parameters['thin'])
results_analysis.print_integrated_ACF(sampler.chain, theta_dict,
mc.emcee_parameters['thin'])
results_analysis.results_resumen(mc, flatchain)
print()
print(sampled, ' steps completed, average lnprob:, ',
np.median(prob))
sys.stdout.flush()
else:
population, prob, state = sampler.run_mcmc(
population,
mc.emcee_parameters['nsteps'],
thin=mc.emcee_parameters['thin'])
theta_dict = results_analysis.get_theta_dictionary(mc)
emcee_save_to_cpickle(mc, starting_point, population, prob, state,
sampler, theta_dict)
flatchain = emcee_flatchain(sampler.chain,
mc.emcee_parameters['nburn'],
mc.emcee_parameters['thin'])
results_analysis.print_integrated_ACF(sampler.chain, theta_dict,
mc.emcee_parameters['thin'])
results_analysis.results_resumen(mc, flatchain)
print()
print('emcee completed')
if mc.use_threading_pool:
# close the pool of threads
threads_pool.close()
threads_pool.terminate()
threads_pool.join()
""" A dummy file is created to let the cpulimit script to proceed with the next step"""
emcee_create_dummy_file(mc)
if return_output:
return mc, sampler.chain, sampler.lnprobability
print 'Variable bounds:', mc.bounds
print
mc.nwalkers = mc.ndim * mc.npop_mult
if mc.nwalkers % 2 == 1: mc.nwalkers += 1
print 'Nwalkers = ', mc.nwalkers
if os.path.isfile(dir_output + 'pyde_pops.pick'):
population = pickle.load(open(dir_output + 'pyde_pops.pick', 'rb'))
pyde_mean = np.median(population, axis=0)
#pyde_mean = pickle.load(open(dir_output + 'pyde_mean.pick', 'rb'))
mc.recenter_bounds(pyde_mean, population)
else:
print 'PyDE'
de = DiffEvol(mc, mc.bounds, mc.nwalkers, maximize=True)
de.optimize(mc.ngen)
print 'PyDE completed'
population = de.population
pyde_mean = np.median(population, axis=0)
pickle.dump(pyde_mean, open(dir_output + 'pyde_mean.pick', 'wb'))
#np.savetxt(dir_output + 'pyDEout_original_bounds.dat', mc.bounds)
#np.savetxt(dir_output + 'pyDEout_original_pops.dat', population)
# bounds redefinition and fix for PyDE anomalous results
if mc.recenter_bounds_flag:
pickle.dump(mc.bounds, open(dir_output + 'bounds_orig.pick', 'wb'))
pickle.dump(population, open(dir_output + 'pyde_pops_orig.pick', 'wb'))
mc.recenter_bounds(pyde_mean, population)
import numpy as np
import matplotlib.pyplot as pl
from pyde.de import DiffEvol
class Model(object):
def __init__(self, x):
self.x = x
def __call__(self, a, p0, f):
return a*np.sin(p0+2*np.pi*self.x*f)
if __name__ == '__main__':
x = np.linspace(0,10,100)
m = Model(x)
y = m(2,1,0.25) + np.random.normal(0,0.5,size=x.size)
de = DiffEvol(lambda pv: ((y-m(*pv))**2).sum(), [[1,3],[0,2],[0,4]], 50)
## Run n number of generations
res = de.optimize(250)
## Or use as an iterator
for res in de(2):
print res
pl.plot(x,y,'.k')
pl.plot(x,m(*res[0]),'k')
pl.show()
mc.create_bounds()
print 'Dimensions = ', mc.ndim
print ' '
print 'Variable list:', mc.variable_list
print
print 'Variable bounds:', mc.bounds
print
mc.nwalkers = mc.ndim * mc.npop_mult
if mc.nwalkers%2 == 1: mc.nwalkers += 1
print 'Nwalkers = ', mc.nwalkers
print 'PyDE'
de = DiffEvol(mc, mc.bounds, mc.nwalkers, maximize=True)
de.optimize(mc.ngen)
print 'PyDE completed'
population = de.population
pyde_mean = np.mean(population, axis=0)
pickle.dump(pyde_mean, open(dir_output + 'pyde_mean.pick', 'wb'))
#np.savetxt(dir_output + 'pyDEout_original_bounds.dat', mc.bounds)
#np.savetxt(dir_output + 'pyDEout_original_pops.dat', population)
# bounds redefinition and fix for PyDE anomalous results
if mc.recenter_bounds_flag:
pickle.dump(mc.bounds, open(dir_output + 'bounds_orig.pick', 'wb'))
pickle.dump(population, open(dir_output + 'pyde_pops_orig.pick', 'wb'))
mc.recenter_bounds(pyde_mean, population)
else:
if not os.path.exists(pyde_dir_output):
os.makedirs(pyde_dir_output)
if os.path.isfile(pyde_dir_output + 'pyde_pops.pick'):
print os.path.isfile(pyde_dir_output + 'pyde_pops.pick')
population = pickle.load(open(pyde_dir_output + 'pyde_pops.pick',
'rb'))
starting_point = np.median(population, axis=0)
mc.recenter_bounds(starting_point, population)
else:
print 'PyDE'
de = DiffEvol(mc,
mc.bounds,
mc.emcee_parameters['nwalkers'],
maximize=True)
de.optimize(mc.pyde_parameters['ngen'])
print 'PyDE completed'
population = de.population
starting_point = np.median(population, axis=0)
pickle.dump(starting_point,
open(pyde_dir_output + 'pyde_mean.pick', 'wb'))
#np.savetxt(pyde_dir_output + 'pyDEout_original_bounds.dat', mc.bounds)
#np.savetxt(pyde_dir_output + 'pyDEout_original_pops.dat', population)
# bounds redefinition and fix for PyDE anomalous results
if mc.recenter_bounds_flag:
pickle.dump(mc.bounds,
class Model(object):
def __init__(self, x):
self.x = x
def __call__(self, a, p0, f):
return a * np.sin(p0 + 2 * np.pi * self.x * f)
if __name__ == '__main__':
niter = 250
x = np.linspace(0, 10, 100)
m = Model(x)
y = m(2, 1, 0.25) + np.random.normal(0, 0.5, size=x.size)
de = DiffEvol(lambda pv: ((y - m(*pv))**2).sum(), [[1, 3], [0, 2], [0, 4]],
50)
## Run n number of generations
tstart = time()
res = de.optimize(niter)
print(((time() - tstart) / niter))
## Or use as an iterator
for res in de(2):
print(res)
pl.plot(x, y, '.k')
pl.plot(x, m(*res[0]), 'k')
pl.show()
if fwp_flag[nk]:
for ii in xrange(0, np.sum(n_amp[0:n_fwa])):
bounds[ip + ii, :] = fwa_bounds[:]
ip += np.sum(n_amp[0:n_fwa])
if bsp_flag[nk]:
for ii in xrange(0, np.sum(n_amp[0:n_bsa])):
bounds[ip + ii, :] = bsa_bounds[:]
ip += np.sum(n_amp[0:n_bsa])
print 'BOUNDS'
print bounds
print 'PyDE'
de = DiffEvol(lnprob_PyDE, bounds, npop, maximize=True)
de.optimize(ngen)
print 'PyDE completed'
np.savetxt('output/' + planet_name + '_pyDEout_output_bounds.dat',bounds)
np.savetxt('output/' + planet_name + '_pyDEout_output_pops.dat',de.population)
pyde_mean = np.mean(de.population, axis=0)
print pyde_mean
np.savetxt('output/' + planet_name + '_pyDEout_mean.dat',pyde_mean)
# fix for PyDE anomalous results
for ii in xrange(0,ndim):
if np.amax(de.population[:,ii])-np.amin(de.population[:,ii]) < 10e-7 :
range_restricted = (bounds[ii,1]-bounds[ii,0])/1000.
if fwp_flag[nk]:
for ii in xrange(0, np.sum(n_amp[0:n_fwa])):
bounds[ip + ii, :] = fwa_bounds[:]
ip += np.sum(n_amp[0:n_fwa])
if bsp_flag[nk]:
for ii in xrange(0, np.sum(n_amp[0:n_bsa])):
bounds[ip + ii, :] = bsa_bounds[:]
ip += np.sum(n_amp[0:n_bsa])
print 'BOUNDS'
print bounds
print 'PyDE'
de = DiffEvol(lnprob_PyDE, bounds, npop, maximize=True)
de.optimize(ngen)
print 'PyDE completed'
np.savetxt('output/' + planet_name + '_pyDEout_output_bounds.dat', bounds)
np.savetxt('output/' + planet_name + '_pyDEout_output_pops.dat', de.population)
pyde_mean = np.mean(de.population, axis=0)
print pyde_mean
np.savetxt('output/' + planet_name + '_pyDEout_mean.dat', pyde_mean)
# fix for PyDE anomalous results
for ii in xrange(0, ndim):
if np.amax(de.population[:, ii]) - np.amin(de.population[:, ii]) < 10e-7:
range_restricted = (bounds[ii, 1] - bounds[ii, 0]) / 1000.
min_bound = np.maximum((pyde_mean[ii] - range_restricted / 2.0),
def pe_runner(lpfun, basename, **kwargs):
result_dir = kwargs.get('result_dir', '.')
plot_dir = kwargs.get('plot_dir', '.')
runname = kwargs.get('run_name', 'default')
n_de_iters = kwargs.get('n_de_iterations', 150)
n_mc_iters = kwargs.get('n_mc_iterations', 150)
pop_size = kwargs.get('pop_size', 100)
thinning = kwargs.get('thinning', 1)
mc_continue = kwargs.get('mc_continues', True)
update_interval = kwargs.get('update_interval', 60)
do_de = kwargs.get('do_de', False)
do_mc = kwargs.get('do_mc', False)
pplot = ProgressPlotter(lpfun, '{:s}_{:s}'.format(basename, runname),
plot_dir)
de_res_fname = join(result_dir,
'{:s}_{:s}_de.pkl'.format(basename, runname))
mc_res_fname = join(result_dir,
'{:s}_{:s}_mc.pkl'.format(basename, runname))
mc_bck_fname = join(result_dir,
'{:s}_{:s}_mc_back.pkl'.format(basename, runname))
de_res_exists = exists(de_res_fname)
mc_res_exists = exists(mc_res_fname)
## ================================================================================
## DE - Run Differential Evolution
## ================================================================================
if do_de or not de_res_exists:
t_start = time()
de = DiffEvol(lambda pv: -lpfun(pv), lpfun.ps.bounds, pop_size)
best_ll = 1e80
for i, res in enumerate(de(n_de_iters)):
if res[1] < best_ll:
print(i, res[1])
best_ll = res[1]
with open(de_res_fname, 'w') as fout:
dump({
'population': de.population,
'best': de.minimum_location
}, fout)
de_population, de_best_fit = de.population, de.minimum_location
print("time taken: {:4.2f}".format(time() - t_start))
else:
with open(de_res_fname, 'r') as fin:
de_res = load(fin)
de_population, de_best_fit = de_res['population'], de_res['best']
if not do_mc and not mc_res_exists:
return de_population, de_best_fit
## ================================================================================
## MCMC - Run emcee
## ================================================================================
if do_mc or not mc_res_exists:
t_iteration_start = time()
t_last_update = time()
t_start = time()
if mc_res_exists and mc_continue:
print('Continuing from the previous MCMC run.')
with open(mc_res_fname, 'r') as fin:
pv0 = load(fin)['chain'][:, -1, :]
else:
pv0 = de_population
if pv0.shape[0] > pop_size:
pv0 = pv0[:pop_size, :].copy()
elif pv0.shape[0] < pop_size:
pv0 = np.tile(
pv0,
[np.ceil(pop_size / pv0.shape[0]), 1])[:pop_size, :].copy()
#pv0[:,14] = np.random.uniform(0.01, 0.98, size=300)
#m = pv0[:,2] > 0.03
#pv0[m,:] = pv0[~m,:][:m.sum(),:]
#pv0[:,2] = np.random.uniform(0.005, 0.1, size=300)
#print pv0.shape;exit()
sampler = emcee.EnsembleSampler(pop_size, lpfun.ps.ndim, lpfun)
for i, e in enumerate(
sampler.sample(pv0, iterations=n_mc_iters, thin=thinning)):
t_cur = time()
print(
"{:4d}/{:4d} Secs/iteration {:6.2f} Last update {:6.2f} s ago Total time {:6.2f} s Acceptance {:6.3f}"
.format(i + 1, n_mc_iters, t_cur - t_iteration_start,
t_cur - t_last_update, t_cur - t_start,
sampler.acceptance_fraction.mean()))
if i != 0 and (t_cur - t_last_update > update_interval):
pplot.update(sampler.chain, i // thinning)
t_last_update = t_cur
with open(mc_bck_fname, 'w') as fout:
dump(
{
'chain': sampler.chain,
'logl': sampler.lnprobability
}, fout)
t_iteration_start = t_cur
with open(mc_res_fname, 'w') as fout:
dump({'chain': sampler.chain, 'logl': sampler.lnprobability}, fout)
chain = sampler.chain
else:
with open(mc_res_fname, 'r') as fin:
chain = load(fin)['chain']
return chain
