def init_folder(working_path, sub_folder):
working_folder = os.path.join(working_path, sub_folder)
if (not os.path.isdir(working_folder)):
os.makedirs(working_folder)
#arg_file = os.path.join(working_path, 'arg.in')
#shutil.copy(arg_file, os.path.join(working_folder,''))
#bodies_file = os.path.join(working_path, 'bodies.lst')
#shutil.copy(bodies_file, os.path.join(working_folder,''))
#obd = open(bodies_file, 'r')
#for line in obd.readlines():
#shutil.copy(os.path.join(working_path, line.strip().split()[0]), os.path.join(working_folder,''))
#obd.close()
#t0files = glob.glob(os.path.join(working_path,'NB*_observations.dat'))
#for t0f in t0files:
#shutil.copy(t0f, os.path.join(working_folder,''))
#if(os.path.exists(os.path.join(working_path,'obsRV.dat'))):
#shutil.copy(os.path.join(working_path,'obsRV.dat'), os.path.join(working_folder,''))
# copy files
anc.copy_simulation_files(working_path, working_folder)
run_log = os.path.join(working_folder, "trades_run.log")
of_run = open(run_log, 'w')
anc.print_both("# pyTRADES LOG FILE", of_run)
anc.print_both("# working_path = %s" %(working_path), of_run)
anc.print_both("# working_folder = %s" %(working_folder), of_run)
anc.print_both("# run_log = %s" %(run_log), of_run)
return working_folder, run_log, of_run
def init_folder(working_path, sub_folder):
working_folder = os.path.join(working_path, sub_folder)
if (not os.path.isdir(working_folder)):
os.makedirs(working_folder)
# copy files
anc.copy_simulation_files(working_path, working_folder)
run_log = os.path.join(working_folder, "trades_run.log")
of_run = open(run_log, 'w')
anc.print_both("# pyTRADES LOG FILE", of_run)
anc.print_both("# working_path = %s" %(working_path), of_run)
anc.print_both("# working_folder = %s" %(working_folder), of_run)
anc.print_both("# run_log = %s" %(run_log), of_run)
return working_folder, run_log, of_run
def init_folder(working_path, sub_folder):
working_folder = os.path.join(working_path, sub_folder)
if (not os.path.isdir(working_folder)):
os.makedirs(working_folder)
# copy files
anc.copy_simulation_files(working_path, working_folder)
run_log = os.path.join(working_folder, "trades_run.log")
of_run = open(run_log, 'w')
anc.print_both("# pyTRADES LOG FILE", of_run)
anc.print_both("# working_path = %s" % (working_path), of_run)
anc.print_both("# working_folder = %s" % (working_folder), of_run)
anc.print_both("# run_log = %s" % (run_log), of_run)
return working_folder, run_log, of_run
def main():
# MAIN -- TRADES + EMCEE
# READ COMMAND LINE ARGUMENTS
cli = get_args()
# STARTING TIME
start = time.time()
# RENAME
working_path = cli.full_path
nthreads = cli.nthreads
# INITIALISE TRADES WITH SUBROUTINE WITHIN TRADES_LIB -> PARAMETER NAMES, MINMAX, INTEGRATION ARGS, READ DATA ...
pytrades_lib.pytrades.initialize_trades(working_path, cli.sub_folder,
nthreads)
# RETRIEVE DATA AND VARIABLES FROM TRADES_LIB MODULE
#global n_bodies, n_planets, ndata, npar, nfit, dof, inv_dof
n_bodies = pytrades_lib.pytrades.n_bodies # NUMBER OF TOTAL BODIES OF THE SYSTEM
n_planets = n_bodies - 1 # NUMBER OF PLANETS IN THE SYSTEM
ndata = pytrades_lib.pytrades.ndata # TOTAL NUMBER OF DATA AVAILABLE
npar = pytrades_lib.pytrades.npar # NUMBER OF TOTAL PARAMATERS ~n_planets X 6
nfit = pytrades_lib.pytrades.nfit # NUMBER OF PARAMETERS TO FIT
nfree = pytrades_lib.pytrades.nfree # NUMBER OF FREE PARAMETERS (ie nrvset)
dof = pytrades_lib.pytrades.dof # NUMBER OF DEGREES OF FREEDOM = NDATA - NFIT
#inv_dof = np.float64(1.0 / dof)
inv_dof = pytrades_lib.pytrades.inv_dof
# READ THE NAMES OF THE PARAMETERS FROM THE TRADES_LIB AND CONVERT IT TO PYTHON STRINGS
#trades_names = anc.convert_fortran2python_strarray(pytrades_lib.pytrades.parameter_names,
#nfit, str_len=10
#)
##parameter_names = anc.trades_names_to_emcee(trades_names)
str_len = pytrades_lib.pytrades.str_len
temp_names = pytrades_lib.pytrades.get_parameter_names(nfit, str_len)
trades_names = anc.convert_fortran_charray2python_strararray(temp_names)
parameter_names = trades_names
fitting_parameters = pytrades_lib.pytrades.fitting_parameters # INITIAL PARAMETER SET (NEEDED ONLY TO HAVE THE PROPER ARRAY/VECTOR)
parameters_minmax = pytrades_lib.pytrades.parameters_minmax # PARAMETER BOUNDARIES
delta_parameters = np.abs(
parameters_minmax[:, 1] -
parameters_minmax[:, 0]) # DELTA BETWEEN MAX AND MIN OF BOUNDARIES
# RADIAL VELOCITIES SET
n_rv = pytrades_lib.pytrades.nrv
n_set_rv = pytrades_lib.pytrades.nrvset
# TRANSITS SET
n_t0 = pytrades_lib.pytrades.nt0
n_t0_sum = pytrades_lib.pytrades.ntts
n_set_t0 = 0
for i in range(0, n_bodies - 1):
if (n_t0[i] > 0): n_set_t0 += 1
# compute global constant for the loglhd
global ln_err_const
#try:
#e_RVo = np.asarray(pytrades_lib.pytrades.ervobs[:], dtype=np.float64) # fortran variable RV in python will be rv!!!
#except:
#e_RVo = np.asarray([0.], dtype=np.float64)
#try:
#e_T0o = np.asarray(pytrades_lib.pytrades.et0obs[:,:], dtype=np.float64).reshape((-1))
#except:
#e_T0o = np.asarray([0.], dtype=np.float64)
#ln_err_const = anc.compute_ln_err_const(ndata, dof, e_RVo, e_T0o, cli.ln_flag)
ln_err_const = pytrades_lib.pytrades.ln_err_const
# SET EMCEE PARAMETERS:
nwalkers, nruns, nsave, npost = get_emcee_arguments(cli, nfit)
# INITIALISE SCRIPT FOLDER/LOG FILE
working_folder, run_log, of_run = init_folder(working_path, cli.sub_folder)
anc.print_both('', of_run)
anc.print_both(' ======== ', of_run)
anc.print_both(' pyTRADES', of_run)
anc.print_both(' ======== ', of_run)
anc.print_both('', of_run)
anc.print_both(' WORKING PATH = %s' % (working_path), of_run)
anc.print_both(' NUMBER OF THREADS = %d' % (nthreads), of_run)
anc.print_both(
' dof = ndata(%d) - nfit(%d) - nfree(%d) = %d' %
(ndata, nfit, nfree, dof), of_run)
anc.print_both(' Total N_RV = %d for %d set(s)' % (n_rv, n_set_rv), of_run)
anc.print_both(
' Total N_T0 = %d for %d out of %d planet(s)' %
(n_t0_sum, n_set_t0, n_planets), of_run)
anc.print_both(' %s = %.7f' % ('log constant error = ', ln_err_const),
of_run)
anc.print_both(
' %s = %.7f' % ('IN FORTRAN log constant error = ',
pytrades_lib.pytrades.ln_err_const), of_run)
# INITIALISE PSO ARGUMENTS FROM pso.opt FILE
pytrades_lib.pytrades.init_pso(1, working_path) # read PSO options
# PSO VARIABLES
np_pso = pytrades_lib.pytrades.np_pso
nit_pso = pytrades_lib.pytrades.nit_pso
n_global = pytrades_lib.pytrades.n_global
#n_global = 1
anc.print_both(
' PSO n_global = %d npop = %d ngen = %d' % (n_global, np_pso, nit_pso),
of_run)
# RUN PSO+EMCEE n_global TIMES
for iter_global in range(0, n_global):
threads_pool = emcee.interruptible_pool.InterruptiblePool(1)
# CREATES PROPER WORKING PATH AND NAME
i_global = iter_global + 1
pso_path = os.path.join(
os.path.join(working_folder, '%04d_pso2emcee' % (i_global)), '')
pytrades_lib.pytrades.path_change(pso_path)
anc.print_both(
'\n\n GLOBAL RUN %04d INTO PATH: %s\n' % (i_global, pso_path),
of_run)
if (cli.pso_type == 'run'):
# RUN PSO
anc.print_both(' RUN PSO', of_run)
pso_start = time.time()
if (not os.path.exists(pso_path)): os.makedirs(pso_path)
# copy files
anc.copy_simulation_files(working_path, pso_path)
# CALL RUN_PSO SUBROUTINE FROM TRADES_LIB: RUNS PSO AND COMPUTES THE BEST SOLUTION, SAVING ALL THE POPULATION EVOLUTION
pso_parameters = fitting_parameters.copy()
pso_fitness = 0.
pso_parameters, pso_fitness = pytrades_lib.pytrades.pyrun_pso(
nfit, i_global)
anc.print_both(' completed run_pso', of_run)
pso_best_evolution = np.asarray(
pytrades_lib.pytrades.pso_best_evolution[...],
dtype=np.float64)
anc.print_both(' pso_best_evolution retrieved', of_run)
anc.print_both(' last pso_best_evolution', of_run)
last_pso_parameters = np.asarray(pso_best_evolution[:nfit, -1],
dtype=np.float64)
last_pso_fitness = pso_best_evolution[-1, -1].astype(np.float64)
anc.print_both(' fitness = %.f' % (last_pso_fitness), of_run)
# SAVE PSO SIMULATION IN pso_run.hdf5 FILE
print ' Creating pso hdf5 file: %s' % (os.path.join(
pso_path, 'pso_run.hdf5'))
pso_hdf5 = h5py.File(os.path.join(pso_path, 'pso_run.hdf5'), 'w')
pso_hdf5.create_dataset('population',
data=pytrades_lib.pytrades.population,
dtype=np.float64)
pso_hdf5.create_dataset(
'population_fitness',
data=pytrades_lib.pytrades.population_fitness,
dtype=np.float64)
pso_hdf5.create_dataset('pso_parameters',
data=pso_parameters,
dtype=np.float64)
pso_hdf5.create_dataset('pso_fitness',
data=np.array(pso_fitness),
dtype=np.float64)
pso_hdf5.create_dataset('pso_best_evolution',
data=pso_best_evolution,
dtype=np.float64)
pso_hdf5.create_dataset('parameters_minmax',
data=parameters_minmax,
dtype=np.float64)
pso_hdf5.create_dataset('parameter_names',
data=parameter_names,
dtype='S10')
pso_hdf5['population'].attrs['npop'] = np_pso
pso_hdf5['population'].attrs['niter'] = nit_pso
pso_hdf5['population'].attrs['iter_global'] = iter_global + 1
pso_hdf5['population'].attrs['nfit'] = nfit
pso_hdf5.close()
population = np.asarray(pytrades_lib.pytrades.population,
dtype=np.float64)
population_fitness = np.asarray(
pytrades_lib.pytrades.population_fitness, dtype=np.float64)
anc.print_both(' ', of_run)
fitness_iter, lgllhd_iter, check_iter = pytrades_lib.pytrades.write_summary_files(
i_global, pso_parameters)
elapsed = time.time() - pso_start
elapsed_d, elapsed_h, elapsed_m, elapsed_s = anc.computation_time(
elapsed)
anc.print_both(' ', of_run)
anc.print_both(
' PSO FINISHED in %2d day %02d hour %02d min %.2f sec - bye bye'
% (int(elapsed_d), int(elapsed_h), int(elapsed_m), elapsed_s),
of_run)
#p0, pso_fitness_p0 = pso_to_emcee(nfit, nwalkers, population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution)
p0 = compute_initial_walkers(nfit, nwalkers, pso_parameters,
parameters_minmax, parameter_names,
delta_sigma, of_run)
elif (cli.pso_type == 'exists'):
# READ PREVIOUS PSO_RUN.HDF5 FILE AND INITIALISE POPULATION FOR EMCEE
anc.print_both(
' READ PREVIOUS PSO_RUN.HDF5 FILE AND INITIALISE POPULATION FOR EMCEE',
of_run)
population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution, pso_parameters_minmax, pso_parameter_names, pop_shape = get_pso_data(
os.path.join(pso_path, 'pso_run.hdf5'))
fitness_iter, lgllhd_iter, check_iter = pytrades_lib.pytrades.write_summary_files(
i_global, pso_parameters)
anc.print_both(
' read pso_run.hdf5 file with best pso_fitness = %.7f' %
(pso_fitness), of_run)
#p0, pso_fitness_p0 = pso_to_emcee(nfit, nwalkers, population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution)
p0 = compute_initial_walkers(nfit, nwalkers, pso_parameters,
parameters_minmax, parameter_names,
delta_sigma, of_run)
elif (cli.pso_type == 'skip'):
# DO NOT RUN PSO, ONLY EMCEE
anc.print_both(' DO NOT RUN PSO, ONLY EMCEE', of_run)
#p0 = [parameters_minmax[:,0] + np.random.random(nfit)*delta_parameters for i in range(0, nwalkers)]
p0 = compute_initial_walkers(nfit, nwalkers, fitting_parameters,
parameters_minmax, parameter_names,
delta_sigma, of_run)
anc.print_both(
' emcee chain: nwalkers = %d nruns = %d' % (nwalkers, nruns),
of_run)
anc.print_both(' sampler ... ', of_run)
# old version with threads
#sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob, threads=nthreads)
#sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob_sq, threads=nthreads, args=[parameter_names])
# close the pool of threads
threads_pool.close()
threads_pool.terminate()
threads_pool.join()
threads_pool = emcee.interruptible_pool.InterruptiblePool(nthreads)
sampler = emcee.EnsembleSampler(nwalkers,
nfit,
lnprob,
pool=threads_pool)
anc.print_both(' ready to go', of_run)
anc.print_both(' with nsave = %r' % (nsave), of_run)
sys.stdout.flush()
#sys.exit()
if (nsave != False):
# save temporary sampling during emcee every nruns*10%
#if(os.path.exists(os.path.join(pso_path, 'emcee_temp.hdf5')) and os.path.isfile(os.path.join(pso_path, 'emcee_temp.hdf5'))):
#os.remove(os.path.join(pso_path, 'emcee_temp.hdf5'))
if (os.path.exists(os.path.join(pso_path, 'emcee_summary.hdf5'))
and os.path.isfile(
os.path.join(pso_path, 'emcee_summary.hdf5'))):
os.remove(os.path.join(pso_path, 'emcee_summary.hdf5'))
f_hdf5 = h5py.File(os.path.join(pso_path, 'emcee_summary.hdf5'),
'a')
f_hdf5.create_dataset('parameter_names',
data=parameter_names,
dtype='S10')
f_hdf5.create_dataset('boundaries',
data=parameters_minmax,
dtype=np.float64)
temp_dset = f_hdf5.create_dataset('chains',
(nwalkers, nruns, nfit),
dtype=np.float64)
f_hdf5['chains'].attrs['nwalkers'] = nwalkers
f_hdf5['chains'].attrs['nruns'] = nruns
f_hdf5['chains'].attrs['nfit'] = nfit
f_hdf5['chains'].attrs['nfree'] = nfree
temp_lnprob = f_hdf5.create_dataset('lnprobability',
(nwalkers, nruns),
dtype=np.float64)
temp_lnprob.attrs['ln_err_const'] = ln_err_const
temp_acceptance = f_hdf5.create_dataset('acceptance_fraction',
data=np.zeros((nfit)),
dtype=np.float64)
temp_acor = f_hdf5.create_dataset('autocor_time',
data=np.zeros((nfit)),
dtype=np.float64)
f_hdf5.close()
pos = p0
niter_save = int(nruns / nsave)
state = None
anc.print_both(' Running emcee with temporary saving', of_run)
sys.stdout.flush()
for i in range(0, niter_save):
anc.print_both('', of_run)
anc.print_both(' iter: %6d ' % (i + 1), of_run)
aaa = i * nsave
bbb = aaa + nsave
pos, prob, state = sampler.run_mcmc(pos,
N=nsave,
rstate0=state)
anc.print_both('completed %d steps of %d' % (bbb, nruns),
of_run)
f_hdf5 = h5py.File(
os.path.join(pso_path, 'emcee_summary.hdf5'), 'a')
temp_dset = f_hdf5['chains'] #[:,:,:]
temp_dset[:, aaa:bbb, :] = sampler.chain[:, aaa:bbb, :]
temp_dset.attrs['completed_steps'] = bbb
temp_lnprob = f_hdf5['lnprobability'] #[:,:]
temp_lnprob[:, aaa:bbb] = sampler.lnprobability[:, aaa:bbb]
shape_lnprob = sampler.lnprobability.shape
acceptance_fraction = sampler.acceptance_fraction
temp_acceptance = f_hdf5['acceptance_fraction']
temp_acceptance = acceptance_fraction
#f_hdf5.create_dataset('acceptance_fraction', data=acceptance_fraction, dtype=np.float64)
mean_acceptance_fraction = np.mean(acceptance_fraction)
#temp_chains_T = np.zeros((bbb, nwalkers, nfit))
#for ifit in range(0,nfit):
#temp_chains_T[:,:,ifit] = sampler.chain[:, :bbb, ifit].T
#acor_time = anc.compute_autocor_time(temp_chains_T, walkers_transposed=True)
acor_time = anc.compute_acor_time(sampler, steps_done=bbb)
temp_acor = f_hdf5['autocor_time']
temp_acor[...] = acor_time
#f_hdf5.create_dataset('autocor_time', data=np.array(acor_temp, dtype=np.float64), dtype=np.float64)
#f_hdf5.create_dataset('autocor_time', data=np.array(sampler.acor, dtype=np.float64), dtype=np.float64) # not working
#print 'aaa = %6d bbb = %6d -> sampler.lnprobability.shape = (%6d , %6d)' %(aaa, bbb, shape_lnprob[0], shape_lnprob[1])
f_hdf5.close()
sys.stdout.flush()
anc.print_both('', of_run)
anc.print_both(
'...done with saving temporary total shape = %s' %
(str(np.shape(sampler.chain))), of_run)
anc.print_both('', of_run)
sys.stdout.flush()
else:
# GOOD COMPLETE SINGLE RUNNING OF EMCEE, WITHOUT REMOVING THE BURN-IN
anc.print_both(' Running full emcee ...', of_run)
sys.stdout.flush()
sampler.run_mcmc(p0, nruns)
anc.print_both('done', of_run)
anc.print_both('', of_run)
sys.stdout.flush()
flatchains = sampler.chain[:, :, :].reshape(
(nwalkers * nruns, nfit)) # full chain values
acceptance_fraction = sampler.acceptance_fraction
mean_acceptance_fraction = np.mean(acceptance_fraction)
#autocor_time = sampler.acor
#temp_chains_T = np.zeros((bbb, nwalkers, nfit))
#for ifit in range(0,nfit):
#temp_chains_T[:,:,ifit] = sampler.chain[:, :, ifit].T
#acor_time = anc.compute_autocor_time(temp_chains_T, walkers_transposed=True)
acor_time = anc.compute_acor_time(sampler)
lnprobability = sampler.lnprobability
# save chains with original shape as hdf5 file
f_hdf5 = h5py.File(os.path.join(pso_path, 'emcee_summary.hdf5'),
'w')
f_hdf5.create_dataset('chains',
data=sampler.chain,
dtype=np.float64)
f_hdf5['chains'].attrs['nwalkers'] = nwalkers
f_hdf5['chains'].attrs['nruns'] = nruns
f_hdf5['chains'].attrs['nfit'] = nfit
f_hdf5['chains'].attrs['nfree'] = nfree
f_hdf5['chains'].attrs['completed_steps'] = nruns
f_hdf5.create_dataset('parameter_names',
data=parameter_names,
dtype='S10')
f_hdf5.create_dataset('boundaries',
data=parameters_minmax,
dtype=np.float64)
f_hdf5.create_dataset('acceptance_fraction',
data=acceptance_fraction,
dtype=np.float64)
f_hdf5.create_dataset('autocor_time',
data=acor_time,
dtype=np.float64)
f_hdf5.create_dataset('lnprobability',
data=lnprobability,
dtype=np.float64)
f_hdf5['lnprobability'].attrs['ln_err_const'] = ln_err_const
f_hdf5.close()
anc.print_both(
" Mean_acceptance_fraction should be between [0.25-0.5] = %.6f" %
(mean_acceptance_fraction), of_run)
anc.print_both('', of_run)
# close the pool of threads
threads_pool.close()
threads_pool.terminate()
threads_pool.join()
anc.print_both('COMPLETED EMCEE', of_run)
elapsed = time.time() - start
elapsed_d, elapsed_h, elapsed_m, elapsed_s = anc.computation_time(
elapsed)
anc.print_both('', of_run)
anc.print_both(
' pyTRADES: EMCEE FINISHED in %2d day %02d hour %02d min %.2f sec - bye bye'
% (int(elapsed_d), int(elapsed_h), int(elapsed_m), elapsed_s),
of_run)
anc.print_both('', of_run)
of_run.close()
pytrades_lib.pytrades.deallocate_variables()
return
def main():
# MAIN -- TRADES + EMCEE
# READ COMMAND LINE ARGUMENTS
cli = get_args()
# STARTING TIME
start = time.time()
# RENAME
working_path = cli.full_path
nthreads = cli.nthreads
# INITIALISE TRADES WITH SUBROUTINE WITHIN TRADES_LIB -> PARAMETER NAMES, MINMAX, INTEGRATION ARGS, READ DATA ...
pytrades_lib.pytrades.initialize_trades(working_path, cli.sub_folder, nthreads)
# RETRIEVE DATA AND VARIABLES FROM TRADES_LIB MODULE
#global n_bodies, n_planets, ndata, npar, nfit, dof, inv_dof
n_bodies = pytrades_lib.pytrades.n_bodies # NUMBER OF TOTAL BODIES OF THE SYSTEM
n_planets = n_bodies - 1 # NUMBER OF PLANETS IN THE SYSTEM
ndata = pytrades_lib.pytrades.ndata # TOTAL NUMBER OF DATA AVAILABLE
npar = pytrades_lib.pytrades.npar # NUMBER OF TOTAL PARAMATERS ~n_planets X 6
nfit = pytrades_lib.pytrades.nfit # NUMBER OF PARAMETERS TO FIT
nfree = pytrades_lib.pytrades.nfree # NUMBER OF FREE PARAMETERS (ie nrvset)
dof = pytrades_lib.pytrades.dof # NUMBER OF DEGREES OF FREEDOM = NDATA - NFIT
#inv_dof = np.float64(1.0 / dof)
inv_dof = pytrades_lib.pytrades.inv_dof
# READ THE NAMES OF THE PARAMETERS FROM THE TRADES_LIB AND CONVERT IT TO PYTHON STRINGS
#trades_names = anc.convert_fortran2python_strarray(pytrades_lib.pytrades.parameter_names,
#nfit, str_len=10
#)
##parameter_names = anc.trades_names_to_emcee(trades_names)
str_len = pytrades_lib.pytrades.str_len
temp_names = pytrades_lib.pytrades.get_parameter_names(nfit,str_len)
trades_names = anc.convert_fortran_charray2python_strararray(temp_names)
parameter_names = trades_names
fitting_parameters = pytrades_lib.pytrades.fitting_parameters # INITIAL PARAMETER SET (NEEDED ONLY TO HAVE THE PROPER ARRAY/VECTOR)
parameters_minmax = pytrades_lib.pytrades.parameters_minmax # PARAMETER BOUNDARIES
delta_parameters = np.abs(parameters_minmax[:,1] - parameters_minmax[:,0]) # DELTA BETWEEN MAX AND MIN OF BOUNDARIES
# RADIAL VELOCITIES SET
n_rv = pytrades_lib.pytrades.nrv
n_set_rv = pytrades_lib.pytrades.nrvset
# TRANSITS SET
n_t0 = pytrades_lib.pytrades.nt0
n_t0_sum = pytrades_lib.pytrades.ntts
n_set_t0 = 0
for i in range(0, n_bodies-1):
if (n_t0[i] > 0): n_set_t0 += 1
# compute global constant for the loglhd
global ln_err_const
#try:
#e_RVo = np.asarray(pytrades_lib.pytrades.ervobs[:], dtype=np.float64) # fortran variable RV in python will be rv!!!
#except:
#e_RVo = np.asarray([0.], dtype=np.float64)
#try:
#e_T0o = np.asarray(pytrades_lib.pytrades.et0obs[:,:], dtype=np.float64).reshape((-1))
#except:
#e_T0o = np.asarray([0.], dtype=np.float64)
#ln_err_const = anc.compute_ln_err_const(ndata, dof, e_RVo, e_T0o, cli.ln_flag)
ln_err_const = pytrades_lib.pytrades.ln_err_const
# SET EMCEE PARAMETERS:
nwalkers, nruns, nsave, npost = get_emcee_arguments(cli,nfit)
# INITIALISE SCRIPT FOLDER/LOG FILE
working_folder, run_log, of_run = init_folder(working_path, cli.sub_folder)
anc.print_both('',of_run)
anc.print_both(' ======== ',of_run)
anc.print_both(' pyTRADES' ,of_run)
anc.print_both(' ======== ',of_run)
anc.print_both('',of_run)
anc.print_both(' WORKING PATH = %s' %(working_path),of_run)
anc.print_both(' NUMBER OF THREADS = %d' %(nthreads),of_run)
anc.print_both(' dof = ndata(%d) - nfit(%d) - nfree(%d) = %d' %(ndata, nfit, nfree, dof),of_run)
anc.print_both(' Total N_RV = %d for %d set(s)' %(n_rv, n_set_rv),of_run)
anc.print_both(' Total N_T0 = %d for %d out of %d planet(s)' %(n_t0_sum, n_set_t0, n_planets),of_run)
anc.print_both(' %s = %.7f' %('log constant error = ', ln_err_const),of_run)
anc.print_both(' %s = %.7f' %('IN FORTRAN log constant error = ', pytrades_lib.pytrades.ln_err_const),of_run)
# INITIALISE PSO ARGUMENTS FROM pso.opt FILE
pytrades_lib.pytrades.init_pso(1,working_path) # read PSO options
# PSO VARIABLES
np_pso = pytrades_lib.pytrades.np_pso
nit_pso = pytrades_lib.pytrades.nit_pso
n_global = pytrades_lib.pytrades.n_global
#n_global = 1
anc.print_both(' PSO n_global = %d npop = %d ngen = %d' %(n_global, np_pso, nit_pso), of_run)
# RUN PSO+EMCEE n_global TIMES
for iter_global in range(0,n_global):
threads_pool = emcee.interruptible_pool.InterruptiblePool(1)
# CREATES PROPER WORKING PATH AND NAME
i_global = iter_global + 1
pso_path = os.path.join(os.path.join(working_folder, '%04d_pso2emcee' %(i_global)), '')
pytrades_lib.pytrades.path_change(pso_path)
anc.print_both('\n\n GLOBAL RUN %04d INTO PATH: %s\n' %(i_global, pso_path), of_run)
if (cli.pso_type == 'run'):
# RUN PSO
anc.print_both(' RUN PSO', of_run)
pso_start = time.time()
if(not os.path.exists(pso_path)): os.makedirs(pso_path)
# copy files
anc.copy_simulation_files(working_path, pso_path)
# CALL RUN_PSO SUBROUTINE FROM TRADES_LIB: RUNS PSO AND COMPUTES THE BEST SOLUTION, SAVING ALL THE POPULATION EVOLUTION
pso_parameters = fitting_parameters.copy()
pso_fitness = 0.
pso_parameters, pso_fitness = pytrades_lib.pytrades.pyrun_pso(nfit,i_global)
anc.print_both(' completed run_pso', of_run)
pso_best_evolution = np.asarray(pytrades_lib.pytrades.pso_best_evolution[...], dtype=np.float64)
anc.print_both(' pso_best_evolution retrieved', of_run)
anc.print_both(' last pso_best_evolution', of_run)
last_pso_parameters = np.asarray(pso_best_evolution[:nfit,-1],dtype=np.float64)
last_pso_fitness = pso_best_evolution[-1,-1].astype(np.float64)
anc.print_both(' fitness = %.f' %(last_pso_fitness), of_run)
# SAVE PSO SIMULATION IN pso_run.hdf5 FILE
print ' Creating pso hdf5 file: %s' %(os.path.join(pso_path, 'pso_run.hdf5'))
pso_hdf5 = h5py.File(os.path.join(pso_path, 'pso_run.hdf5'), 'w')
pso_hdf5.create_dataset('population', data=pytrades_lib.pytrades.population, dtype=np.float64)
pso_hdf5.create_dataset('population_fitness', data=pytrades_lib.pytrades.population_fitness, dtype=np.float64)
pso_hdf5.create_dataset('pso_parameters', data=pso_parameters, dtype=np.float64)
pso_hdf5.create_dataset('pso_fitness', data=np.array(pso_fitness), dtype=np.float64)
pso_hdf5.create_dataset('pso_best_evolution', data=pso_best_evolution, dtype=np.float64)
pso_hdf5.create_dataset('parameters_minmax', data=parameters_minmax, dtype=np.float64)
pso_hdf5.create_dataset('parameter_names', data=parameter_names, dtype='S10')
pso_hdf5['population'].attrs['npop'] = np_pso
pso_hdf5['population'].attrs['niter'] = nit_pso
pso_hdf5['population'].attrs['iter_global'] = iter_global+1
pso_hdf5['population'].attrs['nfit'] = nfit
pso_hdf5.close()
population = np.asarray(pytrades_lib.pytrades.population, dtype=np.float64)
population_fitness = np.asarray(pytrades_lib.pytrades.population_fitness, dtype=np.float64)
anc.print_both(' ', of_run)
fitness_iter, lgllhd_iter, check_iter = pytrades_lib.pytrades.write_summary_files(i_global, pso_parameters)
elapsed = time.time() - pso_start
elapsed_d, elapsed_h, elapsed_m, elapsed_s = anc.computation_time(elapsed)
anc.print_both(' ', of_run)
anc.print_both(' PSO FINISHED in %2d day %02d hour %02d min %.2f sec - bye bye' %(int(elapsed_d), int(elapsed_h), int(elapsed_m), elapsed_s), of_run)
#p0, pso_fitness_p0 = pso_to_emcee(nfit, nwalkers, population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution)
p0 = compute_initial_walkers(nfit, nwalkers, pso_parameters, parameters_minmax, parameter_names, delta_sigma, of_run)
elif (cli.pso_type == 'exists'):
# READ PREVIOUS PSO_RUN.HDF5 FILE AND INITIALISE POPULATION FOR EMCEE
anc.print_both(' READ PREVIOUS PSO_RUN.HDF5 FILE AND INITIALISE POPULATION FOR EMCEE', of_run)
population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution, pso_parameters_minmax, pso_parameter_names, pop_shape = get_pso_data(os.path.join(pso_path, 'pso_run.hdf5'))
fitness_iter, lgllhd_iter, check_iter = pytrades_lib.pytrades.write_summary_files(i_global, pso_parameters)
anc.print_both(' read pso_run.hdf5 file with best pso_fitness = %.7f' %(pso_fitness), of_run)
#p0, pso_fitness_p0 = pso_to_emcee(nfit, nwalkers, population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution)
p0 = compute_initial_walkers(nfit, nwalkers, pso_parameters, parameters_minmax, parameter_names, delta_sigma, of_run)
elif (cli.pso_type == 'skip'):
# DO NOT RUN PSO, ONLY EMCEE
anc.print_both(' DO NOT RUN PSO, ONLY EMCEE', of_run)
#p0 = [parameters_minmax[:,0] + np.random.random(nfit)*delta_parameters for i in range(0, nwalkers)]
p0 = compute_initial_walkers(nfit, nwalkers, fitting_parameters, parameters_minmax, parameter_names, delta_sigma, of_run)
anc.print_both(' emcee chain: nwalkers = %d nruns = %d' %(nwalkers, nruns), of_run)
anc.print_both(' sampler ... ',of_run)
# old version with threads
#sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob, threads=nthreads)
#sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob_sq, threads=nthreads, args=[parameter_names])
# close the pool of threads
threads_pool.close()
threads_pool.terminate()
threads_pool.join()
threads_pool = emcee.interruptible_pool.InterruptiblePool(nthreads)
sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob, pool=threads_pool)
anc.print_both(' ready to go', of_run)
anc.print_both(' with nsave = %r' %(nsave), of_run)
sys.stdout.flush()
#sys.exit()
if (nsave != False):
# save temporary sampling during emcee every nruns*10%
#if(os.path.exists(os.path.join(pso_path, 'emcee_temp.hdf5')) and os.path.isfile(os.path.join(pso_path, 'emcee_temp.hdf5'))):
#os.remove(os.path.join(pso_path, 'emcee_temp.hdf5'))
if(os.path.exists(os.path.join(pso_path, 'emcee_summary.hdf5')) and os.path.isfile(os.path.join(pso_path, 'emcee_summary.hdf5'))):
os.remove(os.path.join(pso_path, 'emcee_summary.hdf5'))
f_hdf5 = h5py.File(os.path.join(pso_path, 'emcee_summary.hdf5'), 'a')
f_hdf5.create_dataset('parameter_names', data=parameter_names, dtype='S10')
f_hdf5.create_dataset('boundaries', data=parameters_minmax, dtype=np.float64)
temp_dset = f_hdf5.create_dataset('chains', (nwalkers, nruns, nfit), dtype=np.float64)
f_hdf5['chains'].attrs['nwalkers'] = nwalkers
f_hdf5['chains'].attrs['nruns'] = nruns
f_hdf5['chains'].attrs['nfit'] = nfit
f_hdf5['chains'].attrs['nfree'] = nfree
temp_lnprob = f_hdf5.create_dataset('lnprobability', (nwalkers, nruns), dtype=np.float64)
temp_lnprob.attrs['ln_err_const'] = ln_err_const
temp_acceptance = f_hdf5.create_dataset('acceptance_fraction', data=np.zeros((nfit)), dtype=np.float64)
temp_acor = f_hdf5.create_dataset('autocor_time', data=np.zeros((nfit)), dtype=np.float64)
f_hdf5.close()
pos = p0
niter_save = int(nruns/nsave)
state=None
anc.print_both(' Running emcee with temporary saving', of_run)
sys.stdout.flush()
for i in range(0, niter_save):
anc.print_both('', of_run)
anc.print_both(' iter: %6d ' %(i+1), of_run)
aaa = i*nsave
bbb = aaa+nsave
pos, prob, state = sampler.run_mcmc(pos, N=nsave, rstate0=state)
anc.print_both('completed %d steps of %d' %(bbb, nruns), of_run)
f_hdf5 = h5py.File(os.path.join(pso_path, 'emcee_summary.hdf5'), 'a')
temp_dset = f_hdf5['chains'] #[:,:,:]
temp_dset[:,aaa:bbb,:] = sampler.chain[:, aaa:bbb, :]
temp_dset.attrs['completed_steps'] = bbb
temp_lnprob = f_hdf5['lnprobability'] #[:,:]
temp_lnprob[:, aaa:bbb] = sampler.lnprobability[:, aaa:bbb]
shape_lnprob = sampler.lnprobability.shape
acceptance_fraction = sampler.acceptance_fraction
temp_acceptance = f_hdf5['acceptance_fraction']
temp_acceptance = acceptance_fraction
#f_hdf5.create_dataset('acceptance_fraction', data=acceptance_fraction, dtype=np.float64)
mean_acceptance_fraction = np.mean(acceptance_fraction)
#temp_chains_T = np.zeros((bbb, nwalkers, nfit))
#for ifit in range(0,nfit):
#temp_chains_T[:,:,ifit] = sampler.chain[:, :bbb, ifit].T
#acor_time = anc.compute_autocor_time(temp_chains_T, walkers_transposed=True)
acor_time = anc.compute_acor_time(sampler, steps_done=bbb)
temp_acor = f_hdf5['autocor_time']
temp_acor[...] = acor_time
#f_hdf5.create_dataset('autocor_time', data=np.array(acor_temp, dtype=np.float64), dtype=np.float64)
#f_hdf5.create_dataset('autocor_time', data=np.array(sampler.acor, dtype=np.float64), dtype=np.float64) # not working
#print 'aaa = %6d bbb = %6d -> sampler.lnprobability.shape = (%6d , %6d)' %(aaa, bbb, shape_lnprob[0], shape_lnprob[1])
f_hdf5.close()
sys.stdout.flush()
anc.print_both('', of_run)
anc.print_both('...done with saving temporary total shape = %s' %(str(np.shape(sampler.chain))), of_run)
anc.print_both('', of_run)
sys.stdout.flush()
else:
# GOOD COMPLETE SINGLE RUNNING OF EMCEE, WITHOUT REMOVING THE BURN-IN
anc.print_both(' Running full emcee ...', of_run)
sys.stdout.flush()
sampler.run_mcmc(p0, nruns)
anc.print_both('done', of_run)
anc.print_both('', of_run)
sys.stdout.flush()
flatchains = sampler.chain[:, :, :].reshape((nwalkers*nruns, nfit)) # full chain values
acceptance_fraction = sampler.acceptance_fraction
mean_acceptance_fraction = np.mean(acceptance_fraction)
#autocor_time = sampler.acor
#temp_chains_T = np.zeros((bbb, nwalkers, nfit))
#for ifit in range(0,nfit):
#temp_chains_T[:,:,ifit] = sampler.chain[:, :, ifit].T
#acor_time = anc.compute_autocor_time(temp_chains_T, walkers_transposed=True)
acor_time = anc.compute_acor_time(sampler)
lnprobability = sampler.lnprobability
# save chains with original shape as hdf5 file
f_hdf5 = h5py.File(os.path.join(pso_path, 'emcee_summary.hdf5'), 'w')
f_hdf5.create_dataset('chains', data=sampler.chain, dtype=np.float64)
f_hdf5['chains'].attrs['nwalkers'] = nwalkers
f_hdf5['chains'].attrs['nruns'] = nruns
f_hdf5['chains'].attrs['nfit'] = nfit
f_hdf5['chains'].attrs['nfree'] = nfree
f_hdf5['chains'].attrs['completed_steps'] = nruns
f_hdf5.create_dataset('parameter_names', data=parameter_names, dtype='S10')
f_hdf5.create_dataset('boundaries', data=parameters_minmax, dtype=np.float64)
f_hdf5.create_dataset('acceptance_fraction', data=acceptance_fraction, dtype=np.float64)
f_hdf5.create_dataset('autocor_time', data=acor_time, dtype=np.float64)
f_hdf5.create_dataset('lnprobability', data=lnprobability, dtype=np.float64)
f_hdf5['lnprobability'].attrs['ln_err_const'] = ln_err_const
f_hdf5.close()
anc.print_both(" Mean_acceptance_fraction should be between [0.25-0.5] = %.6f" %(mean_acceptance_fraction), of_run)
anc.print_both('', of_run)
# close the pool of threads
threads_pool.close()
threads_pool.terminate()
threads_pool.join()
anc.print_both('COMPLETED EMCEE', of_run)
elapsed = time.time() - start
elapsed_d, elapsed_h, elapsed_m, elapsed_s = anc.computation_time(elapsed)
anc.print_both('', of_run)
anc.print_both(' pyTRADES: EMCEE FINISHED in %2d day %02d hour %02d min %.2f sec - bye bye' %(int(elapsed_d), int(elapsed_h), int(elapsed_m), elapsed_s), of_run)
anc.print_both('', of_run)
of_run.close()
pytrades_lib.pytrades.deallocate_variables()
return