def init_folder(working_path, sub_folder): working_folder = os.path.join(working_path, sub_folder) if (not os.path.isdir(working_folder)): os.makedirs(working_folder) #arg_file = os.path.join(working_path, 'arg.in') #shutil.copy(arg_file, os.path.join(working_folder,'')) #bodies_file = os.path.join(working_path, 'bodies.lst') #shutil.copy(bodies_file, os.path.join(working_folder,'')) #obd = open(bodies_file, 'r') #for line in obd.readlines(): #shutil.copy(os.path.join(working_path, line.strip().split()[0]), os.path.join(working_folder,'')) #obd.close() #t0files = glob.glob(os.path.join(working_path,'NB*_observations.dat')) #for t0f in t0files: #shutil.copy(t0f, os.path.join(working_folder,'')) #if(os.path.exists(os.path.join(working_path,'obsRV.dat'))): #shutil.copy(os.path.join(working_path,'obsRV.dat'), os.path.join(working_folder,'')) # copy files anc.copy_simulation_files(working_path, working_folder) run_log = os.path.join(working_folder, "trades_run.log") of_run = open(run_log, 'w') anc.print_both("# pyTRADES LOG FILE", of_run) anc.print_both("# working_path = %s" %(working_path), of_run) anc.print_both("# working_folder = %s" %(working_folder), of_run) anc.print_both("# run_log = %s" %(run_log), of_run) return working_folder, run_log, of_run
def init_folder(working_path, sub_folder): working_folder = os.path.join(working_path, sub_folder) if (not os.path.isdir(working_folder)): os.makedirs(working_folder) # copy files anc.copy_simulation_files(working_path, working_folder) run_log = os.path.join(working_folder, "trades_run.log") of_run = open(run_log, 'w') anc.print_both("# pyTRADES LOG FILE", of_run) anc.print_both("# working_path = %s" %(working_path), of_run) anc.print_both("# working_folder = %s" %(working_folder), of_run) anc.print_both("# run_log = %s" %(run_log), of_run) return working_folder, run_log, of_run
def init_folder(working_path, sub_folder): working_folder = os.path.join(working_path, sub_folder) if (not os.path.isdir(working_folder)): os.makedirs(working_folder) # copy files anc.copy_simulation_files(working_path, working_folder) run_log = os.path.join(working_folder, "trades_run.log") of_run = open(run_log, 'w') anc.print_both("# pyTRADES LOG FILE", of_run) anc.print_both("# working_path = %s" % (working_path), of_run) anc.print_both("# working_folder = %s" % (working_folder), of_run) anc.print_both("# run_log = %s" % (run_log), of_run) return working_folder, run_log, of_run
def main(): # MAIN -- TRADES + EMCEE # READ COMMAND LINE ARGUMENTS cli = get_args() # STARTING TIME start = time.time() # RENAME working_path = cli.full_path nthreads = cli.nthreads # INITIALISE TRADES WITH SUBROUTINE WITHIN TRADES_LIB -> PARAMETER NAMES, MINMAX, INTEGRATION ARGS, READ DATA ... pytrades_lib.pytrades.initialize_trades(working_path, cli.sub_folder, nthreads) # RETRIEVE DATA AND VARIABLES FROM TRADES_LIB MODULE #global n_bodies, n_planets, ndata, npar, nfit, dof, inv_dof n_bodies = pytrades_lib.pytrades.n_bodies # NUMBER OF TOTAL BODIES OF THE SYSTEM n_planets = n_bodies - 1 # NUMBER OF PLANETS IN THE SYSTEM ndata = pytrades_lib.pytrades.ndata # TOTAL NUMBER OF DATA AVAILABLE npar = pytrades_lib.pytrades.npar # NUMBER OF TOTAL PARAMATERS ~n_planets X 6 nfit = pytrades_lib.pytrades.nfit # NUMBER OF PARAMETERS TO FIT nfree = pytrades_lib.pytrades.nfree # NUMBER OF FREE PARAMETERS (ie nrvset) dof = pytrades_lib.pytrades.dof # NUMBER OF DEGREES OF FREEDOM = NDATA - NFIT #inv_dof = np.float64(1.0 / dof) inv_dof = pytrades_lib.pytrades.inv_dof # READ THE NAMES OF THE PARAMETERS FROM THE TRADES_LIB AND CONVERT IT TO PYTHON STRINGS #trades_names = anc.convert_fortran2python_strarray(pytrades_lib.pytrades.parameter_names, #nfit, str_len=10 #) ##parameter_names = anc.trades_names_to_emcee(trades_names) str_len = pytrades_lib.pytrades.str_len temp_names = pytrades_lib.pytrades.get_parameter_names(nfit, str_len) trades_names = anc.convert_fortran_charray2python_strararray(temp_names) parameter_names = trades_names fitting_parameters = pytrades_lib.pytrades.fitting_parameters # INITIAL PARAMETER SET (NEEDED ONLY TO HAVE THE PROPER ARRAY/VECTOR) parameters_minmax = pytrades_lib.pytrades.parameters_minmax # PARAMETER BOUNDARIES delta_parameters = np.abs( parameters_minmax[:, 1] - parameters_minmax[:, 0]) # DELTA BETWEEN MAX AND MIN OF BOUNDARIES # RADIAL VELOCITIES SET n_rv = pytrades_lib.pytrades.nrv n_set_rv = pytrades_lib.pytrades.nrvset # TRANSITS SET n_t0 = pytrades_lib.pytrades.nt0 n_t0_sum = pytrades_lib.pytrades.ntts n_set_t0 = 0 for i in range(0, n_bodies - 1): if (n_t0[i] > 0): n_set_t0 += 1 # compute global constant for the loglhd global ln_err_const #try: #e_RVo = np.asarray(pytrades_lib.pytrades.ervobs[:], dtype=np.float64) # fortran variable RV in python will be rv!!! #except: #e_RVo = np.asarray([0.], dtype=np.float64) #try: #e_T0o = np.asarray(pytrades_lib.pytrades.et0obs[:,:], dtype=np.float64).reshape((-1)) #except: #e_T0o = np.asarray([0.], dtype=np.float64) #ln_err_const = anc.compute_ln_err_const(ndata, dof, e_RVo, e_T0o, cli.ln_flag) ln_err_const = pytrades_lib.pytrades.ln_err_const # SET EMCEE PARAMETERS: nwalkers, nruns, nsave, npost = get_emcee_arguments(cli, nfit) # INITIALISE SCRIPT FOLDER/LOG FILE working_folder, run_log, of_run = init_folder(working_path, cli.sub_folder) anc.print_both('', of_run) anc.print_both(' ======== ', of_run) anc.print_both(' pyTRADES', of_run) anc.print_both(' ======== ', of_run) anc.print_both('', of_run) anc.print_both(' WORKING PATH = %s' % (working_path), of_run) anc.print_both(' NUMBER OF THREADS = %d' % (nthreads), of_run) anc.print_both( ' dof = ndata(%d) - nfit(%d) - nfree(%d) = %d' % (ndata, nfit, nfree, dof), of_run) anc.print_both(' Total N_RV = %d for %d set(s)' % (n_rv, n_set_rv), of_run) anc.print_both( ' Total N_T0 = %d for %d out of %d planet(s)' % (n_t0_sum, n_set_t0, n_planets), of_run) anc.print_both(' %s = %.7f' % ('log constant error = ', ln_err_const), of_run) anc.print_both( ' %s = %.7f' % ('IN FORTRAN log constant error = ', pytrades_lib.pytrades.ln_err_const), of_run) # INITIALISE PSO ARGUMENTS FROM pso.opt FILE pytrades_lib.pytrades.init_pso(1, working_path) # read PSO options # PSO VARIABLES np_pso = pytrades_lib.pytrades.np_pso nit_pso = pytrades_lib.pytrades.nit_pso n_global = pytrades_lib.pytrades.n_global #n_global = 1 anc.print_both( ' PSO n_global = %d npop = %d ngen = %d' % (n_global, np_pso, nit_pso), of_run) # RUN PSO+EMCEE n_global TIMES for iter_global in range(0, n_global): threads_pool = emcee.interruptible_pool.InterruptiblePool(1) # CREATES PROPER WORKING PATH AND NAME i_global = iter_global + 1 pso_path = os.path.join( os.path.join(working_folder, '%04d_pso2emcee' % (i_global)), '') pytrades_lib.pytrades.path_change(pso_path) anc.print_both( '\n\n GLOBAL RUN %04d INTO PATH: %s\n' % (i_global, pso_path), of_run) if (cli.pso_type == 'run'): # RUN PSO anc.print_both(' RUN PSO', of_run) pso_start = time.time() if (not os.path.exists(pso_path)): os.makedirs(pso_path) # copy files anc.copy_simulation_files(working_path, pso_path) # CALL RUN_PSO SUBROUTINE FROM TRADES_LIB: RUNS PSO AND COMPUTES THE BEST SOLUTION, SAVING ALL THE POPULATION EVOLUTION pso_parameters = fitting_parameters.copy() pso_fitness = 0. pso_parameters, pso_fitness = pytrades_lib.pytrades.pyrun_pso( nfit, i_global) anc.print_both(' completed run_pso', of_run) pso_best_evolution = np.asarray( pytrades_lib.pytrades.pso_best_evolution[...], dtype=np.float64) anc.print_both(' pso_best_evolution retrieved', of_run) anc.print_both(' last pso_best_evolution', of_run) last_pso_parameters = np.asarray(pso_best_evolution[:nfit, -1], dtype=np.float64) last_pso_fitness = pso_best_evolution[-1, -1].astype(np.float64) anc.print_both(' fitness = %.f' % (last_pso_fitness), of_run) # SAVE PSO SIMULATION IN pso_run.hdf5 FILE print ' Creating pso hdf5 file: %s' % (os.path.join( pso_path, 'pso_run.hdf5')) pso_hdf5 = h5py.File(os.path.join(pso_path, 'pso_run.hdf5'), 'w') pso_hdf5.create_dataset('population', data=pytrades_lib.pytrades.population, dtype=np.float64) pso_hdf5.create_dataset( 'population_fitness', data=pytrades_lib.pytrades.population_fitness, dtype=np.float64) pso_hdf5.create_dataset('pso_parameters', data=pso_parameters, dtype=np.float64) pso_hdf5.create_dataset('pso_fitness', data=np.array(pso_fitness), dtype=np.float64) pso_hdf5.create_dataset('pso_best_evolution', data=pso_best_evolution, dtype=np.float64) pso_hdf5.create_dataset('parameters_minmax', data=parameters_minmax, dtype=np.float64) pso_hdf5.create_dataset('parameter_names', data=parameter_names, dtype='S10') pso_hdf5['population'].attrs['npop'] = np_pso pso_hdf5['population'].attrs['niter'] = nit_pso pso_hdf5['population'].attrs['iter_global'] = iter_global + 1 pso_hdf5['population'].attrs['nfit'] = nfit pso_hdf5.close() population = np.asarray(pytrades_lib.pytrades.population, dtype=np.float64) population_fitness = np.asarray( pytrades_lib.pytrades.population_fitness, dtype=np.float64) anc.print_both(' ', of_run) fitness_iter, lgllhd_iter, check_iter = pytrades_lib.pytrades.write_summary_files( i_global, pso_parameters) elapsed = time.time() - pso_start elapsed_d, elapsed_h, elapsed_m, elapsed_s = anc.computation_time( elapsed) anc.print_both(' ', of_run) anc.print_both( ' PSO FINISHED in %2d day %02d hour %02d min %.2f sec - bye bye' % (int(elapsed_d), int(elapsed_h), int(elapsed_m), elapsed_s), of_run) #p0, pso_fitness_p0 = pso_to_emcee(nfit, nwalkers, population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution) p0 = compute_initial_walkers(nfit, nwalkers, pso_parameters, parameters_minmax, parameter_names, delta_sigma, of_run) elif (cli.pso_type == 'exists'): # READ PREVIOUS PSO_RUN.HDF5 FILE AND INITIALISE POPULATION FOR EMCEE anc.print_both( ' READ PREVIOUS PSO_RUN.HDF5 FILE AND INITIALISE POPULATION FOR EMCEE', of_run) population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution, pso_parameters_minmax, pso_parameter_names, pop_shape = get_pso_data( os.path.join(pso_path, 'pso_run.hdf5')) fitness_iter, lgllhd_iter, check_iter = pytrades_lib.pytrades.write_summary_files( i_global, pso_parameters) anc.print_both( ' read pso_run.hdf5 file with best pso_fitness = %.7f' % (pso_fitness), of_run) #p0, pso_fitness_p0 = pso_to_emcee(nfit, nwalkers, population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution) p0 = compute_initial_walkers(nfit, nwalkers, pso_parameters, parameters_minmax, parameter_names, delta_sigma, of_run) elif (cli.pso_type == 'skip'): # DO NOT RUN PSO, ONLY EMCEE anc.print_both(' DO NOT RUN PSO, ONLY EMCEE', of_run) #p0 = [parameters_minmax[:,0] + np.random.random(nfit)*delta_parameters for i in range(0, nwalkers)] p0 = compute_initial_walkers(nfit, nwalkers, fitting_parameters, parameters_minmax, parameter_names, delta_sigma, of_run) anc.print_both( ' emcee chain: nwalkers = %d nruns = %d' % (nwalkers, nruns), of_run) anc.print_both(' sampler ... ', of_run) # old version with threads #sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob, threads=nthreads) #sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob_sq, threads=nthreads, args=[parameter_names]) # close the pool of threads threads_pool.close() threads_pool.terminate() threads_pool.join() threads_pool = emcee.interruptible_pool.InterruptiblePool(nthreads) sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob, pool=threads_pool) anc.print_both(' ready to go', of_run) anc.print_both(' with nsave = %r' % (nsave), of_run) sys.stdout.flush() #sys.exit() if (nsave != False): # save temporary sampling during emcee every nruns*10% #if(os.path.exists(os.path.join(pso_path, 'emcee_temp.hdf5')) and os.path.isfile(os.path.join(pso_path, 'emcee_temp.hdf5'))): #os.remove(os.path.join(pso_path, 'emcee_temp.hdf5')) if (os.path.exists(os.path.join(pso_path, 'emcee_summary.hdf5')) and os.path.isfile( os.path.join(pso_path, 'emcee_summary.hdf5'))): os.remove(os.path.join(pso_path, 'emcee_summary.hdf5')) f_hdf5 = h5py.File(os.path.join(pso_path, 'emcee_summary.hdf5'), 'a') f_hdf5.create_dataset('parameter_names', data=parameter_names, dtype='S10') f_hdf5.create_dataset('boundaries', data=parameters_minmax, dtype=np.float64) temp_dset = f_hdf5.create_dataset('chains', (nwalkers, nruns, nfit), dtype=np.float64) f_hdf5['chains'].attrs['nwalkers'] = nwalkers f_hdf5['chains'].attrs['nruns'] = nruns f_hdf5['chains'].attrs['nfit'] = nfit f_hdf5['chains'].attrs['nfree'] = nfree temp_lnprob = f_hdf5.create_dataset('lnprobability', (nwalkers, nruns), dtype=np.float64) temp_lnprob.attrs['ln_err_const'] = ln_err_const temp_acceptance = f_hdf5.create_dataset('acceptance_fraction', data=np.zeros((nfit)), dtype=np.float64) temp_acor = f_hdf5.create_dataset('autocor_time', data=np.zeros((nfit)), dtype=np.float64) f_hdf5.close() pos = p0 niter_save = int(nruns / nsave) state = None anc.print_both(' Running emcee with temporary saving', of_run) sys.stdout.flush() for i in range(0, niter_save): anc.print_both('', of_run) anc.print_both(' iter: %6d ' % (i + 1), of_run) aaa = i * nsave bbb = aaa + nsave pos, prob, state = sampler.run_mcmc(pos, N=nsave, rstate0=state) anc.print_both('completed %d steps of %d' % (bbb, nruns), of_run) f_hdf5 = h5py.File( os.path.join(pso_path, 'emcee_summary.hdf5'), 'a') temp_dset = f_hdf5['chains'] #[:,:,:] temp_dset[:, aaa:bbb, :] = sampler.chain[:, aaa:bbb, :] temp_dset.attrs['completed_steps'] = bbb temp_lnprob = f_hdf5['lnprobability'] #[:,:] temp_lnprob[:, aaa:bbb] = sampler.lnprobability[:, aaa:bbb] shape_lnprob = sampler.lnprobability.shape acceptance_fraction = sampler.acceptance_fraction temp_acceptance = f_hdf5['acceptance_fraction'] temp_acceptance = acceptance_fraction #f_hdf5.create_dataset('acceptance_fraction', data=acceptance_fraction, dtype=np.float64) mean_acceptance_fraction = np.mean(acceptance_fraction) #temp_chains_T = np.zeros((bbb, nwalkers, nfit)) #for ifit in range(0,nfit): #temp_chains_T[:,:,ifit] = sampler.chain[:, :bbb, ifit].T #acor_time = anc.compute_autocor_time(temp_chains_T, walkers_transposed=True) acor_time = anc.compute_acor_time(sampler, steps_done=bbb) temp_acor = f_hdf5['autocor_time'] temp_acor[...] = acor_time #f_hdf5.create_dataset('autocor_time', data=np.array(acor_temp, dtype=np.float64), dtype=np.float64) #f_hdf5.create_dataset('autocor_time', data=np.array(sampler.acor, dtype=np.float64), dtype=np.float64) # not working #print 'aaa = %6d bbb = %6d -> sampler.lnprobability.shape = (%6d , %6d)' %(aaa, bbb, shape_lnprob[0], shape_lnprob[1]) f_hdf5.close() sys.stdout.flush() anc.print_both('', of_run) anc.print_both( '...done with saving temporary total shape = %s' % (str(np.shape(sampler.chain))), of_run) anc.print_both('', of_run) sys.stdout.flush() else: # GOOD COMPLETE SINGLE RUNNING OF EMCEE, WITHOUT REMOVING THE BURN-IN anc.print_both(' Running full emcee ...', of_run) sys.stdout.flush() sampler.run_mcmc(p0, nruns) anc.print_both('done', of_run) anc.print_both('', of_run) sys.stdout.flush() flatchains = sampler.chain[:, :, :].reshape( (nwalkers * nruns, nfit)) # full chain values acceptance_fraction = sampler.acceptance_fraction mean_acceptance_fraction = np.mean(acceptance_fraction) #autocor_time = sampler.acor #temp_chains_T = np.zeros((bbb, nwalkers, nfit)) #for ifit in range(0,nfit): #temp_chains_T[:,:,ifit] = sampler.chain[:, :, ifit].T #acor_time = anc.compute_autocor_time(temp_chains_T, walkers_transposed=True) acor_time = anc.compute_acor_time(sampler) lnprobability = sampler.lnprobability # save chains with original shape as hdf5 file f_hdf5 = h5py.File(os.path.join(pso_path, 'emcee_summary.hdf5'), 'w') f_hdf5.create_dataset('chains', data=sampler.chain, dtype=np.float64) f_hdf5['chains'].attrs['nwalkers'] = nwalkers f_hdf5['chains'].attrs['nruns'] = nruns f_hdf5['chains'].attrs['nfit'] = nfit f_hdf5['chains'].attrs['nfree'] = nfree f_hdf5['chains'].attrs['completed_steps'] = nruns f_hdf5.create_dataset('parameter_names', data=parameter_names, dtype='S10') f_hdf5.create_dataset('boundaries', data=parameters_minmax, dtype=np.float64) f_hdf5.create_dataset('acceptance_fraction', data=acceptance_fraction, dtype=np.float64) f_hdf5.create_dataset('autocor_time', data=acor_time, dtype=np.float64) f_hdf5.create_dataset('lnprobability', data=lnprobability, dtype=np.float64) f_hdf5['lnprobability'].attrs['ln_err_const'] = ln_err_const f_hdf5.close() anc.print_both( " Mean_acceptance_fraction should be between [0.25-0.5] = %.6f" % (mean_acceptance_fraction), of_run) anc.print_both('', of_run) # close the pool of threads threads_pool.close() threads_pool.terminate() threads_pool.join() anc.print_both('COMPLETED EMCEE', of_run) elapsed = time.time() - start elapsed_d, elapsed_h, elapsed_m, elapsed_s = anc.computation_time( elapsed) anc.print_both('', of_run) anc.print_both( ' pyTRADES: EMCEE FINISHED in %2d day %02d hour %02d min %.2f sec - bye bye' % (int(elapsed_d), int(elapsed_h), int(elapsed_m), elapsed_s), of_run) anc.print_both('', of_run) of_run.close() pytrades_lib.pytrades.deallocate_variables() return
def main(): # MAIN -- TRADES + EMCEE # READ COMMAND LINE ARGUMENTS cli = get_args() # STARTING TIME start = time.time() # RENAME working_path = cli.full_path nthreads = cli.nthreads # INITIALISE TRADES WITH SUBROUTINE WITHIN TRADES_LIB -> PARAMETER NAMES, MINMAX, INTEGRATION ARGS, READ DATA ... pytrades_lib.pytrades.initialize_trades(working_path, cli.sub_folder, nthreads) # RETRIEVE DATA AND VARIABLES FROM TRADES_LIB MODULE #global n_bodies, n_planets, ndata, npar, nfit, dof, inv_dof n_bodies = pytrades_lib.pytrades.n_bodies # NUMBER OF TOTAL BODIES OF THE SYSTEM n_planets = n_bodies - 1 # NUMBER OF PLANETS IN THE SYSTEM ndata = pytrades_lib.pytrades.ndata # TOTAL NUMBER OF DATA AVAILABLE npar = pytrades_lib.pytrades.npar # NUMBER OF TOTAL PARAMATERS ~n_planets X 6 nfit = pytrades_lib.pytrades.nfit # NUMBER OF PARAMETERS TO FIT nfree = pytrades_lib.pytrades.nfree # NUMBER OF FREE PARAMETERS (ie nrvset) dof = pytrades_lib.pytrades.dof # NUMBER OF DEGREES OF FREEDOM = NDATA - NFIT #inv_dof = np.float64(1.0 / dof) inv_dof = pytrades_lib.pytrades.inv_dof # READ THE NAMES OF THE PARAMETERS FROM THE TRADES_LIB AND CONVERT IT TO PYTHON STRINGS #trades_names = anc.convert_fortran2python_strarray(pytrades_lib.pytrades.parameter_names, #nfit, str_len=10 #) ##parameter_names = anc.trades_names_to_emcee(trades_names) str_len = pytrades_lib.pytrades.str_len temp_names = pytrades_lib.pytrades.get_parameter_names(nfit,str_len) trades_names = anc.convert_fortran_charray2python_strararray(temp_names) parameter_names = trades_names fitting_parameters = pytrades_lib.pytrades.fitting_parameters # INITIAL PARAMETER SET (NEEDED ONLY TO HAVE THE PROPER ARRAY/VECTOR) parameters_minmax = pytrades_lib.pytrades.parameters_minmax # PARAMETER BOUNDARIES delta_parameters = np.abs(parameters_minmax[:,1] - parameters_minmax[:,0]) # DELTA BETWEEN MAX AND MIN OF BOUNDARIES # RADIAL VELOCITIES SET n_rv = pytrades_lib.pytrades.nrv n_set_rv = pytrades_lib.pytrades.nrvset # TRANSITS SET n_t0 = pytrades_lib.pytrades.nt0 n_t0_sum = pytrades_lib.pytrades.ntts n_set_t0 = 0 for i in range(0, n_bodies-1): if (n_t0[i] > 0): n_set_t0 += 1 # compute global constant for the loglhd global ln_err_const #try: #e_RVo = np.asarray(pytrades_lib.pytrades.ervobs[:], dtype=np.float64) # fortran variable RV in python will be rv!!! #except: #e_RVo = np.asarray([0.], dtype=np.float64) #try: #e_T0o = np.asarray(pytrades_lib.pytrades.et0obs[:,:], dtype=np.float64).reshape((-1)) #except: #e_T0o = np.asarray([0.], dtype=np.float64) #ln_err_const = anc.compute_ln_err_const(ndata, dof, e_RVo, e_T0o, cli.ln_flag) ln_err_const = pytrades_lib.pytrades.ln_err_const # SET EMCEE PARAMETERS: nwalkers, nruns, nsave, npost = get_emcee_arguments(cli,nfit) # INITIALISE SCRIPT FOLDER/LOG FILE working_folder, run_log, of_run = init_folder(working_path, cli.sub_folder) anc.print_both('',of_run) anc.print_both(' ======== ',of_run) anc.print_both(' pyTRADES' ,of_run) anc.print_both(' ======== ',of_run) anc.print_both('',of_run) anc.print_both(' WORKING PATH = %s' %(working_path),of_run) anc.print_both(' NUMBER OF THREADS = %d' %(nthreads),of_run) anc.print_both(' dof = ndata(%d) - nfit(%d) - nfree(%d) = %d' %(ndata, nfit, nfree, dof),of_run) anc.print_both(' Total N_RV = %d for %d set(s)' %(n_rv, n_set_rv),of_run) anc.print_both(' Total N_T0 = %d for %d out of %d planet(s)' %(n_t0_sum, n_set_t0, n_planets),of_run) anc.print_both(' %s = %.7f' %('log constant error = ', ln_err_const),of_run) anc.print_both(' %s = %.7f' %('IN FORTRAN log constant error = ', pytrades_lib.pytrades.ln_err_const),of_run) # INITIALISE PSO ARGUMENTS FROM pso.opt FILE pytrades_lib.pytrades.init_pso(1,working_path) # read PSO options # PSO VARIABLES np_pso = pytrades_lib.pytrades.np_pso nit_pso = pytrades_lib.pytrades.nit_pso n_global = pytrades_lib.pytrades.n_global #n_global = 1 anc.print_both(' PSO n_global = %d npop = %d ngen = %d' %(n_global, np_pso, nit_pso), of_run) # RUN PSO+EMCEE n_global TIMES for iter_global in range(0,n_global): threads_pool = emcee.interruptible_pool.InterruptiblePool(1) # CREATES PROPER WORKING PATH AND NAME i_global = iter_global + 1 pso_path = os.path.join(os.path.join(working_folder, '%04d_pso2emcee' %(i_global)), '') pytrades_lib.pytrades.path_change(pso_path) anc.print_both('\n\n GLOBAL RUN %04d INTO PATH: %s\n' %(i_global, pso_path), of_run) if (cli.pso_type == 'run'): # RUN PSO anc.print_both(' RUN PSO', of_run) pso_start = time.time() if(not os.path.exists(pso_path)): os.makedirs(pso_path) # copy files anc.copy_simulation_files(working_path, pso_path) # CALL RUN_PSO SUBROUTINE FROM TRADES_LIB: RUNS PSO AND COMPUTES THE BEST SOLUTION, SAVING ALL THE POPULATION EVOLUTION pso_parameters = fitting_parameters.copy() pso_fitness = 0. pso_parameters, pso_fitness = pytrades_lib.pytrades.pyrun_pso(nfit,i_global) anc.print_both(' completed run_pso', of_run) pso_best_evolution = np.asarray(pytrades_lib.pytrades.pso_best_evolution[...], dtype=np.float64) anc.print_both(' pso_best_evolution retrieved', of_run) anc.print_both(' last pso_best_evolution', of_run) last_pso_parameters = np.asarray(pso_best_evolution[:nfit,-1],dtype=np.float64) last_pso_fitness = pso_best_evolution[-1,-1].astype(np.float64) anc.print_both(' fitness = %.f' %(last_pso_fitness), of_run) # SAVE PSO SIMULATION IN pso_run.hdf5 FILE print ' Creating pso hdf5 file: %s' %(os.path.join(pso_path, 'pso_run.hdf5')) pso_hdf5 = h5py.File(os.path.join(pso_path, 'pso_run.hdf5'), 'w') pso_hdf5.create_dataset('population', data=pytrades_lib.pytrades.population, dtype=np.float64) pso_hdf5.create_dataset('population_fitness', data=pytrades_lib.pytrades.population_fitness, dtype=np.float64) pso_hdf5.create_dataset('pso_parameters', data=pso_parameters, dtype=np.float64) pso_hdf5.create_dataset('pso_fitness', data=np.array(pso_fitness), dtype=np.float64) pso_hdf5.create_dataset('pso_best_evolution', data=pso_best_evolution, dtype=np.float64) pso_hdf5.create_dataset('parameters_minmax', data=parameters_minmax, dtype=np.float64) pso_hdf5.create_dataset('parameter_names', data=parameter_names, dtype='S10') pso_hdf5['population'].attrs['npop'] = np_pso pso_hdf5['population'].attrs['niter'] = nit_pso pso_hdf5['population'].attrs['iter_global'] = iter_global+1 pso_hdf5['population'].attrs['nfit'] = nfit pso_hdf5.close() population = np.asarray(pytrades_lib.pytrades.population, dtype=np.float64) population_fitness = np.asarray(pytrades_lib.pytrades.population_fitness, dtype=np.float64) anc.print_both(' ', of_run) fitness_iter, lgllhd_iter, check_iter = pytrades_lib.pytrades.write_summary_files(i_global, pso_parameters) elapsed = time.time() - pso_start elapsed_d, elapsed_h, elapsed_m, elapsed_s = anc.computation_time(elapsed) anc.print_both(' ', of_run) anc.print_both(' PSO FINISHED in %2d day %02d hour %02d min %.2f sec - bye bye' %(int(elapsed_d), int(elapsed_h), int(elapsed_m), elapsed_s), of_run) #p0, pso_fitness_p0 = pso_to_emcee(nfit, nwalkers, population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution) p0 = compute_initial_walkers(nfit, nwalkers, pso_parameters, parameters_minmax, parameter_names, delta_sigma, of_run) elif (cli.pso_type == 'exists'): # READ PREVIOUS PSO_RUN.HDF5 FILE AND INITIALISE POPULATION FOR EMCEE anc.print_both(' READ PREVIOUS PSO_RUN.HDF5 FILE AND INITIALISE POPULATION FOR EMCEE', of_run) population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution, pso_parameters_minmax, pso_parameter_names, pop_shape = get_pso_data(os.path.join(pso_path, 'pso_run.hdf5')) fitness_iter, lgllhd_iter, check_iter = pytrades_lib.pytrades.write_summary_files(i_global, pso_parameters) anc.print_both(' read pso_run.hdf5 file with best pso_fitness = %.7f' %(pso_fitness), of_run) #p0, pso_fitness_p0 = pso_to_emcee(nfit, nwalkers, population, population_fitness, pso_parameters, pso_fitness, pso_best_evolution) p0 = compute_initial_walkers(nfit, nwalkers, pso_parameters, parameters_minmax, parameter_names, delta_sigma, of_run) elif (cli.pso_type == 'skip'): # DO NOT RUN PSO, ONLY EMCEE anc.print_both(' DO NOT RUN PSO, ONLY EMCEE', of_run) #p0 = [parameters_minmax[:,0] + np.random.random(nfit)*delta_parameters for i in range(0, nwalkers)] p0 = compute_initial_walkers(nfit, nwalkers, fitting_parameters, parameters_minmax, parameter_names, delta_sigma, of_run) anc.print_both(' emcee chain: nwalkers = %d nruns = %d' %(nwalkers, nruns), of_run) anc.print_both(' sampler ... ',of_run) # old version with threads #sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob, threads=nthreads) #sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob_sq, threads=nthreads, args=[parameter_names]) # close the pool of threads threads_pool.close() threads_pool.terminate() threads_pool.join() threads_pool = emcee.interruptible_pool.InterruptiblePool(nthreads) sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob, pool=threads_pool) anc.print_both(' ready to go', of_run) anc.print_both(' with nsave = %r' %(nsave), of_run) sys.stdout.flush() #sys.exit() if (nsave != False): # save temporary sampling during emcee every nruns*10% #if(os.path.exists(os.path.join(pso_path, 'emcee_temp.hdf5')) and os.path.isfile(os.path.join(pso_path, 'emcee_temp.hdf5'))): #os.remove(os.path.join(pso_path, 'emcee_temp.hdf5')) if(os.path.exists(os.path.join(pso_path, 'emcee_summary.hdf5')) and os.path.isfile(os.path.join(pso_path, 'emcee_summary.hdf5'))): os.remove(os.path.join(pso_path, 'emcee_summary.hdf5')) f_hdf5 = h5py.File(os.path.join(pso_path, 'emcee_summary.hdf5'), 'a') f_hdf5.create_dataset('parameter_names', data=parameter_names, dtype='S10') f_hdf5.create_dataset('boundaries', data=parameters_minmax, dtype=np.float64) temp_dset = f_hdf5.create_dataset('chains', (nwalkers, nruns, nfit), dtype=np.float64) f_hdf5['chains'].attrs['nwalkers'] = nwalkers f_hdf5['chains'].attrs['nruns'] = nruns f_hdf5['chains'].attrs['nfit'] = nfit f_hdf5['chains'].attrs['nfree'] = nfree temp_lnprob = f_hdf5.create_dataset('lnprobability', (nwalkers, nruns), dtype=np.float64) temp_lnprob.attrs['ln_err_const'] = ln_err_const temp_acceptance = f_hdf5.create_dataset('acceptance_fraction', data=np.zeros((nfit)), dtype=np.float64) temp_acor = f_hdf5.create_dataset('autocor_time', data=np.zeros((nfit)), dtype=np.float64) f_hdf5.close() pos = p0 niter_save = int(nruns/nsave) state=None anc.print_both(' Running emcee with temporary saving', of_run) sys.stdout.flush() for i in range(0, niter_save): anc.print_both('', of_run) anc.print_both(' iter: %6d ' %(i+1), of_run) aaa = i*nsave bbb = aaa+nsave pos, prob, state = sampler.run_mcmc(pos, N=nsave, rstate0=state) anc.print_both('completed %d steps of %d' %(bbb, nruns), of_run) f_hdf5 = h5py.File(os.path.join(pso_path, 'emcee_summary.hdf5'), 'a') temp_dset = f_hdf5['chains'] #[:,:,:] temp_dset[:,aaa:bbb,:] = sampler.chain[:, aaa:bbb, :] temp_dset.attrs['completed_steps'] = bbb temp_lnprob = f_hdf5['lnprobability'] #[:,:] temp_lnprob[:, aaa:bbb] = sampler.lnprobability[:, aaa:bbb] shape_lnprob = sampler.lnprobability.shape acceptance_fraction = sampler.acceptance_fraction temp_acceptance = f_hdf5['acceptance_fraction'] temp_acceptance = acceptance_fraction #f_hdf5.create_dataset('acceptance_fraction', data=acceptance_fraction, dtype=np.float64) mean_acceptance_fraction = np.mean(acceptance_fraction) #temp_chains_T = np.zeros((bbb, nwalkers, nfit)) #for ifit in range(0,nfit): #temp_chains_T[:,:,ifit] = sampler.chain[:, :bbb, ifit].T #acor_time = anc.compute_autocor_time(temp_chains_T, walkers_transposed=True) acor_time = anc.compute_acor_time(sampler, steps_done=bbb) temp_acor = f_hdf5['autocor_time'] temp_acor[...] = acor_time #f_hdf5.create_dataset('autocor_time', data=np.array(acor_temp, dtype=np.float64), dtype=np.float64) #f_hdf5.create_dataset('autocor_time', data=np.array(sampler.acor, dtype=np.float64), dtype=np.float64) # not working #print 'aaa = %6d bbb = %6d -> sampler.lnprobability.shape = (%6d , %6d)' %(aaa, bbb, shape_lnprob[0], shape_lnprob[1]) f_hdf5.close() sys.stdout.flush() anc.print_both('', of_run) anc.print_both('...done with saving temporary total shape = %s' %(str(np.shape(sampler.chain))), of_run) anc.print_both('', of_run) sys.stdout.flush() else: # GOOD COMPLETE SINGLE RUNNING OF EMCEE, WITHOUT REMOVING THE BURN-IN anc.print_both(' Running full emcee ...', of_run) sys.stdout.flush() sampler.run_mcmc(p0, nruns) anc.print_both('done', of_run) anc.print_both('', of_run) sys.stdout.flush() flatchains = sampler.chain[:, :, :].reshape((nwalkers*nruns, nfit)) # full chain values acceptance_fraction = sampler.acceptance_fraction mean_acceptance_fraction = np.mean(acceptance_fraction) #autocor_time = sampler.acor #temp_chains_T = np.zeros((bbb, nwalkers, nfit)) #for ifit in range(0,nfit): #temp_chains_T[:,:,ifit] = sampler.chain[:, :, ifit].T #acor_time = anc.compute_autocor_time(temp_chains_T, walkers_transposed=True) acor_time = anc.compute_acor_time(sampler) lnprobability = sampler.lnprobability # save chains with original shape as hdf5 file f_hdf5 = h5py.File(os.path.join(pso_path, 'emcee_summary.hdf5'), 'w') f_hdf5.create_dataset('chains', data=sampler.chain, dtype=np.float64) f_hdf5['chains'].attrs['nwalkers'] = nwalkers f_hdf5['chains'].attrs['nruns'] = nruns f_hdf5['chains'].attrs['nfit'] = nfit f_hdf5['chains'].attrs['nfree'] = nfree f_hdf5['chains'].attrs['completed_steps'] = nruns f_hdf5.create_dataset('parameter_names', data=parameter_names, dtype='S10') f_hdf5.create_dataset('boundaries', data=parameters_minmax, dtype=np.float64) f_hdf5.create_dataset('acceptance_fraction', data=acceptance_fraction, dtype=np.float64) f_hdf5.create_dataset('autocor_time', data=acor_time, dtype=np.float64) f_hdf5.create_dataset('lnprobability', data=lnprobability, dtype=np.float64) f_hdf5['lnprobability'].attrs['ln_err_const'] = ln_err_const f_hdf5.close() anc.print_both(" Mean_acceptance_fraction should be between [0.25-0.5] = %.6f" %(mean_acceptance_fraction), of_run) anc.print_both('', of_run) # close the pool of threads threads_pool.close() threads_pool.terminate() threads_pool.join() anc.print_both('COMPLETED EMCEE', of_run) elapsed = time.time() - start elapsed_d, elapsed_h, elapsed_m, elapsed_s = anc.computation_time(elapsed) anc.print_both('', of_run) anc.print_both(' pyTRADES: EMCEE FINISHED in %2d day %02d hour %02d min %.2f sec - bye bye' %(int(elapsed_d), int(elapsed_h), int(elapsed_m), elapsed_s), of_run) anc.print_both('', of_run) of_run.close() pytrades_lib.pytrades.deallocate_variables() return