Exemplo n.º 1
0
class DiscontinuityType(object):
    name    = ''
    pnames  = []
    npar    = len(pnames)
    penalty = 0.
    
    def __init__(self, discontinuity):
        self.discontinuity = self.d = discontinuity
        self.best_fit_pv = None
        self.best_fit_model = None
        self.bic = None
        self._optimization_result = None
            
#    def __str__(self):
#        return '{:s} {:4.1f}  {:4.1f}'.format(self.name, self.position, self.amplitude)

#    def __repr__(self):
#        return '{:s}({:4.1f}, {:4.1f})'.format(self.name, self.position, self.amplitude)

            
    def nlnlike_wn(self, pv):
        if not self.is_inside_bounds(pv):
            return inf
        else:
            return -lnlikelihood(self.d.flux, self.model(pv), self.d.wn_estimate)

        
    def nlnlike_gp(self, pv):
        if not self.is_inside_bounds(pv):
            return inf
        else:
            return -self.d.gp.lnlikelihood(self.d.cadence, self.d.flux-self.model(pv), freeze_k=True)    

        
    def fit(self, use_de=False, de_npop=30, de_niter=100, method='Powell'):
        jamp, jpos, fstd = self.d.amplitude, self.d.position, self.d.flux.std()
        nlnlike = self.nlnlike_gp if self.d.use_gp else self.nlnlike_wn
        if use_de:
            self._de = DiffEvol(nlnlike, self._de_bounds(jamp, jpos, fstd), npop=de_npop)
            self._de.optimize(de_niter)
            pv0 = self._de.minimum_location
        else:
            pv0 = self._pv0(jamp, jpos, fstd)
            
        self._optimization_result = r = minimize(nlnlike, pv0, method=method)
        self.best_fit_pv = r.x
        self.best_fit_model = self.model(r.x)
        xx = r.x.copy()
        xx[-2:] = 0
        self.best_fit_model_wo_baseline = self.model(xx)
        self.bic = self.c_bic(r.fun)
        return self.bic

    
    def model(self, pv, cad=None):
        raise NotImplementedError

    
    def c_bic(self, nln):
        return 2*nln + self.npar*log(self.d.npt) + self.penalty
Exemplo n.º 2
0
class DiscontinuityType(object):
    name = ''
    pnames = []
    npar = len(pnames)
    penalty = 0.

    def __init__(self, discontinuity):
        self.discontinuity = self.d = discontinuity
        self.best_fit_pv = None
        self.best_fit_model = None
        self.bic = None
        self._optimization_result = None


#    def __str__(self):
#        return '{:s} {:4.1f}  {:4.1f}'.format(self.name, self.position, self.amplitude)

#    def __repr__(self):
#        return '{:s}({:4.1f}, {:4.1f})'.format(self.name, self.position, self.amplitude)

    def nlnlike_wn(self, pv):
        if not self.is_inside_bounds(pv):
            return inf
        else:
            return -lnlikelihood(self.d.flux, self.model(pv),
                                 self.d.wn_estimate)

    def nlnlike_gp(self, pv):
        if not self.is_inside_bounds(pv):
            return inf
        else:
            return -self.d.gp.lnlikelihood(
                self.d.cadence, self.d.flux - self.model(pv), freeze_k=True)

    def fit(self, use_de=False, de_npop=30, de_niter=100, method='Powell'):
        jamp, jpos, fstd = self.d.amplitude, self.d.position, self.d.flux.std()
        nlnlike = self.nlnlike_gp if self.d.use_gp else self.nlnlike_wn
        if use_de:
            self._de = DiffEvol(nlnlike,
                                self._de_bounds(jamp, jpos, fstd),
                                npop=de_npop)
            self._de.optimize(de_niter)
            pv0 = self._de.minimum_location
        else:
            pv0 = self._pv0(jamp, jpos, fstd)

        self._optimization_result = r = minimize(nlnlike, pv0, method=method)
        self.best_fit_pv = r.x
        self.best_fit_model = self.model(r.x)
        xx = r.x.copy()
        xx[-2:] = 0
        self.best_fit_model_wo_baseline = self.model(xx)
        self.bic = self.c_bic(r.fun)
        return self.bic

    def model(self, pv, cad=None):
        raise NotImplementedError

    def c_bic(self, nln):
        return 2 * nln + self.npar * log(self.d.npt) + self.penalty
Exemplo n.º 3
0
 def fit(self, use_de=False, de_npop=30, de_niter=100, method='Powell'):
     jamp, jpos, fstd = self.d.amplitude, self.d.position, self.d.flux.std()
     nlnlike = self.nlnlike_gp if self.d.use_gp else self.nlnlike_wn
     if use_de:
         self._de = DiffEvol(nlnlike, self._de_bounds(jamp, jpos, fstd), npop=de_npop)
         self._de.optimize(de_niter)
         pv0 = self._de.minimum_location
     else:
         pv0 = self._pv0(jamp, jpos, fstd)
         
     self._optimization_result = r = minimize(nlnlike, pv0, method=method)
     self.best_fit_pv = r.x
     self.best_fit_model = self.model(r.x)
     xx = r.x.copy()
     xx[-2:] = 0
     self.best_fit_model_wo_baseline = self.model(xx)
     self.bic = self.c_bic(r.fun)
     return self.bic
Exemplo n.º 4
0
 def fit(self, use_de=False, de_npop=30, de_niter=100, method='Powell'):
     jamp, jpos, fstd = self.d.amplitude, self.d.position, self.d.flux.std()
     nlnlike = self.nlnlike_gp if self.d.use_gp else self.nlnlike_wn
     if use_de:
         self._de = DiffEvol(nlnlike, self._de_bounds(jamp, jpos, fstd), npop=de_npop)
         self._de.optimize(de_niter)
         pv0 = self._de.minimum_location
     else:
         pv0 = self._pv0(jamp, jpos, fstd)
         
     self._optimization_result = r = minimize(nlnlike, pv0, method=method)
     self.best_fit_pv = r.x
     self.best_fit_model = self.model(r.x)
     xx = r.x.copy()
     xx[-2:] = 0
     self.best_fit_model_wo_baseline = self.model(xx)
     self.bic = self.c_bic(r.fun)
     return self.bic
Exemplo n.º 5
0
def pyorbit_emcee(config_in, input_datasets=None, return_output=None):

    try:
        import emcee
    except:
        print("ERROR: emcee not installed, this will not work")
        quit()

    os.environ["OMP_NUM_THREADS"] = "1"

    optimize_dir_output = './' + config_in['output'] + '/optimize/'
    pyde_dir_output = './' + config_in['output'] + '/pyde/'

    emcee_dir_output = './' + config_in['output'] + '/emcee/'

    reloaded_optimize = False
    reloaded_pyde = False
    reloaded_emcee = False

    try:
        mc, population, starting_point, theta_dict = pyde_load_from_cpickle(
            pyde_dir_output, prefix='')
        reloaded_pyde = True
    except FileNotFoundError:
        pass

    try:
        mc, starting_point, population, prob, state, sampler_chain, \
            sampler_lnprobability, _, theta_dict, sampler = \
            emcee_load_from_cpickle(emcee_dir_output)
        reloaded_emcee = True
    except FileNotFoundError:
        pass

    try:
        starting_point, previous_boundaries, theta_dict = starting_point_load_from_cpickle(
            optimize_dir_output)
        reloaded_optimize = True
    except FileNotFoundError:
        pass

    print()
    print('reloaded_optimize: ', reloaded_optimize)
    print('reloaded_pyde: ', reloaded_pyde)
    print('reloaded_emcee: ', reloaded_emcee)
    print()

    if reloaded_pyde or reloaded_emcee:
        previous_boundaries = mc.bounds

    if reloaded_emcee:
        print('Requested steps:', mc.emcee_parameters['nsteps'])

        mc.emcee_parameters['completed_nsteps'] = \
            int(sampler_chain.shape[1] * mc.emcee_parameters['thin'])

        print('Completed:', mc.emcee_parameters['completed_nsteps'])
        pars_input(config_in, mc, input_datasets, reload_emcee=True)
        print('Total:', mc.emcee_parameters['nsteps'])
        """ There's no need to do anything"""
        flatchain = emcee_flatchain(sampler_chain,
                                    mc.emcee_parameters['nburn'],
                                    mc.emcee_parameters['thin'])
        mc.model_setup()
        mc.initialize_logchi2()

        results_analysis.print_integrated_ACF(sampler_chain, theta_dict,
                                              mc.emcee_parameters['thin'])
        """ In case the current startin point comes from a previous analysis """
        mc.emcee_dir_output = emcee_dir_output

        if mc.emcee_parameters['nsteps'] <= mc.emcee_parameters[
                'completed_nsteps']:

            print('Reference Time Tref: ', mc.Tref)
            print()
            print('Dimensions = ', mc.ndim)
            print('Nwalkers = ', mc.emcee_parameters['nwalkers'])
            print('Steps = ', mc.emcee_parameters['nsteps'])
            print()
            print('Original starting point of emcee:')
            print()

            results_analysis.results_resumen(mc,
                                             starting_point,
                                             compute_lnprob=True,
                                             is_starting_point=True)

            print('emcee results:')
            print()

            results_analysis.results_resumen(mc, flatchain)

            print()
            print('emcee completed')
            print()

            if return_output:
                return mc, sampler_chain, sampler_lnprobability
            else:
                return

        elif not sampler:
            print(
                'Sampler file is missing - only analysis performed with PyORBIT >8.1 cn be resumed'
            )
            if return_output:
                return mc, sampler_chain, sampler_lnprobability
            else:
                return

    if reloaded_emcee:
        sampled = mc.emcee_parameters['completed_nsteps']
        nsteps_todo = mc.emcee_parameters['nsteps'] \
            - mc.emcee_parameters['completed_nsteps']

        print('Resuming from a previous run:')
        print('Performed steps = ', mc.emcee_parameters['completed_nsteps'])
        print('Final # of steps = ', mc.emcee_parameters['nsteps'])
        print('Steps to be performed = ', nsteps_todo)
        print()

    else:

        mc = ModelContainerEmcee()
        pars_input(config_in, mc, input_datasets)

        if mc.pyde_parameters['shutdown_jitter'] or mc.emcee_parameters[
                'shutdown_jitter']:
            for dataset_name, dataset in mc.dataset_dict.items():
                dataset.shutdown_jitter()

        # keep track of which version has been used to perform emcee computations
        mc.emcee_parameters['version'] = emcee.__version__[0]

        mc.model_setup()
        mc.create_variables_bounds()
        mc.initialize_logchi2()

        results_analysis.results_resumen(mc, None, skip_theta=True)

        mc.pyde_dir_output = pyde_dir_output
        mc.emcee_dir_output = emcee_dir_output

        mc.emcee_parameters['nwalkers'] = mc.ndim * \
            mc.emcee_parameters['npop_mult']
        if mc.emcee_parameters['nwalkers'] % 2 == 1:
            mc.emcee_parameters['nwalkers'] += 1

        if not os.path.exists(mc.emcee_dir_output):
            os.makedirs(mc.emcee_dir_output)

        state = None
        sampled = 0
        nsteps_todo = mc.emcee_parameters['nsteps']

    print('Include priors: ', mc.include_priors)
    print()
    print('Reference Time Tref: ', mc.Tref)
    print()
    print('Dimensions = ', mc.ndim)
    print('Nwalkers = ', mc.emcee_parameters['nwalkers'])
    print()

    if mc.emcee_parameters['version'] == '2':
        mc.emcee_parameters['use_threading_pool'] = False

    if not mc.pyde_parameters.get('use_threading_pool', False):
        mc.pyde_parameters['use_threading_pool'] = False

    if not mc.emcee_parameters.get('use_threading_pool', False):
        mc.emcee_parameters['use_threading_pool'] = False

    if reloaded_emcee:
        sys.stdout.flush()
        pass
    elif reloaded_pyde:

        theta_dict_legacy = theta_dict.copy()
        population_legacy = population.copy()

        theta_dict = results_analysis.get_theta_dictionary(mc)
        population = np.zeros([mc.emcee_parameters['nwalkers'], mc.ndim],
                              dtype=np.double)

        for theta_name, theta_i in theta_dict.items():
            population[:, theta_i] = population_legacy[:, theta_dict_legacy[
                theta_name]]
            mc.bounds[theta_i] = previous_boundaries[
                theta_dict_legacy[theta_name]]

        starting_point = np.median(population, axis=0)
        # print(starting_point)
        # print(population)

        print('Using previous population as starting point. ')
        print()
        sys.stdout.flush()

    elif mc.starting_point_flag or reloaded_optimize:

        if reloaded_optimize:
            print(
                'Using the output from a previous optimize run as starting point'
            )
            theta_dict_legacy = theta_dict.copy()
            starting_point_legacy = starting_point.copy()
            theta_dict = results_analysis.get_theta_dictionary(mc)
            for theta_name, theta_i in theta_dict.items():
                starting_point[theta_i] = starting_point_legacy[
                    theta_dict_legacy[theta_name]]
        else:
            print('Using user-defined starting point from YAML file')
            mc.create_starting_point()
            starting_point = mc.starting_point

        population = np.zeros([mc.emcee_parameters['nwalkers'], mc.ndim],
                              dtype=np.double)
        for ii in range(0, mc.emcee_parameters['nwalkers']):
            population[ii, :] = np.random.normal(starting_point, 0.0000001)

        print(
            'to create a synthetic population extremely close to the starting values.'
        )
        print()

        sys.stdout.flush()

    else:
        """ None of the previous cases has been satisfied, we have to run PyDE """
        try:
            from pyde.de import DiffEvol
        except ImportError:
            print(
                'ERROR! PyDE is not installed, run first with optimize instead of emcee'
            )
            quit()

        if not os.path.exists(mc.pyde_dir_output):
            os.makedirs(mc.pyde_dir_output)

        print('Using threading pool for PyDE:',
              mc.pyde_parameters['use_threading_pool'])

        print('PyDE running')
        sys.stdout.flush()

        if mc.pyde_parameters['use_threading_pool']:
            with multiprocessing.Pool() as pool:

                de = DiffEvol(mc,
                              mc.bounds,
                              mc.emcee_parameters['nwalkers'],
                              maximize=True,
                              pool=pool)

                de.optimize(int(mc.pyde_parameters['ngen']))
        else:
            de = DiffEvol(mc,
                          mc.bounds,
                          mc.emcee_parameters['nwalkers'],
                          maximize=True)

            de.optimize(int(mc.pyde_parameters['ngen']))

        population = de.population
        starting_point = np.median(population, axis=0)

        theta_dict = results_analysis.get_theta_dictionary(mc)
        """ bounds redefinition and fix for PyDE anomalous results """
        if mc.recenter_bounds_flag:
            pyde_save_to_pickle(mc,
                                population,
                                starting_point,
                                theta_dict,
                                prefix='orig')

            mc.recenter_bounds(starting_point)
            population = mc.fix_population(starting_point, population)
            starting_point = np.median(population, axis=0)

            print('Boundaries redefined after PyDE output')

        pyde_save_to_pickle(mc, population, starting_point, theta_dict)

        print('PyDE completed')
        print()
        sys.stdout.flush()

    if reloaded_emcee:
        print('Original starting point of emcee:')
        print()

    results_analysis.results_resumen(mc,
                                     starting_point,
                                     compute_lnprob=True,
                                     is_starting_point=True)
    sys.stdout.flush()

    print()
    print('*************************************************************')
    print()
    print('Running emcee')
    print()
    print('emcee version: ', emcee.__version__)
    if mc.emcee_parameters['version'] == '2':
        print('WARNING: upgrading to version 3 is strongly advised')
    print('Using threading pool for emcee:',
          mc.emcee_parameters.get('use_threading_pool', False))
    print()

    if reloaded_emcee:
        print('Using reloaded sampler')
        print()
    else:
        sampler = emcee.EnsembleSampler(mc.emcee_parameters['nwalkers'],
                                        mc.ndim, mc)

    if mc.emcee_parameters['nsave'] > 0:
        print('Saving temporary steps')
        print()
        niter = int(np.ceil(nsteps_todo / mc.emcee_parameters['nsave']))

        for i in range(0, niter):

            if mc.emcee_parameters['use_threading_pool']:
                with multiprocessing.Pool() as pool:
                    sampler.pool = pool
                    population, prob, state = sampler.run_mcmc(
                        population,
                        mc.emcee_parameters['nsave'],
                        thin=mc.emcee_parameters['thin'],
                        rstate0=state,
                        progress=True)
            else:
                population, prob, state = sampler.run_mcmc(
                    population,
                    mc.emcee_parameters['nsave'],
                    thin=mc.emcee_parameters['thin'],
                    rstate0=state,
                    progress=True)

            sampled += mc.emcee_parameters['nsave']
            theta_dict = results_analysis.get_theta_dictionary(mc)
            emcee_save_to_cpickle(mc,
                                  starting_point,
                                  population,
                                  prob,
                                  state,
                                  sampler,
                                  theta_dict,
                                  samples=sampled)

            flatchain = emcee_flatchain(sampler.chain,
                                        mc.emcee_parameters['nburn'],
                                        mc.emcee_parameters['thin'])

            results_analysis.print_integrated_ACF(sampler.chain, theta_dict,
                                                  mc.emcee_parameters['thin'])

            results_analysis.results_resumen(mc, flatchain)

            print()
            print(sampled, '  steps completed, average lnprob:, ',
                  np.median(prob))
            print()
            sys.stdout.flush()

            # It turns out that reloading the sampler from the file will
            # result in faster parallelization...
            state, sampler = emcee_simpler_load_from_cpickle(emcee_dir_output)

    else:
        if mc.emcee_parameters['use_threading_pool']:
            with multiprocessing.Pool() as pool:
                sampler.pool = pool
                population, prob, state = sampler.run_mcmc(
                    population,
                    nsteps_todo,
                    thin=mc.emcee_parameters['thin'],
                    rstate0=state,
                    progress=True)

        else:
            population, prob, state = sampler.run_mcmc(
                population,
                nsteps_todo,
                thin=mc.emcee_parameters['thin'],
                rstate0=state,
                progress=True)

        sampled += nsteps_todo

        theta_dict = results_analysis.get_theta_dictionary(mc)
        emcee_save_to_cpickle(mc,
                              starting_point,
                              population,
                              prob,
                              state,
                              sampler,
                              theta_dict,
                              samples=sampled)

        flatchain = emcee_flatchain(sampler.chain,
                                    mc.emcee_parameters['nburn'],
                                    mc.emcee_parameters['thin'])

        results_analysis.print_integrated_ACF(sampler.chain, theta_dict,
                                              mc.emcee_parameters['thin'])

        results_analysis.results_resumen(mc, flatchain)

    print()
    print('emcee completed')
    print()
    """ A dummy file is created to let the cpulimit script to proceed with the next step"""
    emcee_create_dummy_file(mc)

    if return_output:
        return mc, sampler.chain, sampler.lnprobability
Exemplo n.º 6
0
def pyorbit_emcee(config_in, input_datasets=None, return_output=None):

    try:
        import emcee
    except:
        print("ERROR: emcee not installed, this will not work")
        quit()

    os.environ["OMP_NUM_THREADS"] = "1"

    optimize_dir_output = './' + config_in['output'] + '/optimize/'
    pyde_dir_output = './' + config_in['output'] + '/pyde/'
    emcee_dir_output = './' + config_in['output'] + '/emcee/'

    reloaded_optimize = False
    reloaded_pyde = False
    reloaded_emcee_multirun = False
    reloaded_emcee = False

    try:
        mc, population, starting_point, theta_dict = pyde_load_from_cpickle(
            pyde_dir_output, prefix='')
        reloaded_pyde = True
    except:
        pass

    try:
        mc, starting_point, population, _, _, sampler_chain, _, _, theta_dict, _ = \
            emcee_load_from_cpickle(emcee_dir_output, prefix='MR')
        reloaded_emcee_multirun = True
    except:
        pass

    try:
        mc, starting_point, population, _, _, sampler_chain, sampler_lnprobability, _, theta_dict, _ = \
            emcee_load_from_cpickle(emcee_dir_output)
        reloaded_emcee = True
    except:
        pass

    try:
        starting_point, previous_boundaries, theta_dict = starting_point_load_from_cpickle(
            optimize_dir_output)
        reloaded_optimize = True
    except:
        pass

    print()
    print('reloaded_optimize: ', reloaded_pyde)
    print('reloaded_pyde: ', reloaded_pyde)
    print('reloaded_emcee_multirun: ', reloaded_emcee_multirun)
    print('reloaded_emcee: ', reloaded_emcee)

    if reloaded_emcee:
        """ There's no need to do anything"""
        flatchain = emcee_flatchain(sampler_chain,
                                    mc.emcee_parameters['nburn'],
                                    mc.emcee_parameters['thin'])
        mc.model_setup()
        mc.initialize_logchi2()
        results_analysis.print_integrated_ACF(sampler_chain, theta_dict,
                                              mc.emcee_parameters['thin'])
        results_analysis.results_resumen(mc, flatchain)

        if return_output:
            return mc, sampler_chain, sampler_lnprobability
        else:
            return

    reloaded_mc = reloaded_pyde or reloaded_emcee_multirun
    if reloaded_mc:
        previous_boundaries = mc.bounds

    mc = ModelContainerEmcee()

    pars_input(config_in, mc, input_datasets)

    if mc.pyde_parameters['shutdown_jitter'] or mc.emcee_parameters[
            'shutdown_jitter']:
        for dataset_name, dataset in mc.dataset_dict.items():
            dataset.shutdown_jitter()

    # keep track of which version has been used to perform emcee computations
    mc.emcee_parameters['version'] = emcee.__version__[0]

    mc.model_setup()
    mc.create_variables_bounds()
    mc.initialize_logchi2()

    results_analysis.results_resumen(mc, None, skip_theta=True)

    mc.pyde_dir_output = pyde_dir_output
    mc.emcee_dir_output = emcee_dir_output

    mc.emcee_parameters['nwalkers'] = mc.ndim * \
        mc.emcee_parameters['npop_mult']
    if mc.emcee_parameters['nwalkers'] % 2 == 1:
        mc.emcee_parameters['nwalkers'] += 1

    if not os.path.exists(mc.emcee_dir_output):
        os.makedirs(mc.emcee_dir_output)

    print()
    print('emcee version: ', emcee.__version__)
    if mc.emcee_parameters['version'] == '2':
        print('WARNING: upgrading to version 3 is strongly advised')
    print()
    print('Include priors: ', mc.include_priors)
    print()
    print('Reference Time Tref: ', mc.Tref)
    print()
    print('Dimensions = ', mc.ndim)
    print('Nwalkers = ', mc.emcee_parameters['nwalkers'])

    if not getattr(mc, 'use_threading_pool', False):
        mc.use_threading_pool = False

    print()
    print('Using threading pool:', mc.use_threading_pool)
    print()
    print('*************************************************************')
    print()

    if reloaded_mc:

        theta_dict_legacy = theta_dict.copy()
        population_legacy = population.copy()

        theta_dict = results_analysis.get_theta_dictionary(mc)
        population = np.zeros([mc.emcee_parameters['nwalkers'], mc.ndim],
                              dtype=np.double)

        for theta_name, theta_i in theta_dict.items():
            population[:, theta_i] = population_legacy[:, theta_dict_legacy[
                theta_name]]
            mc.bounds[theta_i] = previous_boundaries[
                theta_dict_legacy[theta_name]]

        starting_point = np.median(population, axis=0)
        # print(starting_point)
        # print(population)

        print('Using previous population as starting point. ')
        sys.stdout.flush()
        print()

    else:

        if mc.starting_point_flag or reloaded_optimize:

            if reloaded_optimize:
                print(
                    'Using the output from a previous optimize run as starting point'
                )
                theta_dict_legacy = theta_dict.copy()
                starting_point_legacy = starting_point.copy()
                theta_dict = results_analysis.get_theta_dictionary(mc)
                for theta_name, theta_i in theta_dict.items():
                    starting_point[theta_i] = starting_point_legacy[
                        theta_dict_legacy[theta_name]]
            else:
                print('Using user-defined starting point from YAML file')
                mc.create_starting_point()
                starting_point = mc.starting_point

            population = np.zeros([mc.emcee_parameters['nwalkers'], mc.ndim],
                                  dtype=np.double)
            for ii in range(0, mc.emcee_parameters['nwalkers']):
                population[ii, :] = np.random.normal(starting_point, 0.0000001)

            print(
                'to create a synthetic population extremely close to the starting values.'
            )
            sys.stdout.flush()

        else:

            try:
                from pyde.de import DiffEvol
            except ImportError:
                print(
                    'ERROR! PyDE is not installed, run first with optimize instead of emcee'
                )
                quit()

            if not os.path.exists(mc.pyde_dir_output):
                os.makedirs(mc.pyde_dir_output)

            print('PyDE running')
            sys.stdout.flush()

            de = DiffEvol(mc,
                          mc.bounds,
                          mc.emcee_parameters['nwalkers'],
                          maximize=True)
            de.optimize(int(mc.pyde_parameters['ngen']))

            population = de.population
            starting_point = np.median(population, axis=0)

            theta_dict = results_analysis.get_theta_dictionary(mc)
            """ bounds redefinition and fix for PyDE anomalous results """
            if mc.recenter_bounds_flag:
                pyde_save_to_pickle(mc,
                                    population,
                                    starting_point,
                                    theta_dict,
                                    prefix='orig')

                mc.recenter_bounds(starting_point)
                population = mc.fix_population(starting_point, population)
                starting_point = np.median(population, axis=0)

                print('Boundaries redefined after PyDE output')

            pyde_save_to_pickle(mc, population, starting_point, theta_dict)

            print('PyDE completed')
            sys.stdout.flush()

    results_analysis.results_resumen(mc,
                                     starting_point,
                                     compute_lnprob=True,
                                     is_starting_point=True)

    if mc.use_threading_pool:
        if mc.emcee_parameters['version'] == '2':
            threads_pool = emcee.interruptible_pool.InterruptiblePool(
                mc.emcee_parameters['nwalkers'])
        else:
            from multiprocessing.pool import Pool as InterruptiblePool
            threads_pool = InterruptiblePool(mc.emcee_parameters['nwalkers'])

    if mc.emcee_parameters['multirun'] and not reloaded_emcee_multirun:

        for ii in range(0, mc.emcee_parameters['multirun_iter']):
            print('emcee exploratory run #', ii, ' of ',
                  mc.emcee_parameters['multirun_iter'])
            # sampler = emcee.EnsembleSampler(mc.emcee_parameters['nwalkers'], mc.ndim, mc,
            #                                 threads=mc.emcee_parameters['nwalkers'])
            if mc.use_threading_pool:
                sampler = emcee.EnsembleSampler(
                    mc.emcee_parameters['nwalkers'],
                    mc.ndim,
                    mc,
                    pool=threads_pool)
            else:
                sampler = emcee.EnsembleSampler(
                    mc.emcee_parameters['nwalkers'], mc.ndim, mc)

            population, prob, state = sampler.run_mcmc(
                population, mc.emcee_parameters['multirun'])
            flatchain = emcee_flatchain(sampler.chain,
                                        mc.emcee_parameters['nburn'],
                                        mc.emcee_parameters['thin'])
            results_analysis.results_resumen(mc, flatchain)

            max_ind = np.argmax(prob)
            starting_point = population[max_ind, :]

            population = np.asarray([
                starting_point + 1e-4 * np.random.randn(mc.ndim)
                for i in range(mc.emcee_parameters['nwalkers'])
            ])
            sampler.reset()

            theta_dict = results_analysis.get_theta_dictionary(mc)
            emcee_save_to_cpickle(mc,
                                  starting_point,
                                  population,
                                  prob,
                                  state,
                                  sampler,
                                  theta_dict,
                                  prefix='MR_' + repr(ii))

        emcee_save_to_cpickle(mc,
                              starting_point,
                              population,
                              prob,
                              state,
                              sampler,
                              theta_dict,
                              prefix='MR')

        flatchain = emcee_flatchain(sampler.chain,
                                    mc.emcee_parameters['nburn'],
                                    mc.emcee_parameters['thin'])
        results_analysis.print_integrated_ACF(sampler.chain, theta_dict,
                                              mc.emcee_parameters['thin'])
        results_analysis.results_resumen(mc, flatchain)

        print('emcee exploratory runs completed')
        sys.stdout.flush()

    print()
    print('Running emcee')
    state = None

    if mc.use_threading_pool:
        sampler = emcee.EnsembleSampler(mc.emcee_parameters['nwalkers'],
                                        mc.ndim,
                                        mc,
                                        pool=threads_pool)
    else:
        sampler = emcee.EnsembleSampler(mc.emcee_parameters['nwalkers'],
                                        mc.ndim, mc)

    if mc.emcee_parameters['nsave'] > 0:
        print()
        print(' Saving temporary steps')
        niter = int(mc.emcee_parameters['nsteps'] /
                    mc.emcee_parameters['nsave'])
        sampled = 0
        for i in range(0, niter):
            population, prob, state = sampler.run_mcmc(
                population,
                mc.emcee_parameters['nsave'],
                thin=mc.emcee_parameters['thin'],
                rstate0=state)
            sampled += mc.emcee_parameters['nsave']
            theta_dict = results_analysis.get_theta_dictionary(mc)
            emcee_save_to_cpickle(mc,
                                  starting_point,
                                  population,
                                  prob,
                                  state,
                                  sampler,
                                  theta_dict,
                                  samples=sampled)

            flatchain = emcee_flatchain(sampler.chain,
                                        mc.emcee_parameters['nburn'],
                                        mc.emcee_parameters['thin'])
            results_analysis.print_integrated_ACF(sampler.chain, theta_dict,
                                                  mc.emcee_parameters['thin'])
            results_analysis.results_resumen(mc, flatchain)

            print()
            print(sampled, '  steps completed, average lnprob:, ',
                  np.median(prob))

            sys.stdout.flush()

    else:
        population, prob, state = sampler.run_mcmc(
            population,
            mc.emcee_parameters['nsteps'],
            thin=mc.emcee_parameters['thin'])

        theta_dict = results_analysis.get_theta_dictionary(mc)
        emcee_save_to_cpickle(mc, starting_point, population, prob, state,
                              sampler, theta_dict)

        flatchain = emcee_flatchain(sampler.chain,
                                    mc.emcee_parameters['nburn'],
                                    mc.emcee_parameters['thin'])
        results_analysis.print_integrated_ACF(sampler.chain, theta_dict,
                                              mc.emcee_parameters['thin'])
        results_analysis.results_resumen(mc, flatchain)
    print()
    print('emcee completed')

    if mc.use_threading_pool:
        # close the pool of threads
        threads_pool.close()
        threads_pool.terminate()
        threads_pool.join()
    """ A dummy file is created to let the cpulimit script to proceed with the next step"""
    emcee_create_dummy_file(mc)

    if return_output:
        return mc, sampler.chain, sampler.lnprobability
Exemplo n.º 7
0
print 'Variable bounds:', mc.bounds
print
mc.nwalkers = mc.ndim * mc.npop_mult
if mc.nwalkers % 2 == 1: mc.nwalkers += 1

print 'Nwalkers = ', mc.nwalkers

if os.path.isfile(dir_output + 'pyde_pops.pick'):
    population = pickle.load(open(dir_output + 'pyde_pops.pick', 'rb'))
    pyde_mean = np.median(population, axis=0)
    #pyde_mean = pickle.load(open(dir_output + 'pyde_mean.pick', 'rb'))
    mc.recenter_bounds(pyde_mean, population)

else:
    print 'PyDE'
    de = DiffEvol(mc, mc.bounds, mc.nwalkers, maximize=True)
    de.optimize(mc.ngen)
    print 'PyDE completed'

    population = de.population
    pyde_mean = np.median(population, axis=0)
    pickle.dump(pyde_mean, open(dir_output + 'pyde_mean.pick', 'wb'))

    #np.savetxt(dir_output + 'pyDEout_original_bounds.dat', mc.bounds)
    #np.savetxt(dir_output + 'pyDEout_original_pops.dat', population)

    # bounds redefinition and fix for PyDE anomalous results
    if mc.recenter_bounds_flag:
        pickle.dump(mc.bounds, open(dir_output + 'bounds_orig.pick', 'wb'))
        pickle.dump(population, open(dir_output + 'pyde_pops_orig.pick', 'wb'))
        mc.recenter_bounds(pyde_mean, population)
Exemplo n.º 8
0
Arquivo: test_1.py Projeto: OxES/PyDE
import numpy as np
import matplotlib.pyplot as pl

from pyde.de import DiffEvol

class Model(object):
    def __init__(self, x):
        self.x = x

    def __call__(self, a, p0, f):
        return a*np.sin(p0+2*np.pi*self.x*f)

if __name__ == '__main__':

    x = np.linspace(0,10,100)
    m = Model(x)
    y = m(2,1,0.25) + np.random.normal(0,0.5,size=x.size)

    de = DiffEvol(lambda pv: ((y-m(*pv))**2).sum(), [[1,3],[0,2],[0,4]], 50)

    ## Run n number of generations
    res = de.optimize(250)

    ## Or use as an iterator
    for res in de(2):
        print res

    pl.plot(x,y,'.k')
    pl.plot(x,m(*res[0]),'k')
    pl.show()
Exemplo n.º 9
0
mc.create_bounds()
print 'Dimensions = ', mc.ndim
print '   '
print 'Variable list:', mc.variable_list
print
print 'Variable bounds:', mc.bounds
print
mc.nwalkers = mc.ndim * mc.npop_mult
if mc.nwalkers%2 == 1: mc.nwalkers += 1

print 'Nwalkers = ', mc.nwalkers


print 'PyDE'
de = DiffEvol(mc, mc.bounds, mc.nwalkers, maximize=True)
de.optimize(mc.ngen)
print 'PyDE completed'

population = de.population
pyde_mean = np.mean(population, axis=0)
pickle.dump(pyde_mean, open(dir_output + 'pyde_mean.pick', 'wb'))

#np.savetxt(dir_output + 'pyDEout_original_bounds.dat', mc.bounds)
#np.savetxt(dir_output + 'pyDEout_original_pops.dat', population)

# bounds redefinition and fix for PyDE anomalous results
if mc.recenter_bounds_flag:
    pickle.dump(mc.bounds, open(dir_output + 'bounds_orig.pick', 'wb'))
    pickle.dump(population, open(dir_output + 'pyde_pops_orig.pick', 'wb'))
    mc.recenter_bounds(pyde_mean, population)
Exemplo n.º 10
0
else:
    if not os.path.exists(pyde_dir_output):
        os.makedirs(pyde_dir_output)

    if os.path.isfile(pyde_dir_output + 'pyde_pops.pick'):
        print os.path.isfile(pyde_dir_output + 'pyde_pops.pick')
        population = pickle.load(open(pyde_dir_output + 'pyde_pops.pick',
                                      'rb'))
        starting_point = np.median(population, axis=0)
        mc.recenter_bounds(starting_point, population)

    else:
        print 'PyDE'
        de = DiffEvol(mc,
                      mc.bounds,
                      mc.emcee_parameters['nwalkers'],
                      maximize=True)
        de.optimize(mc.pyde_parameters['ngen'])
        print 'PyDE completed'

        population = de.population
        starting_point = np.median(population, axis=0)
        pickle.dump(starting_point,
                    open(pyde_dir_output + 'pyde_mean.pick', 'wb'))

        #np.savetxt(pyde_dir_output + 'pyDEout_original_bounds.dat', mc.bounds)
        #np.savetxt(pyde_dir_output + 'pyDEout_original_pops.dat', population)

        # bounds redefinition and fix for PyDE anomalous results
        if mc.recenter_bounds_flag:
            pickle.dump(mc.bounds,
Exemplo n.º 11
0
class Model(object):
    def __init__(self, x):
        self.x = x

    def __call__(self, a, p0, f):
        return a * np.sin(p0 + 2 * np.pi * self.x * f)


if __name__ == '__main__':
    niter = 250

    x = np.linspace(0, 10, 100)
    m = Model(x)
    y = m(2, 1, 0.25) + np.random.normal(0, 0.5, size=x.size)

    de = DiffEvol(lambda pv: ((y - m(*pv))**2).sum(), [[1, 3], [0, 2], [0, 4]],
                  50)

    ## Run n number of generations
    tstart = time()
    res = de.optimize(niter)
    print(((time() - tstart) / niter))

    ## Or use as an iterator
    for res in de(2):
        print(res)

    pl.plot(x, y, '.k')
    pl.plot(x, m(*res[0]), 'k')
    pl.show()
        if fwp_flag[nk]:
            for ii in xrange(0, np.sum(n_amp[0:n_fwa])):
                bounds[ip + ii, :] = fwa_bounds[:]
            ip += np.sum(n_amp[0:n_fwa])

        if bsp_flag[nk]:
            for ii in xrange(0, np.sum(n_amp[0:n_bsa])):
                bounds[ip + ii, :] = bsa_bounds[:]
            ip +=  np.sum(n_amp[0:n_bsa])

print 'BOUNDS'
print bounds

print 'PyDE'
de = DiffEvol(lnprob_PyDE, bounds, npop, maximize=True)
de.optimize(ngen)
print 'PyDE completed'

np.savetxt('output/' + planet_name + '_pyDEout_output_bounds.dat',bounds)
np.savetxt('output/' + planet_name + '_pyDEout_output_pops.dat',de.population)


pyde_mean = np.mean(de.population, axis=0)
print pyde_mean
np.savetxt('output/' + planet_name + '_pyDEout_mean.dat',pyde_mean)

# fix for PyDE anomalous results
for ii in xrange(0,ndim):
    if np.amax(de.population[:,ii])-np.amin(de.population[:,ii]) < 10e-7 :
        range_restricted = (bounds[ii,1]-bounds[ii,0])/1000.
Exemplo n.º 13
0
        if fwp_flag[nk]:
            for ii in xrange(0, np.sum(n_amp[0:n_fwa])):
                bounds[ip + ii, :] = fwa_bounds[:]
            ip += np.sum(n_amp[0:n_fwa])

        if bsp_flag[nk]:
            for ii in xrange(0, np.sum(n_amp[0:n_bsa])):
                bounds[ip + ii, :] = bsa_bounds[:]
            ip += np.sum(n_amp[0:n_bsa])

print 'BOUNDS'
print bounds

print 'PyDE'
de = DiffEvol(lnprob_PyDE, bounds, npop, maximize=True)
de.optimize(ngen)
print 'PyDE completed'

np.savetxt('output/' + planet_name + '_pyDEout_output_bounds.dat', bounds)
np.savetxt('output/' + planet_name + '_pyDEout_output_pops.dat', de.population)

pyde_mean = np.mean(de.population, axis=0)
print pyde_mean
np.savetxt('output/' + planet_name + '_pyDEout_mean.dat', pyde_mean)

# fix for PyDE anomalous results
for ii in xrange(0, ndim):
    if np.amax(de.population[:, ii]) - np.amin(de.population[:, ii]) < 10e-7:
        range_restricted = (bounds[ii, 1] - bounds[ii, 0]) / 1000.
        min_bound = np.maximum((pyde_mean[ii] - range_restricted / 2.0),
Exemplo n.º 14
0
def pe_runner(lpfun, basename, **kwargs):
    result_dir = kwargs.get('result_dir', '.')
    plot_dir = kwargs.get('plot_dir', '.')
    runname = kwargs.get('run_name', 'default')
    n_de_iters = kwargs.get('n_de_iterations', 150)
    n_mc_iters = kwargs.get('n_mc_iterations', 150)
    pop_size = kwargs.get('pop_size', 100)
    thinning = kwargs.get('thinning', 1)
    mc_continue = kwargs.get('mc_continues', True)
    update_interval = kwargs.get('update_interval', 60)

    do_de = kwargs.get('do_de', False)
    do_mc = kwargs.get('do_mc', False)

    pplot = ProgressPlotter(lpfun, '{:s}_{:s}'.format(basename, runname),
                            plot_dir)

    de_res_fname = join(result_dir,
                        '{:s}_{:s}_de.pkl'.format(basename, runname))
    mc_res_fname = join(result_dir,
                        '{:s}_{:s}_mc.pkl'.format(basename, runname))
    mc_bck_fname = join(result_dir,
                        '{:s}_{:s}_mc_back.pkl'.format(basename, runname))

    de_res_exists = exists(de_res_fname)
    mc_res_exists = exists(mc_res_fname)

    ## ================================================================================
    ## DE - Run Differential Evolution
    ## ================================================================================
    if do_de or not de_res_exists:
        t_start = time()
        de = DiffEvol(lambda pv: -lpfun(pv), lpfun.ps.bounds, pop_size)
        best_ll = 1e80
        for i, res in enumerate(de(n_de_iters)):
            if res[1] < best_ll:
                print(i, res[1])
                best_ll = res[1]

        with open(de_res_fname, 'w') as fout:
            dump({
                'population': de.population,
                'best': de.minimum_location
            }, fout)
        de_population, de_best_fit = de.population, de.minimum_location
        print("time taken: {:4.2f}".format(time() - t_start))
    else:
        with open(de_res_fname, 'r') as fin:
            de_res = load(fin)
        de_population, de_best_fit = de_res['population'], de_res['best']

    if not do_mc and not mc_res_exists:
        return de_population, de_best_fit

    ## ================================================================================
    ## MCMC - Run emcee
    ## ================================================================================
    if do_mc or not mc_res_exists:
        t_iteration_start = time()
        t_last_update = time()
        t_start = time()

        if mc_res_exists and mc_continue:
            print('Continuing from the previous MCMC run.')
            with open(mc_res_fname, 'r') as fin:
                pv0 = load(fin)['chain'][:, -1, :]
        else:
            pv0 = de_population

        if pv0.shape[0] > pop_size:
            pv0 = pv0[:pop_size, :].copy()
        elif pv0.shape[0] < pop_size:
            pv0 = np.tile(
                pv0,
                [np.ceil(pop_size / pv0.shape[0]), 1])[:pop_size, :].copy()

        #pv0[:,14] = np.random.uniform(0.01, 0.98, size=300)
        #m = pv0[:,2] > 0.03
        #pv0[m,:] = pv0[~m,:][:m.sum(),:]
        #pv0[:,2] = np.random.uniform(0.005, 0.1, size=300)
        #print pv0.shape;exit()

        sampler = emcee.EnsembleSampler(pop_size, lpfun.ps.ndim, lpfun)
        for i, e in enumerate(
                sampler.sample(pv0, iterations=n_mc_iters, thin=thinning)):
            t_cur = time()
            print(
                "{:4d}/{:4d}  Secs/iteration {:6.2f}  Last update {:6.2f} s ago  Total time {:6.2f} s   Acceptance {:6.3f}"
                .format(i + 1, n_mc_iters, t_cur - t_iteration_start,
                        t_cur - t_last_update, t_cur - t_start,
                        sampler.acceptance_fraction.mean()))
            if i != 0 and (t_cur - t_last_update > update_interval):
                pplot.update(sampler.chain, i // thinning)
                t_last_update = t_cur
                with open(mc_bck_fname, 'w') as fout:
                    dump(
                        {
                            'chain': sampler.chain,
                            'logl': sampler.lnprobability
                        }, fout)
            t_iteration_start = t_cur

        with open(mc_res_fname, 'w') as fout:
            dump({'chain': sampler.chain, 'logl': sampler.lnprobability}, fout)
        chain = sampler.chain

    else:
        with open(mc_res_fname, 'r') as fin:
            chain = load(fin)['chain']

    return chain