Ejemplo n.º 1
0
def calculate_point_source_flux(*args, **kwargs):

    custom_warnings.warn("The use of calculate_point_source_flux is deprecated. Please use the .get_point_source_flux()"
                         " method of the JointLikelihood.results or the BayesianAnalysis.results member. For example:"
                         " jl.results.get_point_source_flux().")

    return _calculate_point_source_flux(*args, **kwargs)
Ejemplo n.º 2
0
    def get_randomized_background_counts(self):
        # Now randomize the expectations.

        _, background_model_counts = self.get_current_value()

        # We cannot generate variates with zero sigma. They variates from those channel will always be zero
        # This is a limitation of this whole idea. However, remember that by construction an error of zero
        # it is only allowed when the background counts are zero as well.
        idx = (self._spectrum_plugin.background_count_errors > 0)

        randomized_background_counts = np.zeros_like(background_model_counts)

        randomized_background_counts[idx] = np.random.normal(
            loc=background_model_counts[idx],
            scale=self._spectrum_plugin.background_count_errors[idx])

        # Issue a warning if the generated background is less than zero, and fix it by placing it at zero

        idx = (randomized_background_counts < 0)  # type: np.ndarray

        negative_background_n = np.sum(idx)

        if negative_background_n > 0:
            custom_warnings.warn("Generated background has negative counts "
                                 "in %i channels. Fixing them to zero" %
                                 (negative_background_n))

            randomized_background_counts[idx] = 0

        return randomized_background_counts
Ejemplo n.º 3
0
    def bin_by_bayesian_blocks(cls,
                               arrival_times,
                               p0,
                               bkg_integral_distribution=None):
        """Divide a series of events characterized by their arrival time in blocks
        of perceptibly constant count rate. If the background integral distribution
        is given, divide the series in blocks where the difference with respect to
        the background is perceptibly constant.

        :param arrival_times: An iterable (list, numpy.array...) containing the arrival
                         time of the events.
                         NOTE: the input array MUST be time-ordered, and without
                         duplicated entries. To ensure this, you may execute the
                         following code:
                         tt_array = numpy.asarray(self._arrival_times)
                         tt_array = numpy.unique(tt_array)
                         tt_array.sort()
                         before running the algorithm.
        :param p0: The probability of finding a variations (i.e., creating a new
                      block) when there is none. In other words, the probability of
                      a Type I error, i.e., rejecting the null-hypothesis when is
                      true. All found variations will have a post-trial significance
                      larger than p0.
        :param bkg_integral_distribution : the integral distribution for the
                      background counts. It must be a function of the form f(x),
                      which must return the integral number of counts expected from
                      the background component between time 0 and x.

        """

        try:

            final_edges = bayesian_blocks(
                arrival_times,
                arrival_times[0],
                arrival_times[-1],
                p0,
                bkg_integral_distribution,
            )

        except Exception as e:

            if "duplicate" in str(e):

                custom_warnings.warn(
                    "There where possible duplicate time tags in the data. We will try to run a different algorithm"
                )

                final_edges = bayesian_blocks_not_unique(
                    arrival_times, arrival_times[0], arrival_times[-1], p0)
            else:

                print(e)

                raise RuntimeError()

        starts = np.asarray(final_edges)[:-1]
        stops = np.asarray(final_edges)[1:]

        return cls.from_starts_and_stops(starts, stops)
Ejemplo n.º 4
0
    def _source_is_valid(self, source):
        """
        checks if source name is valid for the 3FGL catalog

        :param source: source name
        :return: bool
        """

        warn_string = (
            "The trigger %s is not valid. Must be in the form '3FGL J0000.0+0000'"
            % source
        )

        match = _3FGL_name_match.match(source)

        if match is None:

            custom_warnings.warn(warn_string)

            answer = False

        else:

            answer = True

        return answer
Ejemplo n.º 5
0
def _gbm_and_lle_valid_source_check(source):
    """
    checks if source name is valid for both GBM and LLE data

    :param source: source name
    :return: bool
    """

    warn_string = (
        "The trigger %s is not valid. Must be in the form GRB080916009" % source
    )

    match = _trigger_name_match.match(source)

    if match is None:

        custom_warnings.warn(warn_string)

        answer = False

    else:

        answer = True

    return answer
Ejemplo n.º 6
0
    def set_uniform_priors(self):
        """
        Automatically set all parameters to uniform or log-uniform priors. The latter is used when the range spanned
        by the parameter is larger than 2 orders of magnitude.

        :return: (none)
        """

        for parameter_name, parameter in self._free_parameters.iteritems():

            if parameter.min_value is None or parameter.max_value is None:

                custom_warnings.warn("Cannot decide the prior for parameter %s, since it has no "
                                     "minimum or no maximum value (or both)")

                continue

            n_orders_of_magnitude = numpy.log10(parameter.max_value - parameter.min_value)

            if n_orders_of_magnitude > 2:

                print("Using log-uniform prior for %s" % parameter_name)

                parameter.prior = log_uniform_prior(lower_bound=parameter.min_value,
                                                    upper_bound=parameter.max_value)

            else:

                print("Using uniform prior for %s" % parameter_name)

                parameter.prior = uniform_prior(lower_bound=parameter.min_value,
                                                upper_bound=parameter.max_value,
                                                value=1.0)
Ejemplo n.º 7
0
    def get_randomized_background_counts(self):
        # Now randomize the expectations.

        _, background_model_counts = self.get_current_value()

        # We cannot generate variates with zero sigma. They variates from those channel will always be zero
        # This is a limitation of this whole idea. However, remember that by construction an error of zero
        # it is only allowed when the background counts are zero as well.
        idx = (self._spectrum_plugin.background_count_errors > 0)

        randomized_background_counts = np.zeros_like(background_model_counts)

        randomized_background_counts[idx] = np.random.normal(loc=background_model_counts[idx],
                                                             scale=self._spectrum_plugin.background_count_errors[idx])

        # Issue a warning if the generated background is less than zero, and fix it by placing it at zero

        idx = (randomized_background_counts < 0)  # type: np.ndarray

        negative_background_n = np.sum(idx)

        if negative_background_n > 0:
            custom_warnings.warn("Generated background has negative counts "
                                 "in %i channels. Fixing them to zero" % (negative_background_n))

            randomized_background_counts[idx] = 0

        return randomized_background_counts
Ejemplo n.º 8
0
    def __init__(self, start, stop, parameter_values, likelihood_values,
                 n_integration_points):

        # Make sure there is no NaN or infinity
        assert np.all(np.isfinite(
            likelihood_values)), "Infinity or NaN in likelihood values"

        likelihood_values = np.asarray(likelihood_values)
        parameter_values = np.asarray(parameter_values)

        self._start = start
        self._stop = stop

        # Make sure the number of integration points is uneven, and that there are at minimum 11 points
        # n_integration_points = max(int(n_integration_points), 11)

        if n_integration_points % 2 == 0:

            # n_points is even, it shouldn't be otherwise things like Simpson rule will have problems
            n_integration_points += 1

            custom_warnings.warn(
                "The number of integration points should not be even. Adding +1"
            )

        self._n_integration_points = int(n_integration_points)

        # Build interpolation of the likelihood curve
        self._minus_likelihood_interp = scipy.interpolate.InterpolatedUnivariateSpline(
            np.log10(parameter_values), -likelihood_values, k=1, ext=0)

        # Find maximum of loglike
        idx = likelihood_values.argmax()

        self._min_par_value = parameter_values.min()
        self._max_par_value = parameter_values.max()

        res = scipy.optimize.minimize_scalar(
            self._minus_likelihood_interp,
            bounds=(np.log10(self._min_par_value),
                    np.log10(self._max_par_value)),
            method="bounded",
            options={
                "maxiter": 10000,
                "disp": True,
                "xatol": 1e-3
            },
        )

        # res = scipy.optimize.minimize(self._minus_likelihood_interp, x0=[np.log10(parameter_values[idx])],
        #                               jac=lambda x:self._minus_likelihood_interp.derivative(1)(x),
        #                                      # bounds=(self._min_par_value, self._max_par_value),
        #                                      # method='bounded',
        #                                      tol=1e-3,
        #                                      options={'maxiter': 10000, 'disp': True})

        assert res.success, "Could not find minimum"

        self._minimum = (10**res.x, float(res.fun))
Ejemplo n.º 9
0
    def _check_fullsky(self, method_name):

        if not self._fullsky:

            custom_warnings.warn("Attempting to use method %s, but fullsky=False during construction. "
                                 "This might fail. If it does, specify `fullsky=True` when instancing "
                                 "the plugin and try again." % method_name,
                                 NoFullSky)
Ejemplo n.º 10
0
    def _check_fullsky(self, method_name):

        if not self._fullsky:

            custom_warnings.warn(
                "Attempting to use method %s, but fullsky=False during construction. "
                "This might fail. If it does, specify `fullsky=True` when instancing "
                "the plugin and try again." % method_name, NoFullSky)
Ejemplo n.º 11
0
    def get_model(self, use_association_name=True):

        assert (
            self._last_query_results is not None
        ), "You have to run a query before getting a model"

        # Loop over the table and build a source for each entry
        sources = []
        source_names = []
        for name, row in self._last_query_results.T.items():
            if name[-1] == "e":
                # Extended source
                custom_warnings.warn(
                    "Source %s is extended, support for extended source is not here yet. I will ignore"
                    "it" % name
                )

            # If there is an association and use_association is True, use that name, otherwise the 3FGL name
            if row["assoc_name"] != "" and use_association_name:

                this_name = row["assoc_name"]

                # The crab is the only source which is present more than once in the 3FGL

                if this_name == "Crab Nebula":

                    if name[-1] == "i":

                        this_name = "Crab_IC"

                    elif name[-1] == "s":

                        this_name = "Crab_synch"

                    else:

                        this_name = "Crab_pulsar"
            else:

                this_name = name

            # in the 4FGL name there are more sources with the same name: this nwill avod any duplicates:
            i = 1
            while this_name in source_names:
                this_name += str(i)
                i += 1
                pass
            # By default all sources are fixed. The user will free the one he/she will need

            source_names.append(this_name)

            this_source = _get_point_source_from_3fgl(this_name, row, fix=True)

            sources.append(this_source)

        return ModelFrom3FGL(self.ra_center, self.dec_center, *sources)
Ejemplo n.º 12
0
    def check_roi_inside_model(self):

        active_pixels = self.active_pixels(self._original_nside)

        radius = np.rad2deg(self._model_radius_radians)
        ra, dec = self.ra_dec_center
        temp_roi =  HealpixConeROI(data_radius = radius , model_radius=radius, ra=ra, dec=dec)

        model_pixels = temp_roi.active_pixels( self._original_nside )

        if not all(p in model_pixels for p in active_pixels):
            custom_warnings.warn("Some pixels inside your ROI are not contained in the model map.")
Ejemplo n.º 13
0
 def get_number_of_data_points( self ):
     """
     Number of data point = number of pixels.
     Implemented in liff as the number of pixels in the ROI per analysis bin.
     """        
     try:
         pixels_per_bin =  np.array( self._theLikeHAWC.GetNumberOfPixels() )
         return int(np.sum( pixels_per_bin ))
     except AttributeError:
         custom_warnings.warn(
           "_theLikeHAWC.GetNumberOfPixels() not available, values for statistical measurements such as AIC or BIC are unreliable. Please update your aerie version." )
         return 1
Ejemplo n.º 14
0
    def get_model(self, use_association_name=True):

        assert self._last_query_results is not None, "You have to run a query before getting a model"

        # Loop over the table and build a source for each entry
        sources = []
        source_names=[]
        for name, row in self._last_query_results.T.iteritems():
            if name[-1] == 'e':
                # Extended source
                custom_warnings.warn("Source %s is extended, support for extended source is not here yet. I will ignore"
                                     "it" % name)

            # If there is an association and use_association is True, use that name, otherwise the 3FGL name
            if row['assoc_name'] != '' and use_association_name:

                this_name = row['assoc_name']

                # The crab is the only source which is present more than once in the 3FGL

                if this_name == 'Crab Nebula':

                    if name[-1] == 'i':

                        this_name = "Crab_IC"

                    elif name[-1] == "s":

                        this_name = "Crab_synch"

                    else:

                        this_name = "Crab_pulsar"
            else:

                this_name = name

            # in the 4FGL name there are more sources with the same name: this nwill avod any duplicates:
            i=1
            while this_name in source_names:
                this_name+=str(i)
                i+=1
                pass
            # By default all sources are fixed. The user will free the one he/she will need

            source_names.append(this_name)

            this_source = _get_point_source_from_3fgl(this_name, row, fix=True)

            sources.append(this_source)

        return ModelFrom3FGL(self.ra_center, self.dec_center, *sources)
Ejemplo n.º 15
0
 def get_number_of_data_points(self):
     """
     Number of data point = number of pixels.
     Implemented in liff as the number of pixels in the ROI per analysis bin.
     """
     try:
         pixels_per_bin = np.array(self._theLikeHAWC.GetNumberOfPixels())
         return int(np.sum(pixels_per_bin))
     except AttributeError:
         custom_warnings.warn(
             "_theLikeHAWC.GetNumberOfPixels() not available, values for statistical measurements such as AIC or BIC are unreliable. Please update your aerie version."
         )
         return 1
Ejemplo n.º 16
0
    def __init__(self, response_file_name, dec_bins, response_bins):

        self._response_file_name = response_file_name
        self._dec_bins = dec_bins
        self._response_bins = response_bins

        if len(dec_bins) < 2:
            custom_warnings.warn(
                "Only {0} dec bins given in {1}, will not try to interpolate.".
                format(len(dec_bins), response_file_name))
            custom_warnings.warn(
                "Single-dec-bin mode is intended for development work only at this time and may not work with extended sources."
            )
Ejemplo n.º 17
0
    def restore_best_fit(self):
        """
        Restore the model to its best fit

        :return: (none)
        """

        if self._minimizer:

            self._minimizer.restore_best_fit()

        else:

            custom_warnings.warn("Cannot restore best fit, since fit has not been executed.")
Ejemplo n.º 18
0
    def __init__(self, start, stop, parameter_values, likelihood_values, n_integration_points):

        # Make sure there is no NaN or infinity
        assert np.all(np.isfinite(likelihood_values)), "Infinity or NaN in likelihood values"

        likelihood_values = np.asarray(likelihood_values)
        parameter_values = np.asarray(parameter_values)

        self._start = start
        self._stop = stop

        # Make sure the number of integration points is uneven, and that there are at minimum 11 points
        #n_integration_points = max(int(n_integration_points), 11)

        if n_integration_points % 2 == 0:

            # n_points is even, it shouldn't be otherwise things like Simpson rule will have problems
            n_integration_points += 1

            custom_warnings.warn("The number of integration points should not be even. Adding +1")

        self._n_integration_points = int(n_integration_points)

        # Build interpolation of the likelihood curve
        self._minus_likelihood_interp = scipy.interpolate.InterpolatedUnivariateSpline(np.log10(parameter_values),
                                                                                       -likelihood_values,
                                                                                       k=1, ext=0)

        # Find maximum of loglike
        idx = likelihood_values.argmax()

        self._min_par_value = parameter_values.min()
        self._max_par_value = parameter_values.max()

        res = scipy.optimize.minimize_scalar(self._minus_likelihood_interp,
                                      bounds=(np.log10(self._min_par_value), np.log10(self._max_par_value)),
                                      method='bounded',
                                      options={'maxiter': 10000, 'disp': True, 'xatol': 1e-3})

        # res = scipy.optimize.minimize(self._minus_likelihood_interp, x0=[np.log10(parameter_values[idx])],
        #                               jac=lambda x:self._minus_likelihood_interp.derivative(1)(x),
        #                                      # bounds=(self._min_par_value, self._max_par_value),
        #                                      # method='bounded',
        #                                      tol=1e-3,
        #                                      options={'maxiter': 10000, 'disp': True})

        assert res.success, "Could not find minimum"

        self._minimum = (10**res.x, float(res.fun))
Ejemplo n.º 19
0
    def __init__(self, name, veritas_root_data):

        # Open file

        f = ROOT.TFile(veritas_root_data)

        try:

            # Loop over the runs
            keys = get_list_of_keys(f)

        finally:

            f.Close()

        # Get the names of all runs included

        run_names = [x for x in keys if x.find("run") == 0]

        self._runs_like = collections.OrderedDict()

        for run_name in run_names:

            # Build the VERITASRun class
            this_run = VERITASRun(veritas_root_data, run_name)

            this_run.display()

            if this_run.total_counts == 0 or this_run.total_background_counts == 0:

                custom_warnings.warn(
                    "%s has 0 source or bkg counts, cannot use it." % run_name
                )
                continue

            else:

                # Get background spectrum and observation spectrum (with response)
                # this_observation = this_run.get_spectrum()
                # this_background = this_run.get_background_spectrum()
                #
                # self._runs_like[run_name] = DispersionSpectrumLike(run_name,
                #                                                    this_observation,
                #                                                    this_background)
                #
                # self._runs_like[run_name].set_active_measurements("c50-c130")
                self._runs_like[run_name] = this_run

        super(VERITASLike, self).__init__(name, {})
Ejemplo n.º 20
0
    def restore_best_fit(self):
        """
        Restore the model to its best fit

        :return: (none)
        """

        if self._minimizer:

            self._minimizer.restore_best_fit()

        else:

            custom_warnings.warn(
                "Cannot restore best fit, since fit has not been executed.")
Ejemplo n.º 21
0
    def get_source_map(self, response_bin_id, tag=None):

        # Get current point source position
        # NOTE: this might change if the point source position is free during the fit,
        # that's why it is here

        ra_src, dec_src = self._source.position.ra.value, self._source.position.dec.value

        if (ra_src, dec_src) != self._last_processed_position:

            # Position changed (or first iteration), let's update the dec bins
            self._update_dec_bins(dec_src)

            self._last_processed_position = (ra_src, dec_src)

        # Get the current response bin
        response_energy_bin = self._response_energy_bins[response_bin_id]
        psf_interpolator = self._psf_interpolators[response_bin_id]

        # Get the PSF image
        # This is cached inside the PSF class, so that if the position doesn't change this line
        # is very fast
        this_map = psf_interpolator.point_source_image(ra_src, dec_src)

        # Check that the point source is contained in the ROI, if not print a warning
        if not np.isclose(this_map.sum(), 1.0, rtol=1e-2):

            custom_warnings.warn(
                "PSF for source %s is not entirely contained in ROI" %
                self._name)

        # Compute the fluxes from the spectral function at the same energies as the simulated function
        energy_centers_keV = response_energy_bin.sim_energy_bin_centers * 1e9  # astromodels expects energies in keV

        # This call needs to be here because the parameters of the model might change,
        # for example during a fit

        source_diff_spectrum = self._source(energy_centers_keV, tag=tag)

        # Transform from keV^-1 cm^-2 s^-1 to TeV^-1 cm^-2 s^-1
        source_diff_spectrum *= 1e9

        # Re-weight the detected counts
        scale = source_diff_spectrum / response_energy_bin.sim_differential_photon_fluxes

        # Now return the map multiplied by the scale factor
        return np.sum(
            scale * response_energy_bin.sim_signal_events_per_bin) * this_map
Ejemplo n.º 22
0
    def get_model(self, use_association_name=True):

        assert self._last_query_results is not None, "You have to run a query before getting a model"

        # Loop over the table and build a source for each entry
        sources = []

        for name, row in self._last_query_results.T.iteritems():

            if name[-1] == 'e':
                # Extended source
                custom_warnings.warn(
                    "Source %s is extended, support for extended source is not here yet. I will ignore"
                    "it" % name)

            # If there is an association and use_association is True, use that name, otherwise the 3FGL name
            if row['assoc_name_1'] != '' and use_association_name:

                this_name = row['assoc_name_1']

                # The crab is the only source which is present more than once in the 3FGL

                if this_name == "Crab":

                    if name[-1] == 'i':

                        this_name = "Crab_IC"

                    elif name[-1] == "s":

                        this_name = "Crab_synch"

                    else:

                        this_name = "Crab_pulsar"

            else:

                this_name = name

            # By default all sources are fixed. The user will free the one he/she will need

            this_source = _get_point_source_from_3fgl(this_name, row, fix=True)

            sources.append(this_source)

        return ModelFrom3FGL(self.ra_center, self.dec_center, *sources)
Ejemplo n.º 23
0
    def __init__(self, name, veritas_root_data):

        # Open file

        f = ROOT.TFile(veritas_root_data)

        try:

            # Loop over the runs
            keys = get_list_of_keys(f)

        finally:

            f.Close()

        # Get the names of all runs included

        run_names = filter(lambda x: x.find("run") == 0, keys)

        self._runs_like = collections.OrderedDict()

        for run_name in run_names:

            # Build the VERITASRun class
            this_run = VERITASRun(veritas_root_data, run_name)

            this_run.display()

            if this_run.total_counts == 0 or this_run.total_background_counts == 0:

                custom_warnings.warn("%s has 0 source or bkg counts, cannot use it." % run_name)
                continue

            else:

                # Get background spectrum and observation spectrum (with response)
                # this_observation = this_run.get_spectrum()
                # this_background = this_run.get_background_spectrum()
                #
                # self._runs_like[run_name] = DispersionSpectrumLike(run_name,
                #                                                    this_observation,
                #                                                    this_background)
                #
                # self._runs_like[run_name].set_active_measurements("c50-c130")
                self._runs_like[run_name] = this_run

        super(VERITASLike, self).__init__(name, {})
Ejemplo n.º 24
0
    def _source_is_valid(self, source):

        warn_string = "The trigger %s is not valid. Must be in the form GRB080916009" % source

        match = _trigger_name_match.match(source)

        if match is None:

            custom_warnings.warn(warn_string)

            answer = False

        else:

            answer = True

        return answer
Ejemplo n.º 25
0
    def __init__(self, joint_likelihood_instance0, joint_likelihood_instance1):

        self._joint_likelihood_instance0 = (joint_likelihood_instance0
                                            )  # type: JointLikelihood
        self._joint_likelihood_instance1 = (joint_likelihood_instance1
                                            )  # type: JointLikelihood

        # Restore best fit and store the reference value for the likelihood
        self._joint_likelihood_instance0.restore_best_fit()
        self._joint_likelihood_instance1.restore_best_fit()

        self._reference_TS = 2 * (
            self._joint_likelihood_instance0.current_minimum -
            self._joint_likelihood_instance1.current_minimum)

        # Safety check that the user has provided the models in the right order
        if self._reference_TS < 0:

            custom_warnings.warn(
                "The reference TS is negative, either you specified the likelihood objects "
                "in the wrong order, or the fit for the alternative hyp. has failed. Since the "
                "two hyp. are nested, by definition the more complex hypothesis should give a "
                "better or equal fit with respect to the null hypothesis.")

        # Check that the dataset is the same

        if (self._joint_likelihood_instance1.data_list !=
                self._joint_likelihood_instance0.data_list):

            # Since this check might fail if the user loaded twice the same data, only issue a warning, instead of
            # an exception.

            custom_warnings.warn(
                "The data lists for the null hyp. and for the alternative hyp. seems to be different."
                " If you loaded twice the same data and made the same data selections, disregard this "
                "message. Otherwise, consider the fact that the LRT is meaningless if the two data "
                "sets are not exactly the same. We will use the data loaded as part of the null "
                "hypothesis JointLikelihood object",
                RuntimeWarning,
            )

        # For saving pha files
        self._save_pha = False
        self._data_container = []
Ejemplo n.º 26
0
    def _log_like(self, trial_values):
        """Compute the log-likelihood, used in the parallel tempering sampling"""

        # Compute the log-likelihood

        # Set the parameters to their trial values

        for i, (src_name, param_name) in enumerate(self._free_parameters.keys()):
            this_param = self._likelihood_model.parameters[src_name][param_name]

            this_param.setValue(trial_values[i])

        # Get the value of the log-likelihood for this parameters

        try:

            # Loop over each dataset and get the likelihood values for each set

            log_like_values = map(lambda dataset: dataset.get_log_like(), self.data_list.values())

        except ModelAssertionViolation:

            # Fit engine or sampler outside of allowed zone

            return -numpy.inf

        except:

            # We don't want to catch more serious issues

            raise

        # Sum the values of the log-like

        log_like = numpy.sum(log_like_values)

        if not numpy.isfinite(log_like):
            # Issue warning

            custom_warnings.warn("Likelihood value is infinite for parameters %s" % trial_values, LikelihoodIsInfinite)

            return -numpy.inf

        return log_like
Ejemplo n.º 27
0
    def compute_covariance_matrix(self, function, best_fit_parameters):
        """
        Compute the covariance matrix of this fit
        :param function: the loglike for the fit
        :param best_fit_parameters: the best fit parameters
        :return:
        """

        minima = np.zeros_like(best_fit_parameters) - 100
        maxima = np.zeros_like(best_fit_parameters) + 100

        try:

            hessian_matrix = get_hessian(function, best_fit_parameters, minima,
                                         maxima)

        except ParameterOnBoundary:

            custom_warnings.warn(
                "One or more of the parameters are at their boundaries. Cannot compute covariance and"
                " errors", CannotComputeCovariance)

            n_dim = len(best_fit_parameters)

            self._cov_matrix = np.zeros((n_dim, n_dim)) * np.nan

        # Invert it to get the covariance matrix

        try:

            covariance_matrix = np.linalg.inv(hessian_matrix)

            self._cov_matrix = covariance_matrix

        except:

            custom_warnings.warn(
                "Cannot invert Hessian matrix, looks like the matrix is singluar"
            )

            n_dim = len(best_fit_parameters)

            self._cov_matrix = np.zeros((n_dim, n_dim)) * np.nan
Ejemplo n.º 28
0
    def _log_like(self, trial_values):
        """Compute the log-likelihood"""

        # Get the value of the log-likelihood for this parameters

        try:

            # Loop over each dataset and get the likelihood values for each set

            log_like_values = [
                dataset.get_log_like()
                for dataset in list(self._data_list.values())
            ]

        except ModelAssertionViolation:

            # Fit engine or sampler outside of allowed zone

            return -np.inf

        except:

            # We don't want to catch more serious issues

            raise

        # Sum the values of the log-like

        log_like = np.sum(log_like_values)

        if not np.isfinite(log_like):
            # Issue warning

            custom_warnings.warn(
                "Likelihood value is infinite for parameters %s" %
                trial_values,
                LikelihoodIsInfinite,
            )

            return -np.inf

        return log_like
Ejemplo n.º 29
0
    def compute_covariance_matrix(self, function, best_fit_parameters):
        """
        Compute the covariance matrix of this fit
        :param function: the loglike for the fit
        :param best_fit_parameters: the best fit parameters
        :return:
        """

        minima = np.zeros_like(best_fit_parameters) - 100
        maxima = np.zeros_like(best_fit_parameters) + 100

        try:

            hessian_matrix = get_hessian(function, best_fit_parameters, minima, maxima)

        except ParameterOnBoundary:

            custom_warnings.warn("One or more of the parameters are at their boundaries. Cannot compute covariance and"
                                 " errors", CannotComputeCovariance)

            n_dim = len(best_fit_parameters)

            self._cov_matrix = np.zeros((n_dim, n_dim)) * np.nan

        # Invert it to get the covariance matrix

        try:

            covariance_matrix = np.linalg.inv(hessian_matrix)

            self._cov_matrix = covariance_matrix



        except:

            custom_warnings.warn("Cannot invert Hessian matrix, looks like the matrix is singluar")

            n_dim = len(best_fit_parameters)

            self._cov_matrix = np.zeros((n_dim, n_dim)) * np.nan
Ejemplo n.º 30
0
    def get_randomized_source_counts(self, source_model_counts):
        idx = (self._spectrum_plugin.observed_count_errors > 0)

        randomized_source_counts = np.zeros_like(source_model_counts)

        randomized_source_counts[idx] = np.random.normal(loc=source_model_counts[idx],
                                                         scale=self._spectrum_plugin.observed_count_errors[idx])

        # Issue a warning if the generated background is less than zero, and fix it by placing it at zero

        idx = (randomized_source_counts < 0)  # type: np.ndarray

        negative_source_n = np.sum(idx)

        if negative_source_n > 0:
            custom_warnings.warn("Generated source has negative counts "
                                 "in %i channels. Fixing them to zero" % (negative_source_n))

            randomized_source_counts[idx] = 0

        return randomized_source_counts
Ejemplo n.º 31
0
    def get_randomized_source_counts(self, source_model_counts):
        idx = (self._spectrum_plugin.observed_count_errors > 0)

        randomized_source_counts = np.zeros_like(source_model_counts)

        randomized_source_counts[idx] = np.random.normal(loc=source_model_counts[idx],
                                                         scale=self._spectrum_plugin.observed_count_errors[idx])

        # Issue a warning if the generated background is less than zero, and fix it by placing it at zero

        idx = (randomized_source_counts < 0)  # type: np.ndarray

        negative_source_n = np.sum(idx)

        if negative_source_n > 0:
            custom_warnings.warn("Generated source has negative counts "
                                 "in %i channels. Fixing them to zero" % (negative_source_n))

            randomized_source_counts[idx] = 0

        return randomized_source_counts
Ejemplo n.º 32
0
def temporary_directory(prefix='', within_directory=None):
    """
    This context manager creates a temporary directory in the most secure possible way (with no race condition), and
    removes it at the end.

    :param prefix: the directory name will start with this prefix, if specified
    :param within_directory: create within a specific directory (assumed to exist). Otherwise, it will be created in the
    default system temp directory (/tmp in unix)
    :return: the absolute pathname of the provided directory
    """

    directory = tempfile.mkdtemp(prefix=prefix, dir=within_directory)

    yield directory

    try:

        shutil.rmtree(directory)

    except:

        custom_warnings.warn("Couldn't remove temporary directory %s" % directory)
Ejemplo n.º 33
0
def temporary_directory(prefix='', within_directory=None):
    """
    This context manager creates a temporary directory in the most secure possible way (with no race condition), and
    removes it at the end.

    :param prefix: the directory name will start with this prefix, if specified
    :param within_directory: create within a specific directory (assumed to exist). Otherwise, it will be created in the
    default system temp directory (/tmp in unix)
    :return: the absolute pathname of the provided directory
    """

    directory = tempfile.mkdtemp(prefix=prefix, dir=within_directory)

    yield directory

    try:

        shutil.rmtree(directory)

    except:

        custom_warnings.warn("Couldn't remove temporary directory %s" % directory)
Ejemplo n.º 34
0
    def _source_is_valid(self, source):
        """
        checks if source name is valid for the 3FGL catalog

        :param source: source name
        :return: bool
        """

        warn_string = "The trigger %s is not valid. Must be in the form '3FGL J0000.0+0000'" % source

        match = _3FGL_name_match.match(source)

        if match is None:

            custom_warnings.warn(warn_string)

            answer = False

        else:

            answer = True

        return answer
Ejemplo n.º 35
0
def _gbm_and_lle_valid_source_check(source):
    """
    checks if source name is valid for both GBM and LLE data

    :param source: source name
    :return: bool
    """

    warn_string = "The trigger %s is not valid. Must be in the form GRB080916009" % source

    match = _trigger_name_match.match(source)

    if match is None:

        custom_warnings.warn(warn_string)

        answer = False

    else:

        answer = True

    return answer
Ejemplo n.º 36
0
    def __init__(self, joint_likelihood_instance0, joint_likelihood_instance1):

        self._joint_likelihood_instance0 = joint_likelihood_instance0  # type: JointLikelihood
        self._joint_likelihood_instance1 = joint_likelihood_instance1  # type: JointLikelihood

        # Restore best fit and store the reference value for the likelihood
        self._joint_likelihood_instance0.restore_best_fit()
        self._joint_likelihood_instance1.restore_best_fit()

        self._reference_TS = 2 * (self._joint_likelihood_instance0.current_minimum -
                                  self._joint_likelihood_instance1.current_minimum)

        # Safety check that the user has provided the models in the right order
        if self._reference_TS < 0:

            custom_warnings.warn("The reference TS is negative, either you specified the likelihood objects "
                                 "in the wrong order, or the fit for the alternative hyp. has failed. Since the "
                                 "two hyp. are nested, by definition the more complex hypothesis should give a "
                                 "better or equal fit with respect to the null hypothesis.")

        # Check that the dataset is the same

        if self._joint_likelihood_instance1.data_list != self._joint_likelihood_instance0.data_list:

            # Since this check might fail if the user loaded twice the same data, only issue a warning, instead of
            # an exception.

            custom_warnings.warn("The data lists for the null hyp. and for the alternative hyp. seems to be different."
                                 " If you loaded twice the same data and made the same data selections, disregard this "
                                 "message. Otherwise, consider the fact that the LRT is meaningless if the two data "
                                 "sets are not exactly the same. We will use the data loaded as part of the null "
                                 "hypothesis JointLikelihood object", RuntimeWarning)

        # For saving pha files
        self._save_pha = False
        self._data_container = []
Ejemplo n.º 37
0
    def _log_like(self, trial_values):
        """Compute the log-likelihood"""

        # Get the value of the log-likelihood for this parameters

        try:

            # Loop over each dataset and get the likelihood values for each set

            log_like_values = map(lambda dataset: dataset.get_log_like(), self._data_list.values())

        except ModelAssertionViolation:

            # Fit engine or sampler outside of allowed zone

            return -np.inf

        except:

            # We don't want to catch more serious issues

            raise

        # Sum the values of the log-like

        log_like = np.sum(log_like_values)

        if not np.isfinite(log_like):
            # Issue warning

            custom_warnings.warn("Likelihood value is infinite for parameters %s" % trial_values,
                                 LikelihoodIsInfinite)

            return -np.inf

        return log_like
Ejemplo n.º 38
0
    def _get_posterior(self, trial_values):
        """Compute the posterior for the normal sampler"""

        # Assign this trial values to the parameters and
        # store the corresponding values for the priors

        self._update_free_parameters()

        assert len(self._free_parameters) == len(trial_values), ("Something is wrong. Number of free parameters "
                                                                 "do not match the number of trial values.")

        log_prior = 0

        for i, (parameter_name, parameter) in enumerate(self._free_parameters.iteritems()):

            prior_value = parameter.prior(trial_values[i])

            if prior_value == 0:
                # Outside allowed region of parameter space

                return -numpy.inf

            else:

                parameter.value = trial_values[i]

                log_prior += math.log10(prior_value)

        # Get the value of the log-likelihood for this parameters

        try:

            # Loop over each dataset and get the likelihood values for each set

            log_like_values = map(lambda dataset: dataset.get_log_like(), self.data_list.values())

        except ModelAssertionViolation:

            # Fit engine or sampler outside of allowed zone

            return -numpy.inf

        except:

            # We don't want to catch more serious issues

            raise

        # Sum the values of the log-like

        log_like = numpy.sum(log_like_values)

        if not numpy.isfinite(log_like):
            # Issue warning

            custom_warnings.warn("Likelihood value is infinite for parameters %s" % trial_values, LikelihoodIsInfinite)

            return -numpy.inf

        #print("Log like is %s, log_prior is %s, for trial values %s" % (log_like, log_prior,trial_values))

        return log_like + log_prior
Ejemplo n.º 39
0
    def _get_one_error(self, parameter_name, target_delta_log_like, sign=-1):
        """
        A generic procedure to numerically compute the error for the parameters. You can override this if the
        minimizer provides its own method to compute the error of one parameter. If it provides a method to compute
        all errors are once, override the _get_errors method instead.

        :param parameter_name:
        :param target_delta_log_like:
        :param sign:
        :return:
        """

        # Since the procedure might find a better minimum, we can repeat it
        # up to a maximum of 10 times

        repeats = 0

        while repeats < 10:

            # Let's start optimistic...

            repeat = False

            repeats += 1

            # Restore best fit (which also updates the internal parameter dictionary)

            self.restore_best_fit()

            current_value, current_delta, current_min, current_max = self._internal_parameters[parameter_name]

            best_fit_value = current_value

            if sign == -1:

                extreme_allowed = current_min

            else:

                extreme_allowed = current_max

            # If the parameter has no boundary in the direction we are sampling, put a hard limit on
            # 10 times the current value (to avoid looping forever)

            if extreme_allowed is None:

                extreme_allowed = best_fit_value + sign * 10 * abs(best_fit_value)

            # We need to look for a value for the parameter where the difference between the minimum of the
            # log-likelihood and the likelihood for that value differs by more than target_delta_log_likelihood.
            # This is needed by the root-finding procedure, which needs to know an interval where the biased likelihood
            # function (see below) changes sign

            trials = best_fit_value + sign * np.linspace(0.1, 0.9, 9) * abs(best_fit_value)

            trials = np.append(trials, extreme_allowed)

            # Make sure we don't go below the allowed minimum or above the allowed maximum

            if sign == -1:

                np.clip(trials, extreme_allowed, np.inf, trials)

            else:

                np.clip(trials, -np.inf, extreme_allowed, trials)

            # There might be more than one value which was below the minimum (or above the maximum), so let's
            # take only unique elements

            trials = np.unique(trials)

            trials.sort()

            if sign == -1:

                trials = trials[::-1]

            # At this point we have a certain number of unique trials which always
            # contain the allowed minimum (or maximum)

            minimum_bound = None
            maximum_bound = None

            # Instance the profile likelihood function
            pl = ProfileLikelihood(self, [parameter_name])

            for i, trial in enumerate(trials):

                this_log_like = pl([trial])

                delta = this_log_like - self._m_log_like_minimum

                if delta < -0.1:

                    custom_warnings.warn("Found a better minimum (%.2f) for %s = %s during error "
                                         "computation." % (this_log_like, parameter_name, trial),
                                         BetterMinimumDuringProfiling)

                    xs = map(lambda x:x.value, self.parameters.values())

                    self._store_fit_results(xs, this_log_like, None)

                    repeat = True

                    break

                if delta > target_delta_log_like:

                    bound1 = trial

                    if i > 0:

                        bound2 = trials[i-1]

                    else:

                        bound2 = best_fit_value

                    minimum_bound = min(bound1, bound2)
                    maximum_bound = max(bound1, bound2)

                    repeat = False

                    break

            if repeat:

                # We found a better minimum, restart from scratch

                custom_warnings.warn("Restarting search...", RuntimeWarning)

                continue

            if minimum_bound is None:

                # Cannot find error in this direction (it's probably outside the allowed boundaries)
                custom_warnings.warn("Cannot find boundary for parameter %s" % parameter_name, CannotComputeErrors)

                error = np.nan
                break

            else:

                # Define the "biased likelihood", since brenq only finds zeros of function

                biased_likelihood = lambda x: pl(x) - self._m_log_like_minimum - target_delta_log_like

                try:

                    precise_bound = scipy.optimize.brentq(biased_likelihood,
                                                          minimum_bound,
                                                          maximum_bound,
                                                          xtol=1e-5, maxiter=1000)  #type: float
                except:

                    custom_warnings.warn("Cannot find boundary for parameter %s" % parameter_name, CannotComputeErrors)

                    error = np.nan
                    break

                error = precise_bound - best_fit_value

                break

        return error
Ejemplo n.º 40
0
    def get_contours(self, param_1, param_1_minimum, param_1_maximum, param_1_n_steps,
                     param_2=None, param_2_minimum=None, param_2_maximum=None, param_2_n_steps=None,
                     progress=True, **options):
        """
        Generate confidence contours for the given parameters by stepping for the given number of steps between
        the given boundaries. Call it specifying only source_1, param_1, param_1_minimum and param_1_maximum to
        generate the profile of the likelihood for parameter 1. Specify all parameters to obtain instead a 2d
        contour of param_1 vs param_2.

        NOTE: if using parallel computation, param_1_n_steps must be an integer multiple of the number of running
        engines. If that is not the case, the code will reduce the number of steps to match that requirement, and
        issue a warning

        :param param_1: fully qualified name of the first parameter or parameter instance
        :param param_1_minimum: lower bound for the range for the first parameter
        :param param_1_maximum: upper bound for the range for the first parameter
        :param param_1_n_steps: number of steps for the first parameter
        :param param_2: fully qualified name of the second parameter or parameter instance
        :param param_2_minimum: lower bound for the range for the second parameter
        :param param_2_maximum: upper bound for the range for the second parameter
        :param param_2_n_steps: number of steps for the second parameter
        :param progress: (True or False) whether to display progress or not
        :param log: by default the steps are taken linearly. With this optional parameter you can provide a tuple of
                    booleans which specify whether the steps are to be taken logarithmically. For example,
                    'log=(True,False)' specify that the steps for the first parameter are to be taken logarithmically,
                    while they are linear for the second parameter. If you are generating the profile for only one
                    parameter, you can specify 'log=(True,)' or 'log=(False,)' (optional)
        :return: a tuple containing an array corresponding to the steps for the first parameter, an array corresponding
                 to the steps for the second parameter (or None if stepping only in one direction), a matrix of size
                 param_1_steps x param_2_steps containing the value of the function at the corresponding points in the
                 grid. If param_2_steps is None (only one parameter), then this reduces to an array of
                 size param_1_steps.
        """

        if hasattr(param_1,"value"):

            # Substitute with the name
            param_1 = param_1.path

        if hasattr(param_2,'value'):

            param_2 = param_2.path

        # Check that the parameters exist
        assert param_1 in self._likelihood_model.free_parameters, "Parameter %s is not a free parameters of the " \
                                                                 "current model" % param_1

        if param_2 is not None:
            assert param_2 in self._likelihood_model.free_parameters, "Parameter %s is not a free parameters of the " \
                                                                      "current model" % param_2


        # Check that we have a valid fit

        assert self._current_minimum is not None, "You have to run the .fit method before calling get_contours."

        # Then restore the best fit

        self._minimizer.restore_best_fit()

        # Check minimal assumptions about the procedure

        assert not (param_1 == param_2), "You have to specify two different parameters"

        assert param_1_minimum < param_1_maximum, "Minimum larger than maximum for parameter 1"

        min1, max1 = self.likelihood_model[param_1].bounds

        if min1 is not None:

            assert param_1_minimum >= min1, "Requested low range for parameter %s (%s) " \
                                            "is below parameter minimum (%s)" % (param_1, param_1_minimum, min1)

        if max1 is not None:

            assert param_1_maximum <= max1, "Requested hi range for parameter %s (%s) " \
                                            "is above parameter maximum (%s)" % (param_1, param_1_maximum, max1)

        if param_2 is not None:

            min2, max2 = self.likelihood_model[param_2].bounds

            if min2 is not None:

                assert param_2_minimum >= min2, "Requested low range for parameter %s (%s) " \
                                                "is below parameter minimum (%s)" % (param_2, param_2_minimum, min2)

            if max2 is not None:

                assert param_2_maximum <= max2, "Requested hi range for parameter %s (%s) " \
                                                "is above parameter maximum (%s)" % (param_2, param_2_maximum, max2)

        # Check whether we are parallelizing or not

        if not threeML_config['parallel']['use-parallel']:

            a, b, cc = self.minimizer.contours(param_1, param_1_minimum, param_1_maximum, param_1_n_steps,
                                               param_2, param_2_minimum, param_2_maximum, param_2_n_steps,
                                               progress, **options)

            # Collapse the second dimension of the results if we are doing a 1d contour

            if param_2 is None:
                cc = cc[:, 0]

        else:

            # With parallel computation

            # In order to distribute fairly the computation, the strategy is to parallelize the computation
            # by assigning to the engines one "line" of the grid at the time

            # Connect to the engines

            client = ParallelClient(**options)

            # Get the number of engines

            n_engines = client.get_number_of_engines()

            # Check whether the number of threads is larger than the number of steps in the first direction

            if n_engines > param_1_n_steps:

                n_engines = int(param_1_n_steps)

                custom_warnings.warn("The number of engines is larger than the number of steps. Using only %s engines."
                                     % n_engines, ReducingNumberOfThreads)

            # Check if the number of steps is divisible by the number
            # of threads, otherwise issue a warning and make it so

            if float(param_1_n_steps) % n_engines != 0:
                # Set the number of steps to an integer multiple of the engines
                # (note that // is the floor division, also called integer division)

                param_1_n_steps = (param_1_n_steps // n_engines) * n_engines

                custom_warnings.warn("Number of steps is not a multiple of the number of threads. Reducing steps to %s"
                                     % param_1_n_steps, ReducingNumberOfSteps)

            # Compute the number of splits, i.e., how many lines in the grid for each engine.
            # (note that this is guaranteed to be an integer number after the previous checks)

            p1_split_steps = param_1_n_steps // n_engines

            # Prepare arrays for results

            if param_2 is None:

                # One array
                pcc = np.zeros(param_1_n_steps)

                pa = np.linspace(param_1_minimum, param_1_maximum, param_1_n_steps)
                pb = None

            else:

                pcc = np.zeros((param_1_n_steps, param_2_n_steps))

                # Prepare the two axes of the parameter space
                pa = np.linspace(param_1_minimum, param_1_maximum, param_1_n_steps)
                pb = np.linspace(param_2_minimum, param_2_maximum, param_2_n_steps)

            # Define the parallel worker which will go through the computation

            # NOTE: I only divide
            # on the first parameter axis so that the different
            # threads are more or less well mixed for points close and
            # far from the best fit

            def worker(start_index):

                # Re-create the minimizer

                backup_freeParameters = map(lambda x:x.value, self._likelihood_model.free_parameters.values())

                this_minimizer = self._get_minimizer(self.minus_log_like_profile,
                                                     self._free_parameters)

                this_p1min = pa[start_index * p1_split_steps]
                this_p1max = pa[(start_index + 1) * p1_split_steps - 1]

                # print("From %s to %s" % (this_p1min, this_p1max))

                aa, bb, ccc = this_minimizer.contours(param_1, this_p1min, this_p1max, p1_split_steps,
                                                      param_2, param_2_minimum, param_2_maximum,
                                                      param_2_n_steps,
                                                      progress=True, **options)

                # Restore best fit values

                for val, par in zip(backup_freeParameters, self._likelihood_model.free_parameters.values()):

                    par.value = val

                return ccc

            # Now re-assemble the vector of results taking the different parts from the engines

            all_results = client.execute_with_progress_bar(worker, range(n_engines), chunk_size=1)

            for i, these_results in enumerate(all_results):

                if param_2 is None:

                    pcc[i * p1_split_steps: (i + 1) * p1_split_steps] = these_results[:, 0]

                else:

                    pcc[i * p1_split_steps: (i + 1) * p1_split_steps, :] = these_results

            # Give the results the names that the following code expect. These are kept separate for debugging
            # purposes

            cc = pcc
            a = pa
            b = pb

        # Here we have done the computation, in parallel computation or not. Let's make the plot
        # with the contour

        if param_2 is not None:

            # 2d contour

            fig = self._plot_contours("%s" % (param_1), a, "%s" % (param_2,), b, cc)

        else:

            # 1d contour (i.e., a profile)

            fig = self._plot_profile("%s" % (param_1), a, cc)

        # Check if we found a better minimum. This shouldn't happen, but in case of very difficult fit
        # it might.

        if self._current_minimum - cc.min() > 0.1:

            if param_2 is not None:

                idx = cc.argmin()

                aidx, bidx = np.unravel_index(idx, cc.shape)

                print("\nFound a better minimum: %s with %s = %s and %s = %s. Run again your fit starting from here."
                      % (cc.min(), param_1, a[aidx], param_2, b[bidx]))

            else:

                idx = cc.argmin()

                print("Found a better minimum: %s with %s = %s. Run again your fit starting from here."
                      % (cc.min(), param_1, a[idx]))

        return a, b, cc, fig
Ejemplo n.º 41
0
    def __init__(self, name, fermipy_config):
        """
        :param name: a name for this instance
        :param fermipy_config: either a path to a YAML configuration file or a dictionary containing the configuration
        (see http://fermipy.readthedocs.io/)
        """

        # There are no nuisance parameters

        nuisance_parameters = {}

        super(FermipyLike,
              self).__init__(name, nuisance_parameters=nuisance_parameters)

        # Check whether the provided configuration is a file

        if not isinstance(fermipy_config, dict):

            # Assume this is a file name
            configuration_file = sanitize_filename(fermipy_config)

            if not os.path.exists(fermipy_config):
                log.critical("Configuration file %s does not exist" %
                             configuration_file)

            # Read the configuration
            with open(configuration_file) as f:

                self._configuration = yaml.load(f, Loader=yaml.SafeLoader)

        else:

            # Configuration is a dictionary. Nothing to do
            self._configuration = fermipy_config

        # If the user provided a 'model' key, issue a warning, as the model will be defined
        # later on and will overwrite the one contained in 'model'

        if "model" in self._configuration:

            custom_warnings.warn(
                "The provided configuration contains a 'model' section, which is useless as it "
                "will be overridden")

            self._configuration.pop("model")

        if "fileio" in self._configuration:

            custom_warnings.warn(
                "The provided configuration contains a 'fileio' section, which will be "
                "overwritten")

            self._configuration.pop("fileio")

        # Now check that the data exists

        # As minimum there must be a evfile and a scfile
        if not "evfile" in self._configuration["data"]:
            log.critical("You must provide a evfile in the data section")
        if not "scfile" in self._configuration["data"]:
            log.critical("You must provide a scfile in the data section")

        for datum in self._configuration["data"]:

            # Sanitize file name, as fermipy is not very good at handling relative paths or env. variables

            filename = str(
                sanitize_filename(self._configuration["data"][datum], True))

            self._configuration["data"][datum] = filename

            if not os.path.exists(self._configuration["data"][datum]):
                log.critical("File %s (%s) not found" % (filename, datum))

        # Prepare the 'fileio' part
        # Save all output in a directory with a unique name which depends on the configuration,
        # so that the same configuration will write in the same directory and fermipy will
        # know that it doesn't need to recompute things

        self._unique_id = "__%s" % _get_unique_tag_from_configuration(
            self._configuration)

        self._configuration["fileio"] = {"outdir": self._unique_id}

        # Ensure that there is a complete definition of a Region Of Interest (ROI)
        if not (("ra" in self._configuration["selection"]) and
                ("dec" in self._configuration["selection"])):
            log.critical(
                "You have to provide 'ra' and 'dec' in the 'selection' section of the configuration. Source name "
                "resolution, as well as Galactic coordinates, are not currently supported"
            )

        # This is empty at the beginning, will be instanced in the set_model method
        self._gta = None
Ejemplo n.º 42
0
def display_photometry_model_magnitudes(analysis, data=(), **kwargs):
    """

    Display the fitted model count spectrum of one or more Spectrum plugins

    NOTE: all parameters passed as keyword arguments that are not in the list below, will be passed as keyword arguments
    to the plt.subplots() constructor. So for example, you can specify the size of the figure using figsize = (20,10)

    :param args: one or more instances of Spectrum plugin
    :param min_rate: (optional) rebin to keep this minimum rate in each channel (if possible). If one number is
    provided, the same minimum rate is used for each dataset, otherwise a list can be provided with the minimum rate
    for each dataset
    :param data_cmap: (str) (optional) the color map used to extract automatically the colors for the data
    :param model_cmap: (str) (optional) the color map used to extract automatically the colors for the models
    :param data_colors: (optional) a tuple or list with the color for each dataset
    :param model_colors: (optional) a tuple or list with the color for each folded model
    :param show_legend: (optional) if True (default), shows a legend
    :param step: (optional) if True (default), show the folded model as steps, if False, the folded model is plotted
    with linear interpolation between each bin
    :return: figure instance


    """

    # If the user supplies a subset of the data, we will use that

    if not data:

        data_keys = analysis.data_list.keys()

    else:

        data_keys = data

    # Now we want to make sure that we only grab OGIP plugins

    new_data_keys = []

    for key in data_keys:

        # Make sure it is a valid key
        if key in analysis.data_list.keys():

            if isinstance(analysis.data_list[key], threeML.plugins.PhotometryLike.PhotometryLike):

                new_data_keys.append(key)

            else:

                custom_warnings.warn("Dataset %s is not of the Photometery kind. Cannot be plotted by "
                                     "display_photometry_model_magnitudes" % key)

    if not new_data_keys:
        RuntimeError(
            'There were no valid Photometry data requested for plotting. Please use the detector names in the data list')

    data_keys = new_data_keys

    # Default is to show the model with steps
    step = True

    data_cmap = threeML_config['photo']['data plot cmap']  # plt.cm.rainbow
    model_cmap = threeML_config['photo']['model plot cmap']  # plt.cm.nipy_spectral_r

    # Legend is on by default
    show_legend = True

    # Default colors

    data_colors = cmap_intervals(len(data_keys), data_cmap)
    model_colors = cmap_intervals(len(data_keys), model_cmap)

    # Now override defaults according to the optional keywords, if present

    if 'show_legend' in kwargs:
        show_legend = bool(kwargs.pop('show_legend'))

    if 'step' in kwargs:
        step = bool(kwargs.pop('step'))



    if 'data_cmap' in kwargs:
        data_cmap = plt.get_cmap(kwargs.pop('data_cmap'))
        data_colors = cmap_intervals(len(data_keys), data_cmap)

    if 'model_cmap' in kwargs:
        model_cmap = kwargs.pop('model_cmap')
        model_colors = cmap_intervals(len(data_keys), model_cmap)

    if 'data_colors' in kwargs:
        data_colors = kwargs.pop('data_colors')

        assert len(data_colors) >= len(data_keys), "You need to provide at least a number of data colors equal to the " \
                                                   "number of datasets"

    if 'model_colors' in kwargs:
        model_colors = kwargs.pop('model_colors')

        assert len(model_colors) >= len(
            data_keys), "You need to provide at least a number of model colors equal to the " \
                        "number of datasets"

    residual_plot = ResidualPlot(**kwargs)


    # go thru the detectors
    for key, data_color, model_color in zip(data_keys, data_colors, model_colors):


        data = analysis.data_list[key]  # type: threeML.plugins.PhotometryLike.PhotometryLike


        # get the expected counts


        avg_wave_length = data._filter_set.effective_wavelength.value #type: np.ndarray

        # need to sort because filters are not always in order

        sort_idx = avg_wave_length.argsort()

        expected_model_magnitudes = data._get_total_expectation()[sort_idx]
        magnitudes = data.magnitudes[sort_idx]
        mag_errors= data.magnitude_errors[sort_idx]
        avg_wave_length = avg_wave_length[sort_idx]

        residuals = (expected_model_magnitudes - magnitudes) / mag_errors

        widths = data._filter_set.wavelength_bounds.widths[sort_idx]


        residual_plot.add_data(x=avg_wave_length,
                               y=magnitudes,
                               xerr=widths,
                               yerr=mag_errors,
                               residuals=residuals,
                               label=data._name,
                               color=data_color)

        residual_plot.add_model(avg_wave_length,
                                expected_model_magnitudes,
                                label='%s Model' % data._name,
                                color=model_color)



        return residual_plot.finalize(xlabel="Wavelength\n(%s)"%data._filter_set.waveunits,
                                      ylabel='Magnitudes',
                                      xscale='linear',
                                      yscale='linear',
                                      invert_y=True)
Ejemplo n.º 43
0
    def from_gbm_cspec_or_ctime(cls, name, cspec_or_ctime_file, rsp_file, restore_background=None,
                                trigger_time=None,
                                poly_order=-1, verbose=True):
        """
               A plugin to natively bin, view, and handle Fermi GBM TTE data.
               A TTE event file are required as well as the associated response



               Background selections are specified as
               a comma separated string e.g. "-10-0,10-20"

               Initial source selection is input as a string e.g. "0-5"

               One can choose a background polynomial order by hand (up to 4th order)
               or leave it as the default polyorder=-1 to decide by LRT test

               :param name: name for your choosing
               :param tte_file: GBM tte event file
               :param rsp_file: Associated TTE CSPEC response file
               :param trigger_time: trigger time if needed
               :param poly_order: 0-4 or -1 for auto
               :param unbinned: unbinned likelihood fit (bool)
               :param verbose: verbose (bool)



               """

        # self._default_unbinned = unbinned

        # Load the relevant information from the TTE file

        cdata = GBMCdata(cspec_or_ctime_file, rsp_file)

        # Set a trigger time if one has not been set

        if trigger_time is not None:
            cdata.trigger_time = trigger_time

        # Create the the event list

        event_list = BinnedSpectrumSeries(cdata.spectrum_set,
                                          first_channel=0,
                                          mission='Fermi',
                                          instrument=cdata.det_name,
                                          verbose=verbose)

        # we need to see if this is an RSP2


        if isinstance(rsp_file,str) or isinstance(rsp_file,unicode):


            test = re.match('^.*\.rsp2$', rsp_file)

            # some GBM RSPs that are not marked RSP2 are in fact RSP2s
            # we need to check

            if test is None:

                with fits.open(rsp_file) as f:

                    # there should only be a header, ebounds and one spec rsp extension

                    if len(f) > 3:
                        # make test a dummy value to trigger the nest loop

                        test = -1

                        custom_warnings.warn(
                            'The RSP file is marked as a single response but in fact has multiple matrices. We will treat it as an RSP2')

            if test is not None:

                rsp = InstrumentResponseSet.from_rsp2_file(rsp2_file=rsp_file,
                                                           counts_getter=event_list.counts_over_interval,
                                                           exposure_getter=event_list.exposure_over_interval,
                                                           reference_time=cdata.trigger_time)





            else:

                rsp = OGIPResponse(rsp_file)

        else:

            assert isinstance(rsp_file, InstrumentResponse), 'The provided response is not a 3ML InstrumentResponse'
            rsp = rsp_file

        # pass to the super class

        return cls(name,
                   event_list,
                   response=rsp,
                   poly_order=poly_order,
                   unbinned=False,
                   verbose=verbose,
                   restore_poly_fit=restore_background,
                   container_type=BinnedSpectrumWithDispersion
                   )
Ejemplo n.º 44
0
    def set_polynomial_fit_interval(self, *time_intervals, **options):
        """Set the time interval to fit the background.
        Multiple intervals can be input as separate arguments
        Specified as 'tmin-tmax'. Intervals are in seconds. Example:

        set_polynomial_fit_interval("-10.0-0.0","10.-15.")

        :param time_intervals: intervals to fit on
        :param options:

        """

        # Find out if we want to binned or unbinned.
        # TODO: add the option to config file
        if 'unbinned' in options:
            unbinned = options.pop('unbinned')
            assert type(
                unbinned) == bool, 'unbinned option must be True or False'

        else:

            # assuming unbinned
            # could use config file here
            # unbinned = threeML_config['ogip']['use-unbinned-poly-fitting']

            unbinned = True

        # we create some time intervals

        poly_intervals = TimeIntervalSet.from_strings(*time_intervals)

        # adjust the selections to the data

        new_intervals = []

        self._poly_selected_counts = []

        self._poly_exposure = 0.

        for i, time_interval in enumerate(poly_intervals):

            t1 = time_interval.start_time
            t2 = time_interval.stop_time

            if (self._stop_time <= t1) or (t2 <= self._start_time):
                custom_warnings.warn(
                    "The time interval %f-%f is out side of the arrival times and will be dropped"
                    % (t1, t2))

            else:

                if t1 < self._start_time:
                    custom_warnings.warn(
                        "The time interval %f-%f started before the first arrival time (%f), so we are changing the intervals to %f-%f"
                        % (t1, t2, self._start_time, self._start_time, t2))

                    t1 = self._start_time  # + 1

                if t2 > self._stop_time:
                    custom_warnings.warn(
                        "The time interval %f-%f ended after the last arrival time (%f), so we are changing the intervals to %f-%f"
                        % (t1, t2, self._stop_time, t1, self._stop_time))

                    t2 = self._stop_time  # - 1.

                new_intervals.append('%f-%f' % (t1, t2))

                self._poly_selected_counts.append(
                    self.count_per_channel_over_interval(t1, t2))
                self._poly_exposure += self.exposure_over_interval(t1, t2)

        # make new intervals after checks

        poly_intervals = TimeIntervalSet.from_strings(*new_intervals)

        self._poly_selected_counts = np.sum(self._poly_selected_counts, axis=0)

        # set the poly intervals as an attribute

        self._poly_intervals = poly_intervals

        # Fit the events with the given intervals
        if unbinned:

            self._unbinned = True  # keep track!

            self._unbinned_fit_polynomials()

        else:

            self._unbinned = False

            self._fit_polynomials()

        # we have a fit now

        self._poly_fit_exists = True

        if self._verbose:
            print("%s %d-order polynomial fit with the %s method" %
                  (self._fit_method_info['bin type'],
                   self._optimal_polynomial_grade,
                   self._fit_method_info['fit method']))
            print('\n')

        # recalculate the selected counts

        if self._time_selection_exists:
            self.set_active_time_intervals(
                *self._time_intervals.to_string().split(','))
Ejemplo n.º 45
0
    def fit(self, quiet=False, compute_covariance=True, n_samples=5000):
        """
        Perform a fit of the current likelihood model on the datasets

        :param quiet: If True, print the results (default), otherwise do not print anything
        :param compute_covariance:If True (default), compute and display the errors and the correlation matrix.
        :return: a dictionary with the results on the parameters, and the values of the likelihood at the minimum
                 for each dataset and the total one.
        """

        # Update the list of free parameters, to be safe against changes the user might do between
        # the creation of this class and the calling of this method

        self._update_free_parameters()

        # Empty the call recorder
        self._record_calls = {}
        self._ncalls = 0

        # Check if we have free parameters, otherwise simply return the value of the log like
        if len(self._free_parameters) == 0:

            custom_warnings.warn(
                "There is no free parameter in the current model",
                RuntimeWarning)

            # Create the minimizer anyway because it will be needed by the following code

            self._minimizer = self._get_minimizer(self.minus_log_like_profile,
                                                  self._free_parameters)

            # Store the "minimum", which is just the current value
            self._current_minimum = float(self.minus_log_like_profile())

        else:

            # Instance the minimizer

            # If we have a global minimizer, use that first (with no covariance)
            if isinstance(self._minimizer_type,
                          minimization.GlobalMinimization):

                # Do global minimization first

                global_minimizer = self._get_minimizer(
                    self.minus_log_like_profile, self._free_parameters)

                xs, global_log_likelihood_minimum = global_minimizer.minimize(
                    compute_covar=False)

                # Gather global results
                paths = []
                values = []
                errors = []
                units = []

                for par in list(self._free_parameters.values()):

                    paths.append(par.path)
                    values.append(par.value)
                    errors.append(0)
                    units.append(par.unit)

                global_results = ResultsTable(paths, values, errors, errors,
                                              units)

                if not quiet:

                    print(
                        "\n\nResults after global minimizer (before secondary optimization):"
                    )

                    global_results.display()

                    print("\nTotal log-likelihood minimum: %.3f\n" %
                          global_log_likelihood_minimum)

                # Now set up secondary minimizer
                self._minimizer = self._minimizer_type.get_second_minimization_instance(
                    self.minus_log_like_profile, self._free_parameters)

            else:

                # Only local minimization to be performed

                self._minimizer = self._get_minimizer(
                    self.minus_log_like_profile, self._free_parameters)

            # Perform the fit, but first flush stdout (so if we have verbose=True the messages there will follow
            # what is already in the buffer)
            sys.stdout.flush()

            xs, log_likelihood_minimum = self._minimizer.minimize(
                compute_covar=compute_covariance)

            if log_likelihood_minimum == minimization.FIT_FAILED:

                raise FitFailed("The fit failed to converge.")

            # Store the current minimum for the -log likelihood

            self._current_minimum = float(log_likelihood_minimum)

            # First restore best fit (to make sure we compute the likelihood at the right point in the following)
            self._minimizer.restore_best_fit()

        # Now collect the values for the likelihood for the various datasets

        # Fill the dictionary with the values of the -log likelihood (dataset by dataset)

        minus_log_likelihood_values = collections.OrderedDict()

        # Keep track of the total for a double check

        total = 0

        # sum up the total number of data points

        total_number_of_data_points = 0

        for dataset in list(self._data_list.values()):

            ml = dataset.inner_fit() * (-1)

            minus_log_likelihood_values[dataset.name] = ml

            total += ml

            total_number_of_data_points += dataset.get_number_of_data_points()

        assert (
            total == self._current_minimum
        ), "Current minimum stored after fit and current do not correspond!"

        # compute additional statistics measures

        statistical_measures = collections.OrderedDict()

        # for MLE we can only compute the AIC and BIC as they
        # are point estimates

        statistical_measures["AIC"] = aic(-total, len(self._free_parameters),
                                          total_number_of_data_points)
        statistical_measures["BIC"] = bic(-total, len(self._free_parameters),
                                          total_number_of_data_points)

        # Now instance an analysis results class
        self._analysis_results = MLEResults(
            self.likelihood_model,
            self._minimizer.covariance_matrix,
            minus_log_likelihood_values,
            statistical_measures=statistical_measures,
            n_samples=n_samples,
        )

        # Show the results

        if not quiet:

            self._analysis_results.display()

        return (
            self._analysis_results.get_data_frame(),
            self._analysis_results.get_statistic_frame(),
        )
Ejemplo n.º 46
0
    def minus_log_like_profile(self, *trial_values):
        """
        Return the minus log likelihood for a given set of trial values

        :param trial_values: the trial values. Must be in the same number as the free parameters in the model
        :return: minus log likelihood
        """

        # Keep track of the number of calls
        self._ncalls += 1

        # Transform the trial values in a numpy array

        trial_values = np.array(trial_values)

        # Check that there are no nans within the trial values

        # This is the fastest way to check for any nan
        # (try other methods if you don't believe me)

        if not np.isfinite(np.dot(trial_values, trial_values.T)):
            # There are nans, something weird is going on. Return FIT_FAILED so the engine
            # stays away from this (or fail)

            return minimization.FIT_FAILED

        # Assign the new values to the parameters

        for i, parameter in enumerate(self._free_parameters.values()):

            # Use the internal representation (see the Parameter class)

            parameter._set_internal_value(trial_values[i])

        # Now profile out nuisance parameters and compute the new value
        # for the likelihood

        summed_log_likelihood = 0

        for dataset in self._data_list.values():

            try:

                this_log_like = dataset.inner_fit()

            except ModelAssertionViolation:

                # This is a zone of the parameter space which is not allowed. Return
                # a big number for the likelihood so that the fit engine will avoid it

                custom_warnings.warn("Fitting engine in forbidden space: %s" % (trial_values,),
                                     custom_exceptions.ForbiddenRegionOfParameterSpace)

                return minimization.FIT_FAILED

            except:

                # Do not intercept other errors

                raise

            summed_log_likelihood += this_log_like

        # Check that the global like is not NaN
        # I use this weird check because it is not guaranteed that the plugins return np.nan,
        # especially if they are written in something other than python

        if "%s" % summed_log_likelihood == 'nan':
            custom_warnings.warn("These parameters returned a logLike = Nan: %s" % (trial_values,),
                                 NotANumberInLikelihood)

            return minimization.FIT_FAILED

        if self.verbose:
            sys.stderr.write("trial values: %s -> logL = %.3f\n" % (",".join(map(lambda x:"%.5g" % x, trial_values)),
                                                                    summed_log_likelihood))

        # Record this call
        if self._record:

            self._record_calls[tuple(trial_values)] = summed_log_likelihood

        # Return the minus log likelihood

        return summed_log_likelihood * (-1)
Ejemplo n.º 47
0
    def minus_log_like_profile(self, *trial_values):
        """
        Return the minus log likelihood for a given set of trial values

        :param trial_values: the trial values. Must be in the same number as the free parameters in the model
        :return: minus log likelihood
        """

        # Keep track of the number of calls
        self._ncalls += 1

        # Transform the trial values in a numpy array

        trial_values = np.array(trial_values)

        # Check that there are no nans within the trial values

        # This is the fastest way to check for any nan
        # (try other methods if you don't believe me)

        if not np.isfinite(np.dot(trial_values, trial_values.T)):
            # There are nans, something weird is going on. Return FIT_FAILED so the engine
            # stays away from this (or fail)

            return minimization.FIT_FAILED

        # Assign the new values to the parameters

        for i, parameter in enumerate(self._free_parameters.values()):

            # Use the internal representation (see the Parameter class)

            parameter._set_internal_value(trial_values[i])

        # Now profile out nuisance parameters and compute the new value
        # for the likelihood

        summed_log_likelihood = 0

        for dataset in list(self._data_list.values()):

            try:

                this_log_like = dataset.inner_fit()

            except ModelAssertionViolation:

                # This is a zone of the parameter space which is not allowed. Return
                # a big number for the likelihood so that the fit engine will avoid it

                custom_warnings.warn(
                    "Fitting engine in forbidden space: %s" % (trial_values, ),
                    custom_exceptions.ForbiddenRegionOfParameterSpace,
                )

                return minimization.FIT_FAILED

            except:

                # Do not intercept other errors

                raise

            summed_log_likelihood += this_log_like

        # Check that the global like is not NaN
        # I use this weird check because it is not guaranteed that the plugins return np.nan,
        # especially if they are written in something other than python

        if "%s" % summed_log_likelihood == "nan":
            custom_warnings.warn(
                "These parameters returned a logLike = Nan: %s" %
                (trial_values, ),
                NotANumberInLikelihood,
            )

            return minimization.FIT_FAILED

        if self.verbose:
            sys.stderr.write(
                "trial values: %s -> logL = %.3f\n" %
                (",".join(["%.5g" % x
                           for x in trial_values]), summed_log_likelihood))

        # Record this call
        if self._record:

            self._record_calls[tuple(trial_values)] = summed_log_likelihood

        # Return the minus log likelihood

        return summed_log_likelihood * (-1)
Ejemplo n.º 48
0
def display_spectrum_model_counts(analysis, data=(), **kwargs):
    """

    Display the fitted model count spectrum of one or more Spectrum plugins

    NOTE: all parameters passed as keyword arguments that are not in the list below, will be passed as keyword arguments
    to the plt.subplots() constructor. So for example, you can specify the size of the figure using figsize = (20,10)

    :param args: one or more instances of Spectrum plugin
    :param min_rate: (optional) rebin to keep this minimum rate in each channel (if possible). If one number is
    provided, the same minimum rate is used for each dataset, otherwise a list can be provided with the minimum rate
    for each dataset
    :param data_cmap: (str) (optional) the color map used to extract automatically the colors for the data
    :param model_cmap: (str) (optional) the color map used to extract automatically the colors for the models
    :param data_colors: (optional) a tuple or list with the color for each dataset
    :param model_colors: (optional) a tuple or list with the color for each folded model
    :param data_color: (optional) color for all datasets
    :param model_color: (optional) color for all folded models
    :param show_legend: (optional) if True (default), shows a legend
    :param step: (optional) if True (default), show the folded model as steps, if False, the folded model is plotted
    :param model_subplot: (optional) axe(s) to plot to for overplotting
    with linear interpolation between each bin
    :return: figure instance


    """

    # If the user supplies a subset of the data, we will use that

    if not data:

        data_keys = analysis.data_list.keys()

    else:

        data_keys = data

    # Now we want to make sure that we only grab OGIP plugins

    new_data_keys = []

    for key in data_keys:

        # Make sure it is a valid key
        if key in analysis.data_list.keys():

            if isinstance(analysis.data_list[key], threeML.plugins.SpectrumLike.SpectrumLike):

                new_data_keys.append(key)

            else:

                custom_warnings.warn("Dataset %s is not of the SpectrumLike kind. Cannot be plotted by "
                                     "display_spectrum_model_counts" % key)

    if not new_data_keys:
        RuntimeError(
            'There were no valid SpectrumLike data requested for plotting. Please use the detector names in the data list')

    data_keys = new_data_keys

    # default settings

    # Default is to show the model with steps
    step = True

    data_cmap = threeML_config['ogip']['data plot cmap']  # plt.cm.rainbow
    model_cmap = threeML_config['ogip']['model plot cmap']  # plt.cm.nipy_spectral_r

    # Legend is on by default
    show_legend = True

    show_residuals = True

    # Default colors

    data_colors = cmap_intervals(len(data_keys), data_cmap)
    model_colors = cmap_intervals(len(data_keys), model_cmap)




    # Now override defaults according to the optional keywords, if present



    if 'show_data' in kwargs:

        show_data = bool(kwargs.pop('show_data'))

    else:

        show_data = True


    if 'show_legend' in kwargs:
        show_legend = bool(kwargs.pop('show_legend'))

    if 'show_residuals' in kwargs:
        show_residuals= bool(kwargs.pop('show_residuals'))

    if 'step' in kwargs:
        step = bool(kwargs.pop('step'))

    if 'min_rate' in kwargs:

        min_rate = kwargs.pop('min_rate')

        # If min_rate is a floating point, use the same for all datasets, otherwise use the provided ones

        try:

            min_rate = float(min_rate)

            min_rates = [min_rate] * len(data_keys)

        except TypeError:

            min_rates = list(min_rate)

            assert len(min_rates) >= len(
                data_keys), "If you provide different minimum rates for each data set, you need" \
                            "to provide an iterable of the same length of the number of datasets"

    else:

        # This is the default (no rebinning)

        min_rates = [NO_REBIN] * len(data_keys)

    if 'data_cmap' in kwargs:
        data_cmap = plt.get_cmap(kwargs.pop('data_cmap'))
        data_colors = cmap_intervals(len(data_keys), data_cmap)

    if 'model_cmap' in kwargs:
        model_cmap = kwargs.pop('model_cmap')
        model_colors = cmap_intervals(len(data_keys), model_cmap)

    if 'data_colors' in kwargs:
        data_colors = kwargs.pop('data_colors')

        assert len(data_colors) >= len(data_keys), "You need to provide at least a number of data colors equal to the " \
                                                   "number of datasets"

    elif 'data_color' in kwargs:

        data_colors = [kwargs.pop('data_color')] * len(data_keys)



    if 'model_colors' in kwargs:
        model_colors = kwargs.pop('model_colors')

        assert len(model_colors) >= len(
            data_keys), "You need to provide at least a number of model colors equal to the " \
                        "number of datasets"
    
    ratio_residuals=False
    if 'ratio_residuals' in kwargs:
        ratio_residuals = bool(kwargs['ratio_residuals'])

    elif 'model_color' in kwargs:

        model_colors = [kwargs.pop('model_color')] * len(data_keys)


    if 'model_labels' in kwargs:


        model_labels = kwargs.pop('model_labels')

        assert len(model_labels) == len(data_keys), 'you must have the same number of model labels as data sets'

    else:

        model_labels = ['%s Model' % analysis.data_list[key]._name for key in data_keys]

    #fig, (ax, ax1) = plt.subplots(2, 1, sharex=True, gridspec_kw={'height_ratios': [2, 1]}, **kwargs)

    residual_plot = ResidualPlot(show_residuals=show_residuals, **kwargs)

    if show_residuals:

        axes = [residual_plot.data_axis,residual_plot.residual_axis]

    else:

        axes = residual_plot.data_axis


    # go thru the detectors
    for key, data_color, model_color, min_rate, model_label in zip(data_keys, data_colors, model_colors, min_rates, model_labels):



        # NOTE: we use the original (unmasked) vectors because we need to rebin ourselves the data later on

        data = analysis.data_list[key] # type: threeML.plugins.SpectrumLike.SpectrumLike

        data.display_model(data_color=data_color,
                           model_color=model_color,
                           min_rate=min_rate,
                           step=step,
                           show_residuals=show_residuals,
                           show_data=show_data,
                           show_legend=show_legend,
                           ratio_residuals=ratio_residuals,
                           model_label=model_label,
                           model_subplot=axes

                           )


    return residual_plot.figure
Ejemplo n.º 49
0
def display_photometry_model_magnitudes(analysis, data=(), **kwargs):
    """

    Display the fitted model count spectrum of one or more Spectrum plugins

    NOTE: all parameters passed as keyword arguments that are not in the list below, will be passed as keyword arguments
    to the plt.subplots() constructor. So for example, you can specify the size of the figure using figsize = (20,10)

    :param args: one or more instances of Spectrum plugin
    :param min_rate: (optional) rebin to keep this minimum rate in each channel (if possible). If one number is
    provided, the same minimum rate is used for each dataset, otherwise a list can be provided with the minimum rate
    for each dataset
    :param data_cmap: (str) (optional) the color map used to extract automatically the colors for the data
    :param model_cmap: (str) (optional) the color map used to extract automatically the colors for the models
    :param data_colors: (optional) a tuple or list with the color for each dataset
    :param model_colors: (optional) a tuple or list with the color for each folded model
    :param show_legend: (optional) if True (default), shows a legend
    :param step: (optional) if True (default), show the folded model as steps, if False, the folded model is plotted
    with linear interpolation between each bin
    :return: figure instance


    """

    # If the user supplies a subset of the data, we will use that

    if not data:

        data_keys = list(analysis.data_list.keys())

    else:

        data_keys = data

    # Now we want to make sure that we only grab OGIP plugins

    new_data_keys = []

    for key in data_keys:

        # Make sure it is a valid key
        if key in list(analysis.data_list.keys()):

            if isinstance(analysis.data_list[key], photolike.PhotometryLike):

                new_data_keys.append(key)

            else:

                custom_warnings.warn(
                    "Dataset %s is not of the Photometery kind. Cannot be plotted by "
                    "display_photometry_model_magnitudes" % key)

    if not new_data_keys:
        RuntimeError(
            "There were no valid Photometry data requested for plotting. Please use the detector names in the data list"
        )

    data_keys = new_data_keys

    # Default is to show the model with steps
    step = threeML_config.plugins.photo.fit_plot.step

    data_cmap = threeML_config.plugins.photo.fit_plot.data_cmap.value  # plt.cm.rainbow

    model_cmap = threeML_config.plugins.photo.fit_plot.model_cmap.value

    # Legend is on by default
    show_legend = True

    # Default colors

    data_colors = cmap_intervals(len(data_keys), data_cmap)
    model_colors = cmap_intervals(len(data_keys), model_cmap)

    # Now override defaults according to the optional keywords, if present

    if "show_legend" in kwargs:

        show_legend = bool(kwargs.pop("show_legend"))

    if "step" in kwargs:
        step = bool(kwargs.pop("step"))

    if "data_cmap" in kwargs:
        data_cmap = plt.get_cmap(kwargs.pop("data_cmap"))
        data_colors = cmap_intervals(len(data_keys), data_cmap)

    if "model_cmap" in kwargs:
        model_cmap = kwargs.pop("model_cmap")
        model_colors = cmap_intervals(len(data_keys), model_cmap)

    if "data_colors" in kwargs:
        data_colors = kwargs.pop("data_colors")

        if len(data_colors) < len(data_keys):
            log.error(
                "You need to provide at least a number of data colors equal to the "
                "number of datasets")
            raise ValueError()

    if "model_colors" in kwargs:
        model_colors = kwargs.pop("model_colors")

        if len(model_colors) < len(data_keys):
            log.error(
                "You need to provide at least a number of model colors equal to the "
                "number of datasets")
            raise ValueError()

    residual_plot = ResidualPlot(**kwargs)

    # go thru the detectors
    for key, data_color, model_color in zip(data_keys, data_colors,
                                            model_colors):

        data = analysis.data_list[key]  # type: photolike

        # get the expected counts

        avg_wave_length = (data._filter_set.effective_wavelength.value
                           )  # type: np.ndarray

        # need to sort because filters are not always in order

        sort_idx = avg_wave_length.argsort()

        expected_model_magnitudes = data._get_total_expectation()[sort_idx]
        magnitudes = data.magnitudes[sort_idx]
        mag_errors = data.magnitude_errors[sort_idx]
        avg_wave_length = avg_wave_length[sort_idx]

        residuals = old_div((expected_model_magnitudes - magnitudes),
                            mag_errors)

        widths = data._filter_set.wavelength_bounds.widths[sort_idx]

        residual_plot.add_data(
            x=avg_wave_length,
            y=magnitudes,
            xerr=widths,
            yerr=mag_errors,
            residuals=residuals,
            label=data._name,
            color=data_color,
        )

        residual_plot.add_model(
            avg_wave_length,
            expected_model_magnitudes,
            label="%s Model" % data._name,
            color=model_color,
        )

        return residual_plot.finalize(
            xlabel="Wavelength\n(%s)" % data._filter_set.waveunits,
            ylabel="Magnitudes",
            xscale="linear",
            yscale="linear",
            invert_y=True,
        )
Ejemplo n.º 50
0
    def __init__(self):

        # Read first the default configuration file
        default_configuration_path = get_path_of_data_file(_config_file_name)

        assert os.path.exists(default_configuration_path), \
            "Default configuration %s does not exist. Re-install 3ML" % default_configuration_path

        with open(default_configuration_path) as f:

            try:

                configuration = yaml.load(f, Loader=yaml.SafeLoader)

            except:

                raise ConfigurationFileCorrupt("Default configuration file %s cannot be parsed!" %
                                               (default_configuration_path))

            # This needs to be here for the _check_configuration to work

            self._default_configuration_raw = configuration

            # Test the default configuration

            try:

                self._check_configuration(configuration, default_configuration_path)

            except:

                raise

            else:

                self._default_path = default_configuration_path

        # Check if the user has a user-supplied config file under .threeML

        user_config_path = os.path.join(get_path_of_user_dir(), _config_file_name)

        if os.path.exists(user_config_path):

            with open(user_config_path) as f:

                configuration = yaml.load(f, Loader=yaml.SafeLoader)

                # Test if the local/configuration is ok

                try:

                    self._configuration = self._check_configuration(configuration, user_config_path)

                except ConfigurationFileCorrupt:

                    # Probably an old configuration file
                    custom_warnings.warn("The user configuration file at %s does not appear to be valid. We will "
                                         "substitute it with the default configuration. You will find a copy of the "
                                         "old configuration at %s so you can transfer any customization you might "
                                         "have from there to the new configuration file. We will use the default "
                                         "configuration for this session."
                                         %(user_config_path, "%s.bak" % user_config_path))

                    # Move the config file to a backup file
                    shutil.copy2(user_config_path, "%s.bak" % user_config_path)

                    # Remove old file
                    os.remove(user_config_path)

                    # Copy the default configuration
                    shutil.copy2(self._default_path, user_config_path)

                    self._configuration = self._check_configuration(self._default_configuration_raw, self._default_path)
                    self._filename = self._default_path

                else:

                    self._filename = user_config_path

                    print("Configuration read from %s" % (user_config_path))

        else:

            custom_warnings.warn("Using default configuration from %s. "
                                 "You might want to copy it to %s to customize it and avoid this warning."
                                 % (self._default_path, user_config_path))

            self._configuration = self._check_configuration(self._default_configuration_raw, self._default_path)
            self._filename = self._default_path
Ejemplo n.º 51
0
    def fit(self, quiet=False, compute_covariance=True, n_samples=5000):
        """
        Perform a fit of the current likelihood model on the datasets

        :param quiet: If True, print the results (default), otherwise do not print anything
        :param compute_covariance:If True (default), compute and display the errors and the correlation matrix.
        :return: a dictionary with the results on the parameters, and the values of the likelihood at the minimum
                 for each dataset and the total one.
        """

        # Update the list of free parameters, to be safe against changes the user might do between
        # the creation of this class and the calling of this method

        self._update_free_parameters()

        # Empty the call recorder
        self._record_calls = {}
        self._ncalls = 0

        # Check if we have free parameters, otherwise simply return the value of the log like
        if len(self._free_parameters) == 0:

            custom_warnings.warn("There is no free parameter in the current model", RuntimeWarning)

            # Create the minimizer anyway because it will be needed by the following code

            self._minimizer = self._get_minimizer(self.minus_log_like_profile,
                                                  self._free_parameters)

            # Store the "minimum", which is just the current value
            self._current_minimum = float(self.minus_log_like_profile())

        else:

            # Instance the minimizer

            # If we have a global minimizer, use that first (with no covariance)
            if isinstance(self._minimizer_type, minimization.GlobalMinimization):

                # Do global minimization first

                global_minimizer = self._get_minimizer(self.minus_log_like_profile, self._free_parameters)

                xs, global_log_likelihood_minimum = global_minimizer.minimize(compute_covar=False)

                # Gather global results
                paths = []
                values = []
                errors = []
                units = []

                for par in self._free_parameters.values():

                    paths.append(par.path)
                    values.append(par.value)
                    errors.append(0)
                    units.append(par.unit)

                global_results = ResultsTable(paths, values, errors, errors, units)

                if not quiet:

                    print("\n\nResults after global minimizer (before secondary optimization):")

                    global_results.display()

                    print("\nTotal log-likelihood minimum: %.3f\n" % global_log_likelihood_minimum)

                # Now set up secondary minimizer
                self._minimizer = self._minimizer_type.get_second_minimization_instance(self.minus_log_like_profile,
                                                                                        self._free_parameters)

            else:

                # Only local minimization to be performed

                self._minimizer = self._get_minimizer(self.minus_log_like_profile,
                                                      self._free_parameters)

            # Perform the fit, but first flush stdout (so if we have verbose=True the messages there will follow
            # what is already in the buffer)
            sys.stdout.flush()

            xs, log_likelihood_minimum = self._minimizer.minimize(compute_covar=compute_covariance)

            if log_likelihood_minimum == minimization.FIT_FAILED:

                raise FitFailed("The fit failed to converge.")

            # Store the current minimum for the -log likelihood

            self._current_minimum = float(log_likelihood_minimum)

            # First restore best fit (to make sure we compute the likelihood at the right point in the following)
            self._minimizer.restore_best_fit()

        # Now collect the values for the likelihood for the various datasets

        # Fill the dictionary with the values of the -log likelihood (dataset by dataset)

        minus_log_likelihood_values = collections.OrderedDict()

        # Keep track of the total for a double check

        total = 0

        # sum up the total number of data points

        total_number_of_data_points = 0

        for dataset in self._data_list.values():

            ml = dataset.inner_fit() * (-1)

            minus_log_likelihood_values[dataset.name] = ml

            total += ml

            total_number_of_data_points += dataset.get_number_of_data_points()

        assert total == self._current_minimum, "Current minimum stored after fit and current do not correspond!"

        # compute additional statistics measures

        statistical_measures = collections.OrderedDict()

        # for MLE we can only compute the AIC and BIC as they
        # are point estimates

        statistical_measures['AIC'] = aic(-total,len(self._free_parameters),total_number_of_data_points)
        statistical_measures['BIC'] = bic(-total,len(self._free_parameters),total_number_of_data_points)


        # Now instance an analysis results class
        self._analysis_results = MLEResults(self.likelihood_model, self._minimizer.covariance_matrix,
                                            minus_log_likelihood_values,statistical_measures=statistical_measures, n_samples=n_samples)

        # Show the results

        if not quiet:

            self._analysis_results.display()

        return self._analysis_results.get_data_frame(), self._analysis_results.get_statistic_frame()
Ejemplo n.º 52
0
    def __init__(self, matrix_list, exposure_getter, counts_getter, reference_time=0.0):
        """

        :param matrix_list:
        :type matrix_list : list[InstrumentResponse]
        :param exposure_getter : a function returning the exposure between t1 and t2
        :param counts_getter : a function returning the number of counts between t1 and t2
        :param reference_time : a reference time to be added to the specifications of the intervals used in the
        weight_by_* methods. Use this if you want to express the time intervals in time units from the reference_time,
        instead of "absolute" time. For GRBs, this is the trigger time. NOTE: if you use a reference time, the
        counts_getter and the exposure_getter must accept times relative to the reference time.
        """

        # Store list of matrices

        self._matrix_list = list(matrix_list)  # type: list[InstrumentResponse]

        # Create the corresponding list of coverage intervals

        self._coverage_intervals = TimeIntervalSet(map(lambda x: x.coverage_interval,
                                                       self._matrix_list))

        # Make sure that all matrices have coverage interval set

        if None in self._coverage_intervals:

            raise NoCoverageIntervals("You need to specify the coverage interval for all matrices in the matrix_list")

        # Remove from the list matrices that cover intervals of zero duration (yes, the GBM publishes those too,
        # one example is in data/ogip_test_gbm_b0.rsp2)
        to_be_removed = []
        for i, interval in enumerate(self._coverage_intervals):

            if interval.duration == 0:

                # Remove it
                with custom_warnings.catch_warnings():

                    custom_warnings.simplefilter("always", RuntimeWarning)

                    custom_warnings.warn("Removing matrix %s (numbering starts at zero) because it has a coverage of "
                                         "zero seconds" % i, RuntimeWarning)

                to_be_removed.append(i)

        # Actually remove them
        if len(to_be_removed) > 0:

            [self._matrix_list.pop(index) for index in to_be_removed]
            [self._coverage_intervals.pop(index) for index in to_be_removed]

        # Order the matrices by time

        idx = self._coverage_intervals.argsort()

        # It is possible that there is only one coverage interval (these are published by GBM e.g. GRB090819607)
        # so we need to be sure that the array is a least 1D

        self._coverage_intervals = TimeIntervalSet(np.atleast_1d(itemgetter(*idx)(self._coverage_intervals)))
        self._matrix_list = np.atleast_1d(itemgetter(*idx)(self._matrix_list))
        # Now make sure that the coverage intervals are contiguous (i.e., there are no gaps)
        if not self._coverage_intervals.is_contiguous():

            raise NonContiguousCoverageIntervals("The provided responses have coverage intervals which are not contiguous!")

        # Apply the reference time shift, if any
        self._coverage_intervals -= reference_time

        # Store callable

        self._exposure_getter = exposure_getter  # type: callable

        self._counts_getter = counts_getter  # type: callable

        # Store reference time

        self._reference_time = float(reference_time)
Ejemplo n.º 53
0
    def from_root_file(cls, response_file_name):
        """
        Build response from a ROOT file. Do not use directly, use the hawc_response_factory function instead.

        :param response_file_name:
        :return: a HAWCResponse instance
        """

        from ..root_handler import open_ROOT_file, get_list_of_keys, tree_to_ndarray

        # Make sure file is readable

        response_file_name = sanitize_filename(response_file_name)

        # Check that they exists and can be read

        if not file_existing_and_readable(
                response_file_name):  # pragma: no cover

            raise IOError("Response %s does not exist or is not readable" %
                          response_file_name)

        # Read response

        with open_ROOT_file(response_file_name) as root_file:

            # Get the name of the trees
            object_names = get_list_of_keys(root_file)

            # Make sure we have all the things we need

            assert 'LogLogSpectrum' in object_names
            assert 'DecBins' in object_names
            assert 'AnalysisBins' in object_names

            # Read spectrum used during the simulation
            log_log_spectrum = root_file.Get("LogLogSpectrum")

            # Get the analysis bins definition
            dec_bins_ = tree_to_ndarray(root_file.Get("DecBins"))

            dec_bins_lower_edge = dec_bins_['lowerEdge']  # type: np.ndarray
            dec_bins_upper_edge = dec_bins_['upperEdge']  # type: np.ndarray
            dec_bins_center = dec_bins_['simdec']  # type: np.ndarray

            dec_bins = zip(dec_bins_lower_edge, dec_bins_center,
                           dec_bins_upper_edge)

            # Read in the ids of the response bins ("analysis bins" in LiFF jargon)
            try:

                response_bins_ids = tree_to_ndarray(
                    root_file.Get("AnalysisBins"), "name")  # type: np.ndarray

            except ValueError:

                try:

                    response_bins_ids = tree_to_ndarray(
                        root_file.Get("AnalysisBins"),
                        "id")  # type: np.ndarray

                except ValueError:

                    # Some old response files (or energy responses) have no "name" branch
                    custom_warnings.warn(
                        "Response %s has no AnalysisBins 'id' or 'name' branch. "
                        "Will try with default names" % response_file_name)

                    response_bins_ids = None

            response_bins_ids = response_bins_ids.astype(str)

            # Now we create a dictionary of ResponseBin instances for each dec bin_name
            response_bins = collections.OrderedDict()

            for dec_id in range(len(dec_bins)):

                this_response_bins = collections.OrderedDict()

                min_dec, dec_center, max_dec = dec_bins[dec_id]

                # If we couldn't get the reponse_bins_ids above, let's use the default names
                if response_bins_ids is None:

                    # Default are just integers. let's read how many nHit bins are from the first dec bin
                    dec_id_label = "dec_%02i" % dec_id

                    n_energy_bins = root_file.Get(dec_id_label).GetNkeys()

                    response_bins_ids = range(n_energy_bins)

                for response_bin_id in response_bins_ids:
                    this_response_bin = ResponseBin.from_ttree(
                        root_file, dec_id, response_bin_id, log_log_spectrum,
                        min_dec, dec_center, max_dec)

                    this_response_bins[response_bin_id] = this_response_bin

                response_bins[dec_bins[dec_id][1]] = this_response_bins

        # Now the file is closed. Let's explicitly remove f so we are sure it is freed
        del root_file

        # Instance the class and return it
        instance = cls(response_file_name, dec_bins, response_bins)

        return instance
Ejemplo n.º 54
0
    def to_spectrumlike(self, from_bins=False, start=None, stop=None, interval_name='_interval', extract_measured_background=False):
        """
        Create plugin(s) from either the current active selection or the time bins.
        If creating from an event list, the
        bins are from create_time_bins. If using a pre-time binned time series, the bins are those
        native to the data. Start and stop times can be used to  control which bins are used.

        :param from_bins: choose to create plugins from the time bins
        :param start: optional start time of the bins
        :param stop: optional stop time of the bins
        :param extract_measured_background: Use the selected background rather than a polynomial fit to the background
        :param interval_name: the name of the interval
        :return: SpectrumLike plugin(s)
        """


        # we can use either the modeled or the measured background. In theory, all the information
        # in the background spectrum should propagate to the likelihood



        if extract_measured_background:

            this_background_spectrum = self._measured_background_spectrum

        else:

            this_background_spectrum = self._background_spectrum

        # this is for a single interval


        if not from_bins:

            assert self._observed_spectrum is not None, 'Must have selected an active time interval'

            assert isinstance(self._observed_spectrum, BinnedSpectrum), 'You are attempting to create a SpectrumLike plugin from the wrong data type'

            if this_background_spectrum is None:

                custom_warnings.warn('No background selection has been made. This plugin will contain no background!')


            if self._response is None:

                return SpectrumLike(name=self._name,
                                    observation=self._observed_spectrum,
                                    background=this_background_spectrum,
                                    verbose=self._verbose,
                                    tstart=self._tstart,
                                    tstop=self._tstop)

            else:

                return DispersionSpectrumLike(name=self._name,
                                              observation=self._observed_spectrum,
                                              background=this_background_spectrum,
                                              verbose=self._verbose,
                                              tstart = self._tstart,
                                              tstop = self._tstop
                                              )


        else:

            # this is for a set of intervals.

            assert self._time_series.bins is not None, 'This time series does not have any bins!'

            # save the original interval if there is one
            old_interval = copy.copy(self._active_interval)
            old_verbose = copy.copy(self._verbose)

            # we will keep it quiet to keep from being annoying

            self._verbose = False

            list_of_speclikes = []

            # get the bins from the time series
            # for event lists, these are from created bins
            # for binned spectra sets, these are the native bines


            these_bins = self._time_series.bins  # type: TimeIntervalSet

            if start is not None:
                assert stop is not None, 'must specify a start AND a stop time'

            if stop is not None:
                assert stop is not None, 'must specify a start AND a stop time'

                these_bins = these_bins.containing_interval(start, stop, inner=False)



           # loop through the intervals and create spec likes

            with progress_bar(len(these_bins), title='Creating plugins') as p:

                for i, interval in enumerate(these_bins):


                    self.set_active_time_interval(interval.to_string())

                    assert isinstance(self._observed_spectrum, BinnedSpectrum), 'You are attempting to create a SpectrumLike plugin from the wrong data type'

                    if extract_measured_background:

                        this_background_spectrum = self._measured_background_spectrum

                    else:

                        this_background_spectrum = self._background_spectrum


                    if this_background_spectrum is None:
                        custom_warnings.warn(
                            'No bakckground selection has been made. This plugin will contain no background!')

                    try:

                        if self._response is None:

                            sl = SpectrumLike(name="%s%s%d" % (self._name, interval_name, i),
                                              observation=self._observed_spectrum,
                                              background=this_background_spectrum,
                                              verbose=self._verbose,
                                              tstart=self._tstart,
                                              tstop=self._tstop)

                        else:

                            sl = DispersionSpectrumLike(name="%s%s%d" % (self._name, interval_name, i),
                                                        observation=self._observed_spectrum,
                                                        background=this_background_spectrum,
                                                        verbose=self._verbose,
                                                        tstart=self._tstart,
                                                        tstop=self._tstop)

                        list_of_speclikes.append(sl)

                    except(NegativeBackground):

                        custom_warnings.warn('Something is wrong with interval %s. skipping.' % interval)

                    p.increase()

            # restore the old interval

            if old_interval is not None:

               self.set_active_time_interval(*old_interval)

            else:

               self._active_interval = None

            self._verbose = old_verbose

            return list_of_speclikes
Ejemplo n.º 55
0
    def get_contours(self,
                     param_1,
                     param_1_minimum,
                     param_1_maximum,
                     param_1_n_steps,
                     param_2=None,
                     param_2_minimum=None,
                     param_2_maximum=None,
                     param_2_n_steps=None,
                     progress=True,
                     **options):
        """
        Generate confidence contours for the given parameters by stepping for the given number of steps between
        the given boundaries. Call it specifying only source_1, param_1, param_1_minimum and param_1_maximum to
        generate the profile of the likelihood for parameter 1. Specify all parameters to obtain instead a 2d
        contour of param_1 vs param_2.

        NOTE: if using parallel computation, param_1_n_steps must be an integer multiple of the number of running
        engines. If that is not the case, the code will reduce the number of steps to match that requirement, and
        issue a warning

        :param param_1: fully qualified name of the first parameter or parameter instance
        :param param_1_minimum: lower bound for the range for the first parameter
        :param param_1_maximum: upper bound for the range for the first parameter
        :param param_1_n_steps: number of steps for the first parameter
        :param param_2: fully qualified name of the second parameter or parameter instance
        :param param_2_minimum: lower bound for the range for the second parameter
        :param param_2_maximum: upper bound for the range for the second parameter
        :param param_2_n_steps: number of steps for the second parameter
        :param progress: (True or False) whether to display progress or not
        :param log: by default the steps are taken linearly. With this optional parameter you can provide a tuple of
                    booleans which specify whether the steps are to be taken logarithmically. For example,
                    'log=(True,False)' specify that the steps for the first parameter are to be taken logarithmically,
                    while they are linear for the second parameter. If you are generating the profile for only one
                    parameter, you can specify 'log=(True,)' or 'log=(False,)' (optional)
        :return: a tuple containing an array corresponding to the steps for the first parameter, an array corresponding
                 to the steps for the second parameter (or None if stepping only in one direction), a matrix of size
                 param_1_steps x param_2_steps containing the value of the function at the corresponding points in the
                 grid. If param_2_steps is None (only one parameter), then this reduces to an array of
                 size param_1_steps.
        """

        if hasattr(param_1, "value"):

            # Substitute with the name
            param_1 = param_1.path

        if hasattr(param_2, "value"):

            param_2 = param_2.path

        # Check that the parameters exist
        assert param_1 in self._likelihood_model.free_parameters, (
            "Parameter %s is not a free parameters of the "
            "current model" % param_1)

        if param_2 is not None:
            assert param_2 in self._likelihood_model.free_parameters, (
                "Parameter %s is not a free parameters of the "
                "current model" % param_2)

        # Check that we have a valid fit

        assert (
            self._current_minimum is not None
        ), "You have to run the .fit method before calling get_contours."

        # Then restore the best fit

        self._minimizer.restore_best_fit()

        # Check minimal assumptions about the procedure

        assert not (param_1
                    == param_2), "You have to specify two different parameters"

        assert (param_1_minimum <
                param_1_maximum), "Minimum larger than maximum for parameter 1"

        min1, max1 = self.likelihood_model[param_1].bounds

        if min1 is not None:

            assert param_1_minimum >= min1, (
                "Requested low range for parameter %s (%s) "
                "is below parameter minimum (%s)" %
                (param_1, param_1_minimum, min1))

        if max1 is not None:

            assert param_1_maximum <= max1, (
                "Requested hi range for parameter %s (%s) "
                "is above parameter maximum (%s)" %
                (param_1, param_1_maximum, max1))

        if param_2 is not None:

            min2, max2 = self.likelihood_model[param_2].bounds

            if min2 is not None:

                assert param_2_minimum >= min2, (
                    "Requested low range for parameter %s (%s) "
                    "is below parameter minimum (%s)" %
                    (param_2, param_2_minimum, min2))

            if max2 is not None:

                assert param_2_maximum <= max2, (
                    "Requested hi range for parameter %s (%s) "
                    "is above parameter maximum (%s)" %
                    (param_2, param_2_maximum, max2))

        # Check whether we are parallelizing or not

        if not threeML_config["parallel"]["use-parallel"]:

            a, b, cc = self.minimizer.contours(
                param_1, param_1_minimum, param_1_maximum, param_1_n_steps,
                param_2, param_2_minimum, param_2_maximum, param_2_n_steps,
                progress, **options)

            # Collapse the second dimension of the results if we are doing a 1d contour

            if param_2 is None:
                cc = cc[:, 0]

        else:

            # With parallel computation

            # In order to distribute fairly the computation, the strategy is to parallelize the computation
            # by assigning to the engines one "line" of the grid at the time

            # Connect to the engines

            client = ParallelClient(**options)

            # Get the number of engines

            n_engines = client.get_number_of_engines()

            # Check whether the number of threads is larger than the number of steps in the first direction

            if n_engines > param_1_n_steps:

                n_engines = int(param_1_n_steps)

                custom_warnings.warn(
                    "The number of engines is larger than the number of steps. Using only %s engines."
                    % n_engines,
                    ReducingNumberOfThreads,
                )

            # Check if the number of steps is divisible by the number
            # of threads, otherwise issue a warning and make it so

            if float(param_1_n_steps) % n_engines != 0:
                # Set the number of steps to an integer multiple of the engines
                # (note that // is the floor division, also called integer division)

                param_1_n_steps = (param_1_n_steps // n_engines) * n_engines

                custom_warnings.warn(
                    "Number of steps is not a multiple of the number of threads. Reducing steps to %s"
                    % param_1_n_steps,
                    ReducingNumberOfSteps,
                )

            # Compute the number of splits, i.e., how many lines in the grid for each engine.
            # (note that this is guaranteed to be an integer number after the previous checks)

            p1_split_steps = param_1_n_steps // n_engines

            # Prepare arrays for results

            if param_2 is None:

                # One array
                pcc = np.zeros(param_1_n_steps)

                pa = np.linspace(param_1_minimum, param_1_maximum,
                                 param_1_n_steps)
                pb = None

            else:

                pcc = np.zeros((param_1_n_steps, param_2_n_steps))

                # Prepare the two axes of the parameter space
                pa = np.linspace(param_1_minimum, param_1_maximum,
                                 param_1_n_steps)
                pb = np.linspace(param_2_minimum, param_2_maximum,
                                 param_2_n_steps)

            # Define the parallel worker which will go through the computation

            # NOTE: I only divide
            # on the first parameter axis so that the different
            # threads are more or less well mixed for points close and
            # far from the best fit

            def worker(start_index):

                # Re-create the minimizer

                backup_freeParameters = [
                    x.value for x in list(
                        self._likelihood_model.free_parameters.values())
                ]

                this_minimizer = self._get_minimizer(
                    self.minus_log_like_profile, self._free_parameters)

                this_p1min = pa[start_index * p1_split_steps]
                this_p1max = pa[(start_index + 1) * p1_split_steps - 1]

                # print("From %s to %s" % (this_p1min, this_p1max))

                aa, bb, ccc = this_minimizer.contours(param_1,
                                                      this_p1min,
                                                      this_p1max,
                                                      p1_split_steps,
                                                      param_2,
                                                      param_2_minimum,
                                                      param_2_maximum,
                                                      param_2_n_steps,
                                                      progress=True,
                                                      **options)

                # Restore best fit values

                for val, par in zip(
                        backup_freeParameters,
                        list(self._likelihood_model.free_parameters.values()),
                ):

                    par.value = val

                return ccc

            # Now re-assemble the vector of results taking the different parts from the engines

            all_results = client.execute_with_progress_bar(
                worker, list(range(n_engines)), chunk_size=1)

            for i, these_results in enumerate(all_results):

                if param_2 is None:

                    pcc[i * p1_split_steps:(i + 1) *
                        p1_split_steps] = these_results[:, 0]

                else:

                    pcc[i * p1_split_steps:(i + 1) *
                        p1_split_steps, :] = these_results

            # Give the results the names that the following code expect. These are kept separate for debugging
            # purposes

            cc = pcc
            a = pa
            b = pb

        # Here we have done the computation, in parallel computation or not. Let's make the plot
        # with the contour

        if param_2 is not None:

            # 2d contour

            fig = self._plot_contours("%s" % (param_1), a, "%s" % (param_2, ),
                                      b, cc)

        else:

            # 1d contour (i.e., a profile)

            fig = self._plot_profile("%s" % (param_1), a, cc)

        # Check if we found a better minimum. This shouldn't happen, but in case of very difficult fit
        # it might.

        if self._current_minimum - cc.min() > 0.1:

            if param_2 is not None:

                idx = cc.argmin()

                aidx, bidx = np.unravel_index(idx, cc.shape)

                print(
                    "\nFound a better minimum: %s with %s = %s and %s = %s. Run again your fit starting from here."
                    % (cc.min(), param_1, a[aidx], param_2, b[bidx]))

            else:

                idx = cc.argmin()

                print(
                    "Found a better minimum: %s with %s = %s. Run again your fit starting from here."
                    % (cc.min(), param_1, a[idx]))

        return a, b, cc, fig
Ejemplo n.º 56
0
    def __init__(self, matrix, ebounds, monte_carlo_energies, coverage_interval=None):
        """

        Generic response class that accepts a full matrix, detector energy boundaries (ebounds) and monte carlo energies,
        and an optional coverage interval which indicates which time interval the matrix applies to.

        If there are n_channels in the detector, and the monte carlo energies are n_mc_energies, then the matrix must
        be n_channels x n_mc_energies.

        Therefore, an OGIP style RSP from a file is not required if the matrix,
        ebounds, and mc channels exist.


        :param matrix: an n_channels x n_mc_energies response matrix representing both effective area and
        energy dispersion effects
        :param ebounds: the energy boundaries of the detector channels (size n_channels + 1)
        :param monte_carlo_energies: the energy boundaries of the monte carlo channels (size n_mc_energies + 1)
        :param coverage_interval: the time interval to which the matrix refers to (if available, None by default)
        :type coverage_interval: TimeInterval
        """

        # we simply store all the variables to the class

        self._matrix = np.array(matrix, float)

        # Make sure there are no nans or inf
        assert np.all(np.isfinite(self._matrix)), "Infinity or nan in matrix"

        self._ebounds = np.array(ebounds, float)

        self._mc_energies = np.array(monte_carlo_energies)

        self._integral_function = None

        # Store the time interval
        if coverage_interval is not None:

            assert isinstance(coverage_interval, TimeInterval), "The coverage interval must be a TimeInterval instance"

            self._coverage_interval = coverage_interval

        else:

            self._coverage_interval = None

        # Safety checks
        assert self._matrix.shape == (self._ebounds.shape[0]-1, self._mc_energies.shape[0]-1), \
            "Matrix has the wrong shape. Got %s, expecting %s" % (self._matrix.shape,
                                                                 [self._ebounds.shape[0]-1,
                                                                  self._mc_energies.shape[0]-1])

        if self._mc_energies.max() < self._ebounds.max():

            custom_warnings.warn("Maximum MC energy (%s) is smaller "
                                 "than maximum EBOUNDS energy (%s)" % (self._mc_energies.max(), self.ebounds.max()),
                                 RuntimeWarning)

        if self._mc_energies.min() > self._ebounds.min():

            custom_warnings.warn("Minimum MC energy (%s) is larger than "
                                 "minimum EBOUNDS energy (%s)" % (self._mc_energies.min(), self._ebounds.min()),
                                 RuntimeWarning)
Ejemplo n.º 57
0
    def _compute_covariance_matrix(self, best_fit_values):
        """
        This function compute the approximate covariance matrix as the inverse of the Hessian matrix,
        which is the matrix of second derivatives of the likelihood function with respect to
        the parameters.

        The sqrt of the diagonal of the result is an accurate estimate of the errors only if the
        log.likelihood is parabolic in the neighborhood of the minimum.

        Derivatives are computed numerically.

        :return: the covariance matrix
        """

        minima = map(lambda parameter:parameter._get_internal_min_value(), self.parameters.values())
        maxima = map(lambda parameter: parameter._get_internal_max_value(), self.parameters.values())

        # Check whether some of the minima or of the maxima are None. If they are, set them
        # to a value 1000 times smaller or larger respectively than the best fit.
        # An error of 3 orders of magnitude is not interesting in general, and this is the only
        # way to be able to compute a derivative numerically

        for i in range(len(minima)):

            if minima[i] is None:

                minima[i] = best_fit_values[i] / 1000.0

            if maxima[i] is None:

                maxima[i] = best_fit_values[i] * 1000.0

        # Transform them in np.array

        minima = np.array(minima)
        maxima = np.array(maxima)

        try:

            hessian_matrix = get_hessian(self.function, best_fit_values, minima, maxima)

        except ParameterOnBoundary:

            custom_warnings.warn("One or more of the parameters are at their boundaries. Cannot compute covariance and"
                                 " errors", CannotComputeCovariance)

            n_dim = len(best_fit_values)

            return np.zeros((n_dim,n_dim)) * np.nan

        # Invert it to get the covariance matrix

        try:

            covariance_matrix = np.linalg.inv(hessian_matrix)

        except:

            custom_warnings.warn("Cannot invert Hessian matrix, looks like the matrix is singular")

            n_dim = len(best_fit_values)

            return np.zeros((n_dim, n_dim)) * np.nan

        # Now check that the covariance matrix is semi-positive definite (it must be unless
        # there have been numerical problems, which can happen when some parameter is unconstrained)

        # The fastest way is to try and compute the Cholesky decomposition, which
        # works only if the matrix is positive definite

        try:

            _ = np.linalg.cholesky(covariance_matrix)

        except:

            custom_warnings.warn("Covariance matrix is NOT semi-positive definite. Cannot estimate errors. This can "
                                 "happen for many reasons, the most common being one or more unconstrained parameters",
                                 CannotComputeCovariance)

        return covariance_matrix
Ejemplo n.º 58
0
    def to_polarlike(self, from_bins=False, start=None, stop=None, interval_name='_interval', extract_measured_background=False):

        assert has_polarpy, 'you must have the polarpy module installed'

        assert issubclass(self._container_type, BinnedModulationCurve), 'You are attempting to create a POLARLike plugin from the wrong data type'



        if extract_measured_background:

            this_background_spectrum = self._measured_background_spectrum

        else:

            this_background_spectrum = self._background_spectrum


        if isinstance(self._response,str):
            self._response = PolarResponse(self._response)

        if not from_bins:

            assert self._observed_spectrum is not None, 'Must have selected an active time interval'

            if this_background_spectrum is None:

                custom_warnings.warn('No background selection has been made. This plugin will contain no background!')


            return PolarLike(name=self._name,
                             observation=self._observed_spectrum,
                             background=this_background_spectrum,
                             response=self._response,
                             verbose=self._verbose,
            #                 tstart=self._tstart,
            #                 tstop=self._tstop
            )



        else:

            # this is for a set of intervals.

            assert self._time_series.bins is not None, 'This time series does not have any bins!'


            # save the original interval if there is one
            old_interval = copy.copy(self._active_interval)
            old_verbose = copy.copy(self._verbose)

            # we will keep it quiet to keep from being annoying

            self._verbose = False

            list_of_polarlikes = []


            # now we make one response to save time




            # get the bins from the time series
            # for event lists, these are from created bins
            # for binned spectra sets, these are the native bines


            these_bins = self._time_series.bins  # type: TimeIntervalSet

            if start is not None:
                assert stop is not None, 'must specify a start AND a stop time'

            if stop is not None:
                assert stop is not None, 'must specify a start AND a stop time'

                these_bins = these_bins.containing_interval(start, stop, inner=False)



           # loop through the intervals and create spec likes

            with progress_bar(len(these_bins), title='Creating plugins') as p:

                for i, interval in enumerate(these_bins):


                    self.set_active_time_interval(interval.to_string())


                    if extract_measured_background:

                        this_background_spectrum = self._measured_background_spectrum

                    else:

                        this_background_spectrum = self._background_spectrum



                    if this_background_spectrum is None:
                        custom_warnings.warn(
                            'No bakckground selection has been made. This plugin will contain no background!')

                    try:



                        pl = PolarLike(name="%s%s%d" % (self._name, interval_name, i),
                                       observation=self._observed_spectrum,
                                       background=this_background_spectrum,
                                       response=self._response,
                                       verbose=self._verbose,
                        #               tstart=self._tstart,
                        #               tstop=self._tstop
                        )



                        list_of_polarlikes.append(pl)

                    except(NegativeBackground):

                        custom_warnings.warn('Something is wrong with interval %s. skipping.' % interval)

                    p.increase()

            # restore the old interval

            if old_interval is not None:

               self.set_active_time_interval(*old_interval)

            else:

               self._active_interval = None

            self._verbose = old_verbose

            return list_of_polarlikes
Ejemplo n.º 59
0
    def __init__(self,
                 start_time,
                 stop_time,
                 n_channels,
                 native_quality=None,
                 first_channel=1,
                 ra=None,
                 dec=None,
                 mission=None,
                 instrument=None,
                 verbose=True,
                 edges=None):
        """
        The EventList is a container for event data that is tagged in time and in PHA/energy. It handles event selection,
        temporal polynomial fitting, temporal binning, and exposure calculations (in subclasses). Once events are selected
        and/or polynomials are fit, the selections can be extracted via a PHAContainer which is can be read by an OGIPLike
        instance and translated into a PHA instance.


        :param  n_channels: Number of detector channels
        :param  start_time: start time of the event list
        :param  stop_time: stop time of the event list
        :param  first_channel: where detchans begin indexing
        :param  rsp_file: the response file corresponding to these events
        :param  arrival_times: list of event arrival times
        :param  energies: list of event energies or pha channels
        :param native_quality: native pha quality flags
        :param edges: The histogram boundaries if not specified by a response
        :param mission:
        :param instrument:
        :param verbose:
        :param  ra:
        :param  dec:
        """

        self._verbose = verbose
        self._n_channels = n_channels
        self._first_channel = first_channel
        self._native_quality = native_quality

        # we haven't made selections yet

        self._time_intervals = None
        self._poly_intervals = None
        self._counts = None
        self._exposure = None
        self._poly_counts = None
        self._poly_count_err = None
        self._poly_selected_counts = None
        self._poly_exposure = None

        # ebounds for objects w/o a response
        self._edges = edges

        if native_quality is not None:
            assert len(
                native_quality
            ) == n_channels, "the native quality has length %d but you specified there were %d channels" % (
                len(native_quality), n_channels)

        self._start_time = start_time

        self._stop_time = stop_time

        # name the instrument if there is not one

        if instrument is None:

            custom_warnings.warn(
                'No instrument name is given. Setting to UNKNOWN')

            self._instrument = "UNKNOWN"

        else:

            self._instrument = instrument

        if mission is None:

            custom_warnings.warn(
                'No mission name is given. Setting to UNKNOWN')

            self._mission = "UNKNOWN"

        else:

            self._mission = mission

        self._user_poly_order = -1
        self._time_selection_exists = False
        self._poly_fit_exists = False

        self._fit_method_info = {"bin type": None, 'fit method': None}
Ejemplo n.º 60
0
    def __init__(self, name, time_series, response=None,
                 poly_order=-1, unbinned=True, verbose=True, restore_poly_fit=None, container_type=BinnedSpectrumWithDispersion):
        """
        Class for handling generic time series data including binned and event list
        series. Depending on the data, this class builds either a  SpectrumLike or
        DisperisonSpectrumLike plugin

        For specific instruments, use the TimeSeries.from() classmethods


        :param name: name for the plugin
        :param time_series: a TimeSeries instance
        :param response: options InstrumentResponse instance
        :param poly_order: the polynomial order to use for background fitting
        :param unbinned: if the background should be fit unbinned
        :param verbose: the verbosity switch
        :param restore_poly_fit: file from which to read a prefitted background
        """

        assert isinstance(time_series, TimeSeries), "must be a TimeSeries instance"

        assert issubclass(container_type,Histogram), 'must be a subclass of Histogram'


        self._name = name

        self._container_type = container_type

        self._time_series = time_series  # type: TimeSeries

        # make sure we have a proper response

        if response is not None:
            assert isinstance(response, InstrumentResponse) or isinstance(response,
                                                                          InstrumentResponseSet) or isinstance(response, str), 'Response must be an instance of InstrumentResponse'

        # deal with RSP weighting if need be

        if isinstance(response, InstrumentResponseSet):

            # we have a weighted response
            self._rsp_is_weighted = True
            self._weighted_rsp = response

            # just get a dummy response for the moment
            # it will be corrected when we set the interval

            self._response = InstrumentResponse.create_dummy_response(response.ebounds,
                                                                      response.monte_carlo_energies)

        else:

            self._rsp_is_weighted = False
            self._weighted_rsp = None

            self._response = response

        self._verbose = verbose
        self._active_interval = None
        self._observed_spectrum = None
        self._background_spectrum = None
        self._measured_background_spectrum = None

        self._time_series.poly_order = poly_order

        self._default_unbinned = unbinned

        # try and restore the poly fit if requested

        if restore_poly_fit is not None:

            if file_existing_and_readable(restore_poly_fit):
                self._time_series.restore_fit(restore_poly_fit)

                if verbose:
                    print('Successfully restored fit from %s'%restore_poly_fit)


            else:

                custom_warnings.warn(
                    "Could not find saved background %s." % restore_poly_fit)