def calculate_point_source_flux(*args, **kwargs):
custom_warnings.warn("The use of calculate_point_source_flux is deprecated. Please use the .get_point_source_flux()"
" method of the JointLikelihood.results or the BayesianAnalysis.results member. For example:"
" jl.results.get_point_source_flux().")
return _calculate_point_source_flux(*args, **kwargs)
def get_randomized_background_counts(self):
# Now randomize the expectations.
_, background_model_counts = self.get_current_value()
# We cannot generate variates with zero sigma. They variates from those channel will always be zero
# This is a limitation of this whole idea. However, remember that by construction an error of zero
# it is only allowed when the background counts are zero as well.
idx = (self._spectrum_plugin.background_count_errors > 0)
randomized_background_counts = np.zeros_like(background_model_counts)
randomized_background_counts[idx] = np.random.normal(
loc=background_model_counts[idx],
scale=self._spectrum_plugin.background_count_errors[idx])
# Issue a warning if the generated background is less than zero, and fix it by placing it at zero
idx = (randomized_background_counts < 0) # type: np.ndarray
negative_background_n = np.sum(idx)
if negative_background_n > 0:
custom_warnings.warn("Generated background has negative counts "
"in %i channels. Fixing them to zero" %
(negative_background_n))
randomized_background_counts[idx] = 0
return randomized_background_counts
def bin_by_bayesian_blocks(cls,
arrival_times,
p0,
bkg_integral_distribution=None):
"""Divide a series of events characterized by their arrival time in blocks
of perceptibly constant count rate. If the background integral distribution
is given, divide the series in blocks where the difference with respect to
the background is perceptibly constant.
:param arrival_times: An iterable (list, numpy.array...) containing the arrival
time of the events.
NOTE: the input array MUST be time-ordered, and without
duplicated entries. To ensure this, you may execute the
following code:
tt_array = numpy.asarray(self._arrival_times)
tt_array = numpy.unique(tt_array)
tt_array.sort()
before running the algorithm.
:param p0: The probability of finding a variations (i.e., creating a new
block) when there is none. In other words, the probability of
a Type I error, i.e., rejecting the null-hypothesis when is
true. All found variations will have a post-trial significance
larger than p0.
:param bkg_integral_distribution : the integral distribution for the
background counts. It must be a function of the form f(x),
which must return the integral number of counts expected from
the background component between time 0 and x.
"""
try:
final_edges = bayesian_blocks(
arrival_times,
arrival_times[0],
arrival_times[-1],
p0,
bkg_integral_distribution,
)
except Exception as e:
if "duplicate" in str(e):
custom_warnings.warn(
"There where possible duplicate time tags in the data. We will try to run a different algorithm"
)
final_edges = bayesian_blocks_not_unique(
arrival_times, arrival_times[0], arrival_times[-1], p0)
else:
print(e)
raise RuntimeError()
starts = np.asarray(final_edges)[:-1]
stops = np.asarray(final_edges)[1:]
return cls.from_starts_and_stops(starts, stops)
def _source_is_valid(self, source):
"""
checks if source name is valid for the 3FGL catalog
:param source: source name
:return: bool
"""
warn_string = (
"The trigger %s is not valid. Must be in the form '3FGL J0000.0+0000'"
% source
)
match = _3FGL_name_match.match(source)
if match is None:
custom_warnings.warn(warn_string)
answer = False
else:
answer = True
return answer
def _gbm_and_lle_valid_source_check(source):
"""
checks if source name is valid for both GBM and LLE data
:param source: source name
:return: bool
"""
warn_string = (
"The trigger %s is not valid. Must be in the form GRB080916009" % source
)
match = _trigger_name_match.match(source)
if match is None:
custom_warnings.warn(warn_string)
answer = False
else:
answer = True
return answer
def set_uniform_priors(self):
"""
Automatically set all parameters to uniform or log-uniform priors. The latter is used when the range spanned
by the parameter is larger than 2 orders of magnitude.
:return: (none)
"""
for parameter_name, parameter in self._free_parameters.iteritems():
if parameter.min_value is None or parameter.max_value is None:
custom_warnings.warn("Cannot decide the prior for parameter %s, since it has no "
"minimum or no maximum value (or both)")
continue
n_orders_of_magnitude = numpy.log10(parameter.max_value - parameter.min_value)
if n_orders_of_magnitude > 2:
print("Using log-uniform prior for %s" % parameter_name)
parameter.prior = log_uniform_prior(lower_bound=parameter.min_value,
upper_bound=parameter.max_value)
else:
print("Using uniform prior for %s" % parameter_name)
parameter.prior = uniform_prior(lower_bound=parameter.min_value,
upper_bound=parameter.max_value,
value=1.0)
def get_randomized_background_counts(self):
# Now randomize the expectations.
_, background_model_counts = self.get_current_value()
# We cannot generate variates with zero sigma. They variates from those channel will always be zero
# This is a limitation of this whole idea. However, remember that by construction an error of zero
# it is only allowed when the background counts are zero as well.
idx = (self._spectrum_plugin.background_count_errors > 0)
randomized_background_counts = np.zeros_like(background_model_counts)
randomized_background_counts[idx] = np.random.normal(loc=background_model_counts[idx],
scale=self._spectrum_plugin.background_count_errors[idx])
# Issue a warning if the generated background is less than zero, and fix it by placing it at zero
idx = (randomized_background_counts < 0) # type: np.ndarray
negative_background_n = np.sum(idx)
if negative_background_n > 0:
custom_warnings.warn("Generated background has negative counts "
"in %i channels. Fixing them to zero" % (negative_background_n))
randomized_background_counts[idx] = 0
return randomized_background_counts
def __init__(self, start, stop, parameter_values, likelihood_values,
n_integration_points):
# Make sure there is no NaN or infinity
assert np.all(np.isfinite(
likelihood_values)), "Infinity or NaN in likelihood values"
likelihood_values = np.asarray(likelihood_values)
parameter_values = np.asarray(parameter_values)
self._start = start
self._stop = stop
# Make sure the number of integration points is uneven, and that there are at minimum 11 points
# n_integration_points = max(int(n_integration_points), 11)
if n_integration_points % 2 == 0:
# n_points is even, it shouldn't be otherwise things like Simpson rule will have problems
n_integration_points += 1
custom_warnings.warn(
"The number of integration points should not be even. Adding +1"
)
self._n_integration_points = int(n_integration_points)
# Build interpolation of the likelihood curve
self._minus_likelihood_interp = scipy.interpolate.InterpolatedUnivariateSpline(
np.log10(parameter_values), -likelihood_values, k=1, ext=0)
# Find maximum of loglike
idx = likelihood_values.argmax()
self._min_par_value = parameter_values.min()
self._max_par_value = parameter_values.max()
res = scipy.optimize.minimize_scalar(
self._minus_likelihood_interp,
bounds=(np.log10(self._min_par_value),
np.log10(self._max_par_value)),
method="bounded",
options={
"maxiter": 10000,
"disp": True,
"xatol": 1e-3
},
)
# res = scipy.optimize.minimize(self._minus_likelihood_interp, x0=[np.log10(parameter_values[idx])],
# jac=lambda x:self._minus_likelihood_interp.derivative(1)(x),
# # bounds=(self._min_par_value, self._max_par_value),
# # method='bounded',
# tol=1e-3,
# options={'maxiter': 10000, 'disp': True})
assert res.success, "Could not find minimum"
self._minimum = (10**res.x, float(res.fun))
def _check_fullsky(self, method_name):
if not self._fullsky:
custom_warnings.warn("Attempting to use method %s, but fullsky=False during construction. "
"This might fail. If it does, specify `fullsky=True` when instancing "
"the plugin and try again." % method_name,
NoFullSky)
def _check_fullsky(self, method_name):
if not self._fullsky:
custom_warnings.warn(
"Attempting to use method %s, but fullsky=False during construction. "
"This might fail. If it does, specify `fullsky=True` when instancing "
"the plugin and try again." % method_name, NoFullSky)
def get_model(self, use_association_name=True):
assert (
self._last_query_results is not None
), "You have to run a query before getting a model"
# Loop over the table and build a source for each entry
sources = []
source_names = []
for name, row in self._last_query_results.T.items():
if name[-1] == "e":
# Extended source
custom_warnings.warn(
"Source %s is extended, support for extended source is not here yet. I will ignore"
"it" % name
)
# If there is an association and use_association is True, use that name, otherwise the 3FGL name
if row["assoc_name"] != "" and use_association_name:
this_name = row["assoc_name"]
# The crab is the only source which is present more than once in the 3FGL
if this_name == "Crab Nebula":
if name[-1] == "i":
this_name = "Crab_IC"
elif name[-1] == "s":
this_name = "Crab_synch"
else:
this_name = "Crab_pulsar"
else:
this_name = name
# in the 4FGL name there are more sources with the same name: this nwill avod any duplicates:
i = 1
while this_name in source_names:
this_name += str(i)
i += 1
pass
# By default all sources are fixed. The user will free the one he/she will need
source_names.append(this_name)
this_source = _get_point_source_from_3fgl(this_name, row, fix=True)
sources.append(this_source)
return ModelFrom3FGL(self.ra_center, self.dec_center, *sources)
def check_roi_inside_model(self):
active_pixels = self.active_pixels(self._original_nside)
radius = np.rad2deg(self._model_radius_radians)
ra, dec = self.ra_dec_center
temp_roi = HealpixConeROI(data_radius = radius , model_radius=radius, ra=ra, dec=dec)
model_pixels = temp_roi.active_pixels( self._original_nside )
if not all(p in model_pixels for p in active_pixels):
custom_warnings.warn("Some pixels inside your ROI are not contained in the model map.")
def get_number_of_data_points( self ):
"""
Number of data point = number of pixels.
Implemented in liff as the number of pixels in the ROI per analysis bin.
"""
try:
pixels_per_bin = np.array( self._theLikeHAWC.GetNumberOfPixels() )
return int(np.sum( pixels_per_bin ))
except AttributeError:
custom_warnings.warn(
"_theLikeHAWC.GetNumberOfPixels() not available, values for statistical measurements such as AIC or BIC are unreliable. Please update your aerie version." )
return 1
def get_model(self, use_association_name=True):
assert self._last_query_results is not None, "You have to run a query before getting a model"
# Loop over the table and build a source for each entry
sources = []
source_names=[]
for name, row in self._last_query_results.T.iteritems():
if name[-1] == 'e':
# Extended source
custom_warnings.warn("Source %s is extended, support for extended source is not here yet. I will ignore"
"it" % name)
# If there is an association and use_association is True, use that name, otherwise the 3FGL name
if row['assoc_name'] != '' and use_association_name:
this_name = row['assoc_name']
# The crab is the only source which is present more than once in the 3FGL
if this_name == 'Crab Nebula':
if name[-1] == 'i':
this_name = "Crab_IC"
elif name[-1] == "s":
this_name = "Crab_synch"
else:
this_name = "Crab_pulsar"
else:
this_name = name
# in the 4FGL name there are more sources with the same name: this nwill avod any duplicates:
i=1
while this_name in source_names:
this_name+=str(i)
i+=1
pass
# By default all sources are fixed. The user will free the one he/she will need
source_names.append(this_name)
this_source = _get_point_source_from_3fgl(this_name, row, fix=True)
sources.append(this_source)
return ModelFrom3FGL(self.ra_center, self.dec_center, *sources)
def get_number_of_data_points(self):
"""
Number of data point = number of pixels.
Implemented in liff as the number of pixels in the ROI per analysis bin.
"""
try:
pixels_per_bin = np.array(self._theLikeHAWC.GetNumberOfPixels())
return int(np.sum(pixels_per_bin))
except AttributeError:
custom_warnings.warn(
"_theLikeHAWC.GetNumberOfPixels() not available, values for statistical measurements such as AIC or BIC are unreliable. Please update your aerie version."
)
return 1
def __init__(self, response_file_name, dec_bins, response_bins):
self._response_file_name = response_file_name
self._dec_bins = dec_bins
self._response_bins = response_bins
if len(dec_bins) < 2:
custom_warnings.warn(
"Only {0} dec bins given in {1}, will not try to interpolate.".
format(len(dec_bins), response_file_name))
custom_warnings.warn(
"Single-dec-bin mode is intended for development work only at this time and may not work with extended sources."
)
def restore_best_fit(self):
"""
Restore the model to its best fit
:return: (none)
"""
if self._minimizer:
self._minimizer.restore_best_fit()
else:
custom_warnings.warn("Cannot restore best fit, since fit has not been executed.")
def __init__(self, start, stop, parameter_values, likelihood_values, n_integration_points):
# Make sure there is no NaN or infinity
assert np.all(np.isfinite(likelihood_values)), "Infinity or NaN in likelihood values"
likelihood_values = np.asarray(likelihood_values)
parameter_values = np.asarray(parameter_values)
self._start = start
self._stop = stop
# Make sure the number of integration points is uneven, and that there are at minimum 11 points
#n_integration_points = max(int(n_integration_points), 11)
if n_integration_points % 2 == 0:
# n_points is even, it shouldn't be otherwise things like Simpson rule will have problems
n_integration_points += 1
custom_warnings.warn("The number of integration points should not be even. Adding +1")
self._n_integration_points = int(n_integration_points)
# Build interpolation of the likelihood curve
self._minus_likelihood_interp = scipy.interpolate.InterpolatedUnivariateSpline(np.log10(parameter_values),
-likelihood_values,
k=1, ext=0)
# Find maximum of loglike
idx = likelihood_values.argmax()
self._min_par_value = parameter_values.min()
self._max_par_value = parameter_values.max()
res = scipy.optimize.minimize_scalar(self._minus_likelihood_interp,
bounds=(np.log10(self._min_par_value), np.log10(self._max_par_value)),
method='bounded',
options={'maxiter': 10000, 'disp': True, 'xatol': 1e-3})
# res = scipy.optimize.minimize(self._minus_likelihood_interp, x0=[np.log10(parameter_values[idx])],
# jac=lambda x:self._minus_likelihood_interp.derivative(1)(x),
# # bounds=(self._min_par_value, self._max_par_value),
# # method='bounded',
# tol=1e-3,
# options={'maxiter': 10000, 'disp': True})
assert res.success, "Could not find minimum"
self._minimum = (10**res.x, float(res.fun))
def __init__(self, name, veritas_root_data):
# Open file
f = ROOT.TFile(veritas_root_data)
try:
# Loop over the runs
keys = get_list_of_keys(f)
finally:
f.Close()
# Get the names of all runs included
run_names = [x for x in keys if x.find("run") == 0]
self._runs_like = collections.OrderedDict()
for run_name in run_names:
# Build the VERITASRun class
this_run = VERITASRun(veritas_root_data, run_name)
this_run.display()
if this_run.total_counts == 0 or this_run.total_background_counts == 0:
custom_warnings.warn(
"%s has 0 source or bkg counts, cannot use it." % run_name
)
continue
else:
# Get background spectrum and observation spectrum (with response)
# this_observation = this_run.get_spectrum()
# this_background = this_run.get_background_spectrum()
#
# self._runs_like[run_name] = DispersionSpectrumLike(run_name,
# this_observation,
# this_background)
#
# self._runs_like[run_name].set_active_measurements("c50-c130")
self._runs_like[run_name] = this_run
super(VERITASLike, self).__init__(name, {})
def restore_best_fit(self):
"""
Restore the model to its best fit
:return: (none)
"""
if self._minimizer:
self._minimizer.restore_best_fit()
else:
custom_warnings.warn(
"Cannot restore best fit, since fit has not been executed.")
def get_source_map(self, response_bin_id, tag=None):
# Get current point source position
# NOTE: this might change if the point source position is free during the fit,
# that's why it is here
ra_src, dec_src = self._source.position.ra.value, self._source.position.dec.value
if (ra_src, dec_src) != self._last_processed_position:
# Position changed (or first iteration), let's update the dec bins
self._update_dec_bins(dec_src)
self._last_processed_position = (ra_src, dec_src)
# Get the current response bin
response_energy_bin = self._response_energy_bins[response_bin_id]
psf_interpolator = self._psf_interpolators[response_bin_id]
# Get the PSF image
# This is cached inside the PSF class, so that if the position doesn't change this line
# is very fast
this_map = psf_interpolator.point_source_image(ra_src, dec_src)
# Check that the point source is contained in the ROI, if not print a warning
if not np.isclose(this_map.sum(), 1.0, rtol=1e-2):
custom_warnings.warn(
"PSF for source %s is not entirely contained in ROI" %
self._name)
# Compute the fluxes from the spectral function at the same energies as the simulated function
energy_centers_keV = response_energy_bin.sim_energy_bin_centers * 1e9 # astromodels expects energies in keV
# This call needs to be here because the parameters of the model might change,
# for example during a fit
source_diff_spectrum = self._source(energy_centers_keV, tag=tag)
# Transform from keV^-1 cm^-2 s^-1 to TeV^-1 cm^-2 s^-1
source_diff_spectrum *= 1e9
# Re-weight the detected counts
scale = source_diff_spectrum / response_energy_bin.sim_differential_photon_fluxes
# Now return the map multiplied by the scale factor
return np.sum(
scale * response_energy_bin.sim_signal_events_per_bin) * this_map
def get_model(self, use_association_name=True):
assert self._last_query_results is not None, "You have to run a query before getting a model"
# Loop over the table and build a source for each entry
sources = []
for name, row in self._last_query_results.T.iteritems():
if name[-1] == 'e':
# Extended source
custom_warnings.warn(
"Source %s is extended, support for extended source is not here yet. I will ignore"
"it" % name)
# If there is an association and use_association is True, use that name, otherwise the 3FGL name
if row['assoc_name_1'] != '' and use_association_name:
this_name = row['assoc_name_1']
# The crab is the only source which is present more than once in the 3FGL
if this_name == "Crab":
if name[-1] == 'i':
this_name = "Crab_IC"
elif name[-1] == "s":
this_name = "Crab_synch"
else:
this_name = "Crab_pulsar"
else:
this_name = name
# By default all sources are fixed. The user will free the one he/she will need
this_source = _get_point_source_from_3fgl(this_name, row, fix=True)
sources.append(this_source)
return ModelFrom3FGL(self.ra_center, self.dec_center, *sources)
def __init__(self, name, veritas_root_data):
# Open file
f = ROOT.TFile(veritas_root_data)
try:
# Loop over the runs
keys = get_list_of_keys(f)
finally:
f.Close()
# Get the names of all runs included
run_names = filter(lambda x: x.find("run") == 0, keys)
self._runs_like = collections.OrderedDict()
for run_name in run_names:
# Build the VERITASRun class
this_run = VERITASRun(veritas_root_data, run_name)
this_run.display()
if this_run.total_counts == 0 or this_run.total_background_counts == 0:
custom_warnings.warn("%s has 0 source or bkg counts, cannot use it." % run_name)
continue
else:
# Get background spectrum and observation spectrum (with response)
# this_observation = this_run.get_spectrum()
# this_background = this_run.get_background_spectrum()
#
# self._runs_like[run_name] = DispersionSpectrumLike(run_name,
# this_observation,
# this_background)
#
# self._runs_like[run_name].set_active_measurements("c50-c130")
self._runs_like[run_name] = this_run
super(VERITASLike, self).__init__(name, {})
def _source_is_valid(self, source):
warn_string = "The trigger %s is not valid. Must be in the form GRB080916009" % source
match = _trigger_name_match.match(source)
if match is None:
custom_warnings.warn(warn_string)
answer = False
else:
answer = True
return answer
def __init__(self, joint_likelihood_instance0, joint_likelihood_instance1):
self._joint_likelihood_instance0 = (joint_likelihood_instance0
) # type: JointLikelihood
self._joint_likelihood_instance1 = (joint_likelihood_instance1
) # type: JointLikelihood
# Restore best fit and store the reference value for the likelihood
self._joint_likelihood_instance0.restore_best_fit()
self._joint_likelihood_instance1.restore_best_fit()
self._reference_TS = 2 * (
self._joint_likelihood_instance0.current_minimum -
self._joint_likelihood_instance1.current_minimum)
# Safety check that the user has provided the models in the right order
if self._reference_TS < 0:
custom_warnings.warn(
"The reference TS is negative, either you specified the likelihood objects "
"in the wrong order, or the fit for the alternative hyp. has failed. Since the "
"two hyp. are nested, by definition the more complex hypothesis should give a "
"better or equal fit with respect to the null hypothesis.")
# Check that the dataset is the same
if (self._joint_likelihood_instance1.data_list !=
self._joint_likelihood_instance0.data_list):
# Since this check might fail if the user loaded twice the same data, only issue a warning, instead of
# an exception.
custom_warnings.warn(
"The data lists for the null hyp. and for the alternative hyp. seems to be different."
" If you loaded twice the same data and made the same data selections, disregard this "
"message. Otherwise, consider the fact that the LRT is meaningless if the two data "
"sets are not exactly the same. We will use the data loaded as part of the null "
"hypothesis JointLikelihood object",
RuntimeWarning,
)
# For saving pha files
self._save_pha = False
self._data_container = []
def _log_like(self, trial_values):
"""Compute the log-likelihood, used in the parallel tempering sampling"""
# Compute the log-likelihood
# Set the parameters to their trial values
for i, (src_name, param_name) in enumerate(self._free_parameters.keys()):
this_param = self._likelihood_model.parameters[src_name][param_name]
this_param.setValue(trial_values[i])
# Get the value of the log-likelihood for this parameters
try:
# Loop over each dataset and get the likelihood values for each set
log_like_values = map(lambda dataset: dataset.get_log_like(), self.data_list.values())
except ModelAssertionViolation:
# Fit engine or sampler outside of allowed zone
return -numpy.inf
except:
# We don't want to catch more serious issues
raise
# Sum the values of the log-like
log_like = numpy.sum(log_like_values)
if not numpy.isfinite(log_like):
# Issue warning
custom_warnings.warn("Likelihood value is infinite for parameters %s" % trial_values, LikelihoodIsInfinite)
return -numpy.inf
return log_like
def compute_covariance_matrix(self, function, best_fit_parameters):
"""
Compute the covariance matrix of this fit
:param function: the loglike for the fit
:param best_fit_parameters: the best fit parameters
:return:
"""
minima = np.zeros_like(best_fit_parameters) - 100
maxima = np.zeros_like(best_fit_parameters) + 100
try:
hessian_matrix = get_hessian(function, best_fit_parameters, minima,
maxima)
except ParameterOnBoundary:
custom_warnings.warn(
"One or more of the parameters are at their boundaries. Cannot compute covariance and"
" errors", CannotComputeCovariance)
n_dim = len(best_fit_parameters)
self._cov_matrix = np.zeros((n_dim, n_dim)) * np.nan
# Invert it to get the covariance matrix
try:
covariance_matrix = np.linalg.inv(hessian_matrix)
self._cov_matrix = covariance_matrix
except:
custom_warnings.warn(
"Cannot invert Hessian matrix, looks like the matrix is singluar"
)
n_dim = len(best_fit_parameters)
self._cov_matrix = np.zeros((n_dim, n_dim)) * np.nan
def _log_like(self, trial_values):
"""Compute the log-likelihood"""
# Get the value of the log-likelihood for this parameters
try:
# Loop over each dataset and get the likelihood values for each set
log_like_values = [
dataset.get_log_like()
for dataset in list(self._data_list.values())
]
except ModelAssertionViolation:
# Fit engine or sampler outside of allowed zone
return -np.inf
except:
# We don't want to catch more serious issues
raise
# Sum the values of the log-like
log_like = np.sum(log_like_values)
if not np.isfinite(log_like):
# Issue warning
custom_warnings.warn(
"Likelihood value is infinite for parameters %s" %
trial_values,
LikelihoodIsInfinite,
)
return -np.inf
return log_like
def compute_covariance_matrix(self, function, best_fit_parameters):
"""
Compute the covariance matrix of this fit
:param function: the loglike for the fit
:param best_fit_parameters: the best fit parameters
:return:
"""
minima = np.zeros_like(best_fit_parameters) - 100
maxima = np.zeros_like(best_fit_parameters) + 100
try:
hessian_matrix = get_hessian(function, best_fit_parameters, minima, maxima)
except ParameterOnBoundary:
custom_warnings.warn("One or more of the parameters are at their boundaries. Cannot compute covariance and"
" errors", CannotComputeCovariance)
n_dim = len(best_fit_parameters)
self._cov_matrix = np.zeros((n_dim, n_dim)) * np.nan
# Invert it to get the covariance matrix
try:
covariance_matrix = np.linalg.inv(hessian_matrix)
self._cov_matrix = covariance_matrix
except:
custom_warnings.warn("Cannot invert Hessian matrix, looks like the matrix is singluar")
n_dim = len(best_fit_parameters)
self._cov_matrix = np.zeros((n_dim, n_dim)) * np.nan
def get_randomized_source_counts(self, source_model_counts):
idx = (self._spectrum_plugin.observed_count_errors > 0)
randomized_source_counts = np.zeros_like(source_model_counts)
randomized_source_counts[idx] = np.random.normal(loc=source_model_counts[idx],
scale=self._spectrum_plugin.observed_count_errors[idx])
# Issue a warning if the generated background is less than zero, and fix it by placing it at zero
idx = (randomized_source_counts < 0) # type: np.ndarray
negative_source_n = np.sum(idx)
if negative_source_n > 0:
custom_warnings.warn("Generated source has negative counts "
"in %i channels. Fixing them to zero" % (negative_source_n))
randomized_source_counts[idx] = 0
return randomized_source_counts
def get_randomized_source_counts(self, source_model_counts):
idx = (self._spectrum_plugin.observed_count_errors > 0)
randomized_source_counts = np.zeros_like(source_model_counts)
randomized_source_counts[idx] = np.random.normal(loc=source_model_counts[idx],
scale=self._spectrum_plugin.observed_count_errors[idx])
# Issue a warning if the generated background is less than zero, and fix it by placing it at zero
idx = (randomized_source_counts < 0) # type: np.ndarray
negative_source_n = np.sum(idx)
if negative_source_n > 0:
custom_warnings.warn("Generated source has negative counts "
"in %i channels. Fixing them to zero" % (negative_source_n))
randomized_source_counts[idx] = 0
return randomized_source_counts
def temporary_directory(prefix='', within_directory=None):
"""
This context manager creates a temporary directory in the most secure possible way (with no race condition), and
removes it at the end.
:param prefix: the directory name will start with this prefix, if specified
:param within_directory: create within a specific directory (assumed to exist). Otherwise, it will be created in the
default system temp directory (/tmp in unix)
:return: the absolute pathname of the provided directory
"""
directory = tempfile.mkdtemp(prefix=prefix, dir=within_directory)
yield directory
try:
shutil.rmtree(directory)
except:
custom_warnings.warn("Couldn't remove temporary directory %s" % directory)
def temporary_directory(prefix='', within_directory=None):
"""
This context manager creates a temporary directory in the most secure possible way (with no race condition), and
removes it at the end.
:param prefix: the directory name will start with this prefix, if specified
:param within_directory: create within a specific directory (assumed to exist). Otherwise, it will be created in the
default system temp directory (/tmp in unix)
:return: the absolute pathname of the provided directory
"""
directory = tempfile.mkdtemp(prefix=prefix, dir=within_directory)
yield directory
try:
shutil.rmtree(directory)
except:
custom_warnings.warn("Couldn't remove temporary directory %s" % directory)
def _source_is_valid(self, source):
"""
checks if source name is valid for the 3FGL catalog
:param source: source name
:return: bool
"""
warn_string = "The trigger %s is not valid. Must be in the form '3FGL J0000.0+0000'" % source
match = _3FGL_name_match.match(source)
if match is None:
custom_warnings.warn(warn_string)
answer = False
else:
answer = True
return answer
def _gbm_and_lle_valid_source_check(source):
"""
checks if source name is valid for both GBM and LLE data
:param source: source name
:return: bool
"""
warn_string = "The trigger %s is not valid. Must be in the form GRB080916009" % source
match = _trigger_name_match.match(source)
if match is None:
custom_warnings.warn(warn_string)
answer = False
else:
answer = True
return answer
def __init__(self, joint_likelihood_instance0, joint_likelihood_instance1):
self._joint_likelihood_instance0 = joint_likelihood_instance0 # type: JointLikelihood
self._joint_likelihood_instance1 = joint_likelihood_instance1 # type: JointLikelihood
# Restore best fit and store the reference value for the likelihood
self._joint_likelihood_instance0.restore_best_fit()
self._joint_likelihood_instance1.restore_best_fit()
self._reference_TS = 2 * (self._joint_likelihood_instance0.current_minimum -
self._joint_likelihood_instance1.current_minimum)
# Safety check that the user has provided the models in the right order
if self._reference_TS < 0:
custom_warnings.warn("The reference TS is negative, either you specified the likelihood objects "
"in the wrong order, or the fit for the alternative hyp. has failed. Since the "
"two hyp. are nested, by definition the more complex hypothesis should give a "
"better or equal fit with respect to the null hypothesis.")
# Check that the dataset is the same
if self._joint_likelihood_instance1.data_list != self._joint_likelihood_instance0.data_list:
# Since this check might fail if the user loaded twice the same data, only issue a warning, instead of
# an exception.
custom_warnings.warn("The data lists for the null hyp. and for the alternative hyp. seems to be different."
" If you loaded twice the same data and made the same data selections, disregard this "
"message. Otherwise, consider the fact that the LRT is meaningless if the two data "
"sets are not exactly the same. We will use the data loaded as part of the null "
"hypothesis JointLikelihood object", RuntimeWarning)
# For saving pha files
self._save_pha = False
self._data_container = []
def _log_like(self, trial_values):
"""Compute the log-likelihood"""
# Get the value of the log-likelihood for this parameters
try:
# Loop over each dataset and get the likelihood values for each set
log_like_values = map(lambda dataset: dataset.get_log_like(), self._data_list.values())
except ModelAssertionViolation:
# Fit engine or sampler outside of allowed zone
return -np.inf
except:
# We don't want to catch more serious issues
raise
# Sum the values of the log-like
log_like = np.sum(log_like_values)
if not np.isfinite(log_like):
# Issue warning
custom_warnings.warn("Likelihood value is infinite for parameters %s" % trial_values,
LikelihoodIsInfinite)
return -np.inf
return log_like
def _get_posterior(self, trial_values):
"""Compute the posterior for the normal sampler"""
# Assign this trial values to the parameters and
# store the corresponding values for the priors
self._update_free_parameters()
assert len(self._free_parameters) == len(trial_values), ("Something is wrong. Number of free parameters "
"do not match the number of trial values.")
log_prior = 0
for i, (parameter_name, parameter) in enumerate(self._free_parameters.iteritems()):
prior_value = parameter.prior(trial_values[i])
if prior_value == 0:
# Outside allowed region of parameter space
return -numpy.inf
else:
parameter.value = trial_values[i]
log_prior += math.log10(prior_value)
# Get the value of the log-likelihood for this parameters
try:
# Loop over each dataset and get the likelihood values for each set
log_like_values = map(lambda dataset: dataset.get_log_like(), self.data_list.values())
except ModelAssertionViolation:
# Fit engine or sampler outside of allowed zone
return -numpy.inf
except:
# We don't want to catch more serious issues
raise
# Sum the values of the log-like
log_like = numpy.sum(log_like_values)
if not numpy.isfinite(log_like):
# Issue warning
custom_warnings.warn("Likelihood value is infinite for parameters %s" % trial_values, LikelihoodIsInfinite)
return -numpy.inf
#print("Log like is %s, log_prior is %s, for trial values %s" % (log_like, log_prior,trial_values))
return log_like + log_prior
def _get_one_error(self, parameter_name, target_delta_log_like, sign=-1):
"""
A generic procedure to numerically compute the error for the parameters. You can override this if the
minimizer provides its own method to compute the error of one parameter. If it provides a method to compute
all errors are once, override the _get_errors method instead.
:param parameter_name:
:param target_delta_log_like:
:param sign:
:return:
"""
# Since the procedure might find a better minimum, we can repeat it
# up to a maximum of 10 times
repeats = 0
while repeats < 10:
# Let's start optimistic...
repeat = False
repeats += 1
# Restore best fit (which also updates the internal parameter dictionary)
self.restore_best_fit()
current_value, current_delta, current_min, current_max = self._internal_parameters[parameter_name]
best_fit_value = current_value
if sign == -1:
extreme_allowed = current_min
else:
extreme_allowed = current_max
# If the parameter has no boundary in the direction we are sampling, put a hard limit on
# 10 times the current value (to avoid looping forever)
if extreme_allowed is None:
extreme_allowed = best_fit_value + sign * 10 * abs(best_fit_value)
# We need to look for a value for the parameter where the difference between the minimum of the
# log-likelihood and the likelihood for that value differs by more than target_delta_log_likelihood.
# This is needed by the root-finding procedure, which needs to know an interval where the biased likelihood
# function (see below) changes sign
trials = best_fit_value + sign * np.linspace(0.1, 0.9, 9) * abs(best_fit_value)
trials = np.append(trials, extreme_allowed)
# Make sure we don't go below the allowed minimum or above the allowed maximum
if sign == -1:
np.clip(trials, extreme_allowed, np.inf, trials)
else:
np.clip(trials, -np.inf, extreme_allowed, trials)
# There might be more than one value which was below the minimum (or above the maximum), so let's
# take only unique elements
trials = np.unique(trials)
trials.sort()
if sign == -1:
trials = trials[::-1]
# At this point we have a certain number of unique trials which always
# contain the allowed minimum (or maximum)
minimum_bound = None
maximum_bound = None
# Instance the profile likelihood function
pl = ProfileLikelihood(self, [parameter_name])
for i, trial in enumerate(trials):
this_log_like = pl([trial])
delta = this_log_like - self._m_log_like_minimum
if delta < -0.1:
custom_warnings.warn("Found a better minimum (%.2f) for %s = %s during error "
"computation." % (this_log_like, parameter_name, trial),
BetterMinimumDuringProfiling)
xs = map(lambda x:x.value, self.parameters.values())
self._store_fit_results(xs, this_log_like, None)
repeat = True
break
if delta > target_delta_log_like:
bound1 = trial
if i > 0:
bound2 = trials[i-1]
else:
bound2 = best_fit_value
minimum_bound = min(bound1, bound2)
maximum_bound = max(bound1, bound2)
repeat = False
break
if repeat:
# We found a better minimum, restart from scratch
custom_warnings.warn("Restarting search...", RuntimeWarning)
continue
if minimum_bound is None:
# Cannot find error in this direction (it's probably outside the allowed boundaries)
custom_warnings.warn("Cannot find boundary for parameter %s" % parameter_name, CannotComputeErrors)
error = np.nan
break
else:
# Define the "biased likelihood", since brenq only finds zeros of function
biased_likelihood = lambda x: pl(x) - self._m_log_like_minimum - target_delta_log_like
try:
precise_bound = scipy.optimize.brentq(biased_likelihood,
minimum_bound,
maximum_bound,
xtol=1e-5, maxiter=1000) #type: float
except:
custom_warnings.warn("Cannot find boundary for parameter %s" % parameter_name, CannotComputeErrors)
error = np.nan
break
error = precise_bound - best_fit_value
break
return error
def get_contours(self, param_1, param_1_minimum, param_1_maximum, param_1_n_steps,
param_2=None, param_2_minimum=None, param_2_maximum=None, param_2_n_steps=None,
progress=True, **options):
"""
Generate confidence contours for the given parameters by stepping for the given number of steps between
the given boundaries. Call it specifying only source_1, param_1, param_1_minimum and param_1_maximum to
generate the profile of the likelihood for parameter 1. Specify all parameters to obtain instead a 2d
contour of param_1 vs param_2.
NOTE: if using parallel computation, param_1_n_steps must be an integer multiple of the number of running
engines. If that is not the case, the code will reduce the number of steps to match that requirement, and
issue a warning
:param param_1: fully qualified name of the first parameter or parameter instance
:param param_1_minimum: lower bound for the range for the first parameter
:param param_1_maximum: upper bound for the range for the first parameter
:param param_1_n_steps: number of steps for the first parameter
:param param_2: fully qualified name of the second parameter or parameter instance
:param param_2_minimum: lower bound for the range for the second parameter
:param param_2_maximum: upper bound for the range for the second parameter
:param param_2_n_steps: number of steps for the second parameter
:param progress: (True or False) whether to display progress or not
:param log: by default the steps are taken linearly. With this optional parameter you can provide a tuple of
booleans which specify whether the steps are to be taken logarithmically. For example,
'log=(True,False)' specify that the steps for the first parameter are to be taken logarithmically,
while they are linear for the second parameter. If you are generating the profile for only one
parameter, you can specify 'log=(True,)' or 'log=(False,)' (optional)
:return: a tuple containing an array corresponding to the steps for the first parameter, an array corresponding
to the steps for the second parameter (or None if stepping only in one direction), a matrix of size
param_1_steps x param_2_steps containing the value of the function at the corresponding points in the
grid. If param_2_steps is None (only one parameter), then this reduces to an array of
size param_1_steps.
"""
if hasattr(param_1,"value"):
# Substitute with the name
param_1 = param_1.path
if hasattr(param_2,'value'):
param_2 = param_2.path
# Check that the parameters exist
assert param_1 in self._likelihood_model.free_parameters, "Parameter %s is not a free parameters of the " \
"current model" % param_1
if param_2 is not None:
assert param_2 in self._likelihood_model.free_parameters, "Parameter %s is not a free parameters of the " \
"current model" % param_2
# Check that we have a valid fit
assert self._current_minimum is not None, "You have to run the .fit method before calling get_contours."
# Then restore the best fit
self._minimizer.restore_best_fit()
# Check minimal assumptions about the procedure
assert not (param_1 == param_2), "You have to specify two different parameters"
assert param_1_minimum < param_1_maximum, "Minimum larger than maximum for parameter 1"
min1, max1 = self.likelihood_model[param_1].bounds
if min1 is not None:
assert param_1_minimum >= min1, "Requested low range for parameter %s (%s) " \
"is below parameter minimum (%s)" % (param_1, param_1_minimum, min1)
if max1 is not None:
assert param_1_maximum <= max1, "Requested hi range for parameter %s (%s) " \
"is above parameter maximum (%s)" % (param_1, param_1_maximum, max1)
if param_2 is not None:
min2, max2 = self.likelihood_model[param_2].bounds
if min2 is not None:
assert param_2_minimum >= min2, "Requested low range for parameter %s (%s) " \
"is below parameter minimum (%s)" % (param_2, param_2_minimum, min2)
if max2 is not None:
assert param_2_maximum <= max2, "Requested hi range for parameter %s (%s) " \
"is above parameter maximum (%s)" % (param_2, param_2_maximum, max2)
# Check whether we are parallelizing or not
if not threeML_config['parallel']['use-parallel']:
a, b, cc = self.minimizer.contours(param_1, param_1_minimum, param_1_maximum, param_1_n_steps,
param_2, param_2_minimum, param_2_maximum, param_2_n_steps,
progress, **options)
# Collapse the second dimension of the results if we are doing a 1d contour
if param_2 is None:
cc = cc[:, 0]
else:
# With parallel computation
# In order to distribute fairly the computation, the strategy is to parallelize the computation
# by assigning to the engines one "line" of the grid at the time
# Connect to the engines
client = ParallelClient(**options)
# Get the number of engines
n_engines = client.get_number_of_engines()
# Check whether the number of threads is larger than the number of steps in the first direction
if n_engines > param_1_n_steps:
n_engines = int(param_1_n_steps)
custom_warnings.warn("The number of engines is larger than the number of steps. Using only %s engines."
% n_engines, ReducingNumberOfThreads)
# Check if the number of steps is divisible by the number
# of threads, otherwise issue a warning and make it so
if float(param_1_n_steps) % n_engines != 0:
# Set the number of steps to an integer multiple of the engines
# (note that // is the floor division, also called integer division)
param_1_n_steps = (param_1_n_steps // n_engines) * n_engines
custom_warnings.warn("Number of steps is not a multiple of the number of threads. Reducing steps to %s"
% param_1_n_steps, ReducingNumberOfSteps)
# Compute the number of splits, i.e., how many lines in the grid for each engine.
# (note that this is guaranteed to be an integer number after the previous checks)
p1_split_steps = param_1_n_steps // n_engines
# Prepare arrays for results
if param_2 is None:
# One array
pcc = np.zeros(param_1_n_steps)
pa = np.linspace(param_1_minimum, param_1_maximum, param_1_n_steps)
pb = None
else:
pcc = np.zeros((param_1_n_steps, param_2_n_steps))
# Prepare the two axes of the parameter space
pa = np.linspace(param_1_minimum, param_1_maximum, param_1_n_steps)
pb = np.linspace(param_2_minimum, param_2_maximum, param_2_n_steps)
# Define the parallel worker which will go through the computation
# NOTE: I only divide
# on the first parameter axis so that the different
# threads are more or less well mixed for points close and
# far from the best fit
def worker(start_index):
# Re-create the minimizer
backup_freeParameters = map(lambda x:x.value, self._likelihood_model.free_parameters.values())
this_minimizer = self._get_minimizer(self.minus_log_like_profile,
self._free_parameters)
this_p1min = pa[start_index * p1_split_steps]
this_p1max = pa[(start_index + 1) * p1_split_steps - 1]
# print("From %s to %s" % (this_p1min, this_p1max))
aa, bb, ccc = this_minimizer.contours(param_1, this_p1min, this_p1max, p1_split_steps,
param_2, param_2_minimum, param_2_maximum,
param_2_n_steps,
progress=True, **options)
# Restore best fit values
for val, par in zip(backup_freeParameters, self._likelihood_model.free_parameters.values()):
par.value = val
return ccc
# Now re-assemble the vector of results taking the different parts from the engines
all_results = client.execute_with_progress_bar(worker, range(n_engines), chunk_size=1)
for i, these_results in enumerate(all_results):
if param_2 is None:
pcc[i * p1_split_steps: (i + 1) * p1_split_steps] = these_results[:, 0]
else:
pcc[i * p1_split_steps: (i + 1) * p1_split_steps, :] = these_results
# Give the results the names that the following code expect. These are kept separate for debugging
# purposes
cc = pcc
a = pa
b = pb
# Here we have done the computation, in parallel computation or not. Let's make the plot
# with the contour
if param_2 is not None:
# 2d contour
fig = self._plot_contours("%s" % (param_1), a, "%s" % (param_2,), b, cc)
else:
# 1d contour (i.e., a profile)
fig = self._plot_profile("%s" % (param_1), a, cc)
# Check if we found a better minimum. This shouldn't happen, but in case of very difficult fit
# it might.
if self._current_minimum - cc.min() > 0.1:
if param_2 is not None:
idx = cc.argmin()
aidx, bidx = np.unravel_index(idx, cc.shape)
print("\nFound a better minimum: %s with %s = %s and %s = %s. Run again your fit starting from here."
% (cc.min(), param_1, a[aidx], param_2, b[bidx]))
else:
idx = cc.argmin()
print("Found a better minimum: %s with %s = %s. Run again your fit starting from here."
% (cc.min(), param_1, a[idx]))
return a, b, cc, fig
def __init__(self, name, fermipy_config):
"""
:param name: a name for this instance
:param fermipy_config: either a path to a YAML configuration file or a dictionary containing the configuration
(see http://fermipy.readthedocs.io/)
"""
# There are no nuisance parameters
nuisance_parameters = {}
super(FermipyLike,
self).__init__(name, nuisance_parameters=nuisance_parameters)
# Check whether the provided configuration is a file
if not isinstance(fermipy_config, dict):
# Assume this is a file name
configuration_file = sanitize_filename(fermipy_config)
if not os.path.exists(fermipy_config):
log.critical("Configuration file %s does not exist" %
configuration_file)
# Read the configuration
with open(configuration_file) as f:
self._configuration = yaml.load(f, Loader=yaml.SafeLoader)
else:
# Configuration is a dictionary. Nothing to do
self._configuration = fermipy_config
# If the user provided a 'model' key, issue a warning, as the model will be defined
# later on and will overwrite the one contained in 'model'
if "model" in self._configuration:
custom_warnings.warn(
"The provided configuration contains a 'model' section, which is useless as it "
"will be overridden")
self._configuration.pop("model")
if "fileio" in self._configuration:
custom_warnings.warn(
"The provided configuration contains a 'fileio' section, which will be "
"overwritten")
self._configuration.pop("fileio")
# Now check that the data exists
# As minimum there must be a evfile and a scfile
if not "evfile" in self._configuration["data"]:
log.critical("You must provide a evfile in the data section")
if not "scfile" in self._configuration["data"]:
log.critical("You must provide a scfile in the data section")
for datum in self._configuration["data"]:
# Sanitize file name, as fermipy is not very good at handling relative paths or env. variables
filename = str(
sanitize_filename(self._configuration["data"][datum], True))
self._configuration["data"][datum] = filename
if not os.path.exists(self._configuration["data"][datum]):
log.critical("File %s (%s) not found" % (filename, datum))
# Prepare the 'fileio' part
# Save all output in a directory with a unique name which depends on the configuration,
# so that the same configuration will write in the same directory and fermipy will
# know that it doesn't need to recompute things
self._unique_id = "__%s" % _get_unique_tag_from_configuration(
self._configuration)
self._configuration["fileio"] = {"outdir": self._unique_id}
# Ensure that there is a complete definition of a Region Of Interest (ROI)
if not (("ra" in self._configuration["selection"]) and
("dec" in self._configuration["selection"])):
log.critical(
"You have to provide 'ra' and 'dec' in the 'selection' section of the configuration. Source name "
"resolution, as well as Galactic coordinates, are not currently supported"
)
# This is empty at the beginning, will be instanced in the set_model method
self._gta = None
def display_photometry_model_magnitudes(analysis, data=(), **kwargs):
"""
Display the fitted model count spectrum of one or more Spectrum plugins
NOTE: all parameters passed as keyword arguments that are not in the list below, will be passed as keyword arguments
to the plt.subplots() constructor. So for example, you can specify the size of the figure using figsize = (20,10)
:param args: one or more instances of Spectrum plugin
:param min_rate: (optional) rebin to keep this minimum rate in each channel (if possible). If one number is
provided, the same minimum rate is used for each dataset, otherwise a list can be provided with the minimum rate
for each dataset
:param data_cmap: (str) (optional) the color map used to extract automatically the colors for the data
:param model_cmap: (str) (optional) the color map used to extract automatically the colors for the models
:param data_colors: (optional) a tuple or list with the color for each dataset
:param model_colors: (optional) a tuple or list with the color for each folded model
:param show_legend: (optional) if True (default), shows a legend
:param step: (optional) if True (default), show the folded model as steps, if False, the folded model is plotted
with linear interpolation between each bin
:return: figure instance
"""
# If the user supplies a subset of the data, we will use that
if not data:
data_keys = analysis.data_list.keys()
else:
data_keys = data
# Now we want to make sure that we only grab OGIP plugins
new_data_keys = []
for key in data_keys:
# Make sure it is a valid key
if key in analysis.data_list.keys():
if isinstance(analysis.data_list[key], threeML.plugins.PhotometryLike.PhotometryLike):
new_data_keys.append(key)
else:
custom_warnings.warn("Dataset %s is not of the Photometery kind. Cannot be plotted by "
"display_photometry_model_magnitudes" % key)
if not new_data_keys:
RuntimeError(
'There were no valid Photometry data requested for plotting. Please use the detector names in the data list')
data_keys = new_data_keys
# Default is to show the model with steps
step = True
data_cmap = threeML_config['photo']['data plot cmap'] # plt.cm.rainbow
model_cmap = threeML_config['photo']['model plot cmap'] # plt.cm.nipy_spectral_r
# Legend is on by default
show_legend = True
# Default colors
data_colors = cmap_intervals(len(data_keys), data_cmap)
model_colors = cmap_intervals(len(data_keys), model_cmap)
# Now override defaults according to the optional keywords, if present
if 'show_legend' in kwargs:
show_legend = bool(kwargs.pop('show_legend'))
if 'step' in kwargs:
step = bool(kwargs.pop('step'))
if 'data_cmap' in kwargs:
data_cmap = plt.get_cmap(kwargs.pop('data_cmap'))
data_colors = cmap_intervals(len(data_keys), data_cmap)
if 'model_cmap' in kwargs:
model_cmap = kwargs.pop('model_cmap')
model_colors = cmap_intervals(len(data_keys), model_cmap)
if 'data_colors' in kwargs:
data_colors = kwargs.pop('data_colors')
assert len(data_colors) >= len(data_keys), "You need to provide at least a number of data colors equal to the " \
"number of datasets"
if 'model_colors' in kwargs:
model_colors = kwargs.pop('model_colors')
assert len(model_colors) >= len(
data_keys), "You need to provide at least a number of model colors equal to the " \
"number of datasets"
residual_plot = ResidualPlot(**kwargs)
# go thru the detectors
for key, data_color, model_color in zip(data_keys, data_colors, model_colors):
data = analysis.data_list[key] # type: threeML.plugins.PhotometryLike.PhotometryLike
# get the expected counts
avg_wave_length = data._filter_set.effective_wavelength.value #type: np.ndarray
# need to sort because filters are not always in order
sort_idx = avg_wave_length.argsort()
expected_model_magnitudes = data._get_total_expectation()[sort_idx]
magnitudes = data.magnitudes[sort_idx]
mag_errors= data.magnitude_errors[sort_idx]
avg_wave_length = avg_wave_length[sort_idx]
residuals = (expected_model_magnitudes - magnitudes) / mag_errors
widths = data._filter_set.wavelength_bounds.widths[sort_idx]
residual_plot.add_data(x=avg_wave_length,
y=magnitudes,
xerr=widths,
yerr=mag_errors,
residuals=residuals,
label=data._name,
color=data_color)
residual_plot.add_model(avg_wave_length,
expected_model_magnitudes,
label='%s Model' % data._name,
color=model_color)
return residual_plot.finalize(xlabel="Wavelength\n(%s)"%data._filter_set.waveunits,
ylabel='Magnitudes',
xscale='linear',
yscale='linear',
invert_y=True)
def from_gbm_cspec_or_ctime(cls, name, cspec_or_ctime_file, rsp_file, restore_background=None,
trigger_time=None,
poly_order=-1, verbose=True):
"""
A plugin to natively bin, view, and handle Fermi GBM TTE data.
A TTE event file are required as well as the associated response
Background selections are specified as
a comma separated string e.g. "-10-0,10-20"
Initial source selection is input as a string e.g. "0-5"
One can choose a background polynomial order by hand (up to 4th order)
or leave it as the default polyorder=-1 to decide by LRT test
:param name: name for your choosing
:param tte_file: GBM tte event file
:param rsp_file: Associated TTE CSPEC response file
:param trigger_time: trigger time if needed
:param poly_order: 0-4 or -1 for auto
:param unbinned: unbinned likelihood fit (bool)
:param verbose: verbose (bool)
"""
# self._default_unbinned = unbinned
# Load the relevant information from the TTE file
cdata = GBMCdata(cspec_or_ctime_file, rsp_file)
# Set a trigger time if one has not been set
if trigger_time is not None:
cdata.trigger_time = trigger_time
# Create the the event list
event_list = BinnedSpectrumSeries(cdata.spectrum_set,
first_channel=0,
mission='Fermi',
instrument=cdata.det_name,
verbose=verbose)
# we need to see if this is an RSP2
if isinstance(rsp_file,str) or isinstance(rsp_file,unicode):
test = re.match('^.*\.rsp2$', rsp_file)
# some GBM RSPs that are not marked RSP2 are in fact RSP2s
# we need to check
if test is None:
with fits.open(rsp_file) as f:
# there should only be a header, ebounds and one spec rsp extension
if len(f) > 3:
# make test a dummy value to trigger the nest loop
test = -1
custom_warnings.warn(
'The RSP file is marked as a single response but in fact has multiple matrices. We will treat it as an RSP2')
if test is not None:
rsp = InstrumentResponseSet.from_rsp2_file(rsp2_file=rsp_file,
counts_getter=event_list.counts_over_interval,
exposure_getter=event_list.exposure_over_interval,
reference_time=cdata.trigger_time)
else:
rsp = OGIPResponse(rsp_file)
else:
assert isinstance(rsp_file, InstrumentResponse), 'The provided response is not a 3ML InstrumentResponse'
rsp = rsp_file
# pass to the super class
return cls(name,
event_list,
response=rsp,
poly_order=poly_order,
unbinned=False,
verbose=verbose,
restore_poly_fit=restore_background,
container_type=BinnedSpectrumWithDispersion
)
def set_polynomial_fit_interval(self, *time_intervals, **options):
"""Set the time interval to fit the background.
Multiple intervals can be input as separate arguments
Specified as 'tmin-tmax'. Intervals are in seconds. Example:
set_polynomial_fit_interval("-10.0-0.0","10.-15.")
:param time_intervals: intervals to fit on
:param options:
"""
# Find out if we want to binned or unbinned.
# TODO: add the option to config file
if 'unbinned' in options:
unbinned = options.pop('unbinned')
assert type(
unbinned) == bool, 'unbinned option must be True or False'
else:
# assuming unbinned
# could use config file here
# unbinned = threeML_config['ogip']['use-unbinned-poly-fitting']
unbinned = True
# we create some time intervals
poly_intervals = TimeIntervalSet.from_strings(*time_intervals)
# adjust the selections to the data
new_intervals = []
self._poly_selected_counts = []
self._poly_exposure = 0.
for i, time_interval in enumerate(poly_intervals):
t1 = time_interval.start_time
t2 = time_interval.stop_time
if (self._stop_time <= t1) or (t2 <= self._start_time):
custom_warnings.warn(
"The time interval %f-%f is out side of the arrival times and will be dropped"
% (t1, t2))
else:
if t1 < self._start_time:
custom_warnings.warn(
"The time interval %f-%f started before the first arrival time (%f), so we are changing the intervals to %f-%f"
% (t1, t2, self._start_time, self._start_time, t2))
t1 = self._start_time # + 1
if t2 > self._stop_time:
custom_warnings.warn(
"The time interval %f-%f ended after the last arrival time (%f), so we are changing the intervals to %f-%f"
% (t1, t2, self._stop_time, t1, self._stop_time))
t2 = self._stop_time # - 1.
new_intervals.append('%f-%f' % (t1, t2))
self._poly_selected_counts.append(
self.count_per_channel_over_interval(t1, t2))
self._poly_exposure += self.exposure_over_interval(t1, t2)
# make new intervals after checks
poly_intervals = TimeIntervalSet.from_strings(*new_intervals)
self._poly_selected_counts = np.sum(self._poly_selected_counts, axis=0)
# set the poly intervals as an attribute
self._poly_intervals = poly_intervals
# Fit the events with the given intervals
if unbinned:
self._unbinned = True # keep track!
self._unbinned_fit_polynomials()
else:
self._unbinned = False
self._fit_polynomials()
# we have a fit now
self._poly_fit_exists = True
if self._verbose:
print("%s %d-order polynomial fit with the %s method" %
(self._fit_method_info['bin type'],
self._optimal_polynomial_grade,
self._fit_method_info['fit method']))
print('\n')
# recalculate the selected counts
if self._time_selection_exists:
self.set_active_time_intervals(
*self._time_intervals.to_string().split(','))
def fit(self, quiet=False, compute_covariance=True, n_samples=5000):
"""
Perform a fit of the current likelihood model on the datasets
:param quiet: If True, print the results (default), otherwise do not print anything
:param compute_covariance:If True (default), compute and display the errors and the correlation matrix.
:return: a dictionary with the results on the parameters, and the values of the likelihood at the minimum
for each dataset and the total one.
"""
# Update the list of free parameters, to be safe against changes the user might do between
# the creation of this class and the calling of this method
self._update_free_parameters()
# Empty the call recorder
self._record_calls = {}
self._ncalls = 0
# Check if we have free parameters, otherwise simply return the value of the log like
if len(self._free_parameters) == 0:
custom_warnings.warn(
"There is no free parameter in the current model",
RuntimeWarning)
# Create the minimizer anyway because it will be needed by the following code
self._minimizer = self._get_minimizer(self.minus_log_like_profile,
self._free_parameters)
# Store the "minimum", which is just the current value
self._current_minimum = float(self.minus_log_like_profile())
else:
# Instance the minimizer
# If we have a global minimizer, use that first (with no covariance)
if isinstance(self._minimizer_type,
minimization.GlobalMinimization):
# Do global minimization first
global_minimizer = self._get_minimizer(
self.minus_log_like_profile, self._free_parameters)
xs, global_log_likelihood_minimum = global_minimizer.minimize(
compute_covar=False)
# Gather global results
paths = []
values = []
errors = []
units = []
for par in list(self._free_parameters.values()):
paths.append(par.path)
values.append(par.value)
errors.append(0)
units.append(par.unit)
global_results = ResultsTable(paths, values, errors, errors,
units)
if not quiet:
print(
"\n\nResults after global minimizer (before secondary optimization):"
)
global_results.display()
print("\nTotal log-likelihood minimum: %.3f\n" %
global_log_likelihood_minimum)
# Now set up secondary minimizer
self._minimizer = self._minimizer_type.get_second_minimization_instance(
self.minus_log_like_profile, self._free_parameters)
else:
# Only local minimization to be performed
self._minimizer = self._get_minimizer(
self.minus_log_like_profile, self._free_parameters)
# Perform the fit, but first flush stdout (so if we have verbose=True the messages there will follow
# what is already in the buffer)
sys.stdout.flush()
xs, log_likelihood_minimum = self._minimizer.minimize(
compute_covar=compute_covariance)
if log_likelihood_minimum == minimization.FIT_FAILED:
raise FitFailed("The fit failed to converge.")
# Store the current minimum for the -log likelihood
self._current_minimum = float(log_likelihood_minimum)
# First restore best fit (to make sure we compute the likelihood at the right point in the following)
self._minimizer.restore_best_fit()
# Now collect the values for the likelihood for the various datasets
# Fill the dictionary with the values of the -log likelihood (dataset by dataset)
minus_log_likelihood_values = collections.OrderedDict()
# Keep track of the total for a double check
total = 0
# sum up the total number of data points
total_number_of_data_points = 0
for dataset in list(self._data_list.values()):
ml = dataset.inner_fit() * (-1)
minus_log_likelihood_values[dataset.name] = ml
total += ml
total_number_of_data_points += dataset.get_number_of_data_points()
assert (
total == self._current_minimum
), "Current minimum stored after fit and current do not correspond!"
# compute additional statistics measures
statistical_measures = collections.OrderedDict()
# for MLE we can only compute the AIC and BIC as they
# are point estimates
statistical_measures["AIC"] = aic(-total, len(self._free_parameters),
total_number_of_data_points)
statistical_measures["BIC"] = bic(-total, len(self._free_parameters),
total_number_of_data_points)
# Now instance an analysis results class
self._analysis_results = MLEResults(
self.likelihood_model,
self._minimizer.covariance_matrix,
minus_log_likelihood_values,
statistical_measures=statistical_measures,
n_samples=n_samples,
)
# Show the results
if not quiet:
self._analysis_results.display()
return (
self._analysis_results.get_data_frame(),
self._analysis_results.get_statistic_frame(),
)
def minus_log_like_profile(self, *trial_values):
"""
Return the minus log likelihood for a given set of trial values
:param trial_values: the trial values. Must be in the same number as the free parameters in the model
:return: minus log likelihood
"""
# Keep track of the number of calls
self._ncalls += 1
# Transform the trial values in a numpy array
trial_values = np.array(trial_values)
# Check that there are no nans within the trial values
# This is the fastest way to check for any nan
# (try other methods if you don't believe me)
if not np.isfinite(np.dot(trial_values, trial_values.T)):
# There are nans, something weird is going on. Return FIT_FAILED so the engine
# stays away from this (or fail)
return minimization.FIT_FAILED
# Assign the new values to the parameters
for i, parameter in enumerate(self._free_parameters.values()):
# Use the internal representation (see the Parameter class)
parameter._set_internal_value(trial_values[i])
# Now profile out nuisance parameters and compute the new value
# for the likelihood
summed_log_likelihood = 0
for dataset in self._data_list.values():
try:
this_log_like = dataset.inner_fit()
except ModelAssertionViolation:
# This is a zone of the parameter space which is not allowed. Return
# a big number for the likelihood so that the fit engine will avoid it
custom_warnings.warn("Fitting engine in forbidden space: %s" % (trial_values,),
custom_exceptions.ForbiddenRegionOfParameterSpace)
return minimization.FIT_FAILED
except:
# Do not intercept other errors
raise
summed_log_likelihood += this_log_like
# Check that the global like is not NaN
# I use this weird check because it is not guaranteed that the plugins return np.nan,
# especially if they are written in something other than python
if "%s" % summed_log_likelihood == 'nan':
custom_warnings.warn("These parameters returned a logLike = Nan: %s" % (trial_values,),
NotANumberInLikelihood)
return minimization.FIT_FAILED
if self.verbose:
sys.stderr.write("trial values: %s -> logL = %.3f\n" % (",".join(map(lambda x:"%.5g" % x, trial_values)),
summed_log_likelihood))
# Record this call
if self._record:
self._record_calls[tuple(trial_values)] = summed_log_likelihood
# Return the minus log likelihood
return summed_log_likelihood * (-1)
def minus_log_like_profile(self, *trial_values):
"""
Return the minus log likelihood for a given set of trial values
:param trial_values: the trial values. Must be in the same number as the free parameters in the model
:return: minus log likelihood
"""
# Keep track of the number of calls
self._ncalls += 1
# Transform the trial values in a numpy array
trial_values = np.array(trial_values)
# Check that there are no nans within the trial values
# This is the fastest way to check for any nan
# (try other methods if you don't believe me)
if not np.isfinite(np.dot(trial_values, trial_values.T)):
# There are nans, something weird is going on. Return FIT_FAILED so the engine
# stays away from this (or fail)
return minimization.FIT_FAILED
# Assign the new values to the parameters
for i, parameter in enumerate(self._free_parameters.values()):
# Use the internal representation (see the Parameter class)
parameter._set_internal_value(trial_values[i])
# Now profile out nuisance parameters and compute the new value
# for the likelihood
summed_log_likelihood = 0
for dataset in list(self._data_list.values()):
try:
this_log_like = dataset.inner_fit()
except ModelAssertionViolation:
# This is a zone of the parameter space which is not allowed. Return
# a big number for the likelihood so that the fit engine will avoid it
custom_warnings.warn(
"Fitting engine in forbidden space: %s" % (trial_values, ),
custom_exceptions.ForbiddenRegionOfParameterSpace,
)
return minimization.FIT_FAILED
except:
# Do not intercept other errors
raise
summed_log_likelihood += this_log_like
# Check that the global like is not NaN
# I use this weird check because it is not guaranteed that the plugins return np.nan,
# especially if they are written in something other than python
if "%s" % summed_log_likelihood == "nan":
custom_warnings.warn(
"These parameters returned a logLike = Nan: %s" %
(trial_values, ),
NotANumberInLikelihood,
)
return minimization.FIT_FAILED
if self.verbose:
sys.stderr.write(
"trial values: %s -> logL = %.3f\n" %
(",".join(["%.5g" % x
for x in trial_values]), summed_log_likelihood))
# Record this call
if self._record:
self._record_calls[tuple(trial_values)] = summed_log_likelihood
# Return the minus log likelihood
return summed_log_likelihood * (-1)
def display_spectrum_model_counts(analysis, data=(), **kwargs):
"""
Display the fitted model count spectrum of one or more Spectrum plugins
NOTE: all parameters passed as keyword arguments that are not in the list below, will be passed as keyword arguments
to the plt.subplots() constructor. So for example, you can specify the size of the figure using figsize = (20,10)
:param args: one or more instances of Spectrum plugin
:param min_rate: (optional) rebin to keep this minimum rate in each channel (if possible). If one number is
provided, the same minimum rate is used for each dataset, otherwise a list can be provided with the minimum rate
for each dataset
:param data_cmap: (str) (optional) the color map used to extract automatically the colors for the data
:param model_cmap: (str) (optional) the color map used to extract automatically the colors for the models
:param data_colors: (optional) a tuple or list with the color for each dataset
:param model_colors: (optional) a tuple or list with the color for each folded model
:param data_color: (optional) color for all datasets
:param model_color: (optional) color for all folded models
:param show_legend: (optional) if True (default), shows a legend
:param step: (optional) if True (default), show the folded model as steps, if False, the folded model is plotted
:param model_subplot: (optional) axe(s) to plot to for overplotting
with linear interpolation between each bin
:return: figure instance
"""
# If the user supplies a subset of the data, we will use that
if not data:
data_keys = analysis.data_list.keys()
else:
data_keys = data
# Now we want to make sure that we only grab OGIP plugins
new_data_keys = []
for key in data_keys:
# Make sure it is a valid key
if key in analysis.data_list.keys():
if isinstance(analysis.data_list[key], threeML.plugins.SpectrumLike.SpectrumLike):
new_data_keys.append(key)
else:
custom_warnings.warn("Dataset %s is not of the SpectrumLike kind. Cannot be plotted by "
"display_spectrum_model_counts" % key)
if not new_data_keys:
RuntimeError(
'There were no valid SpectrumLike data requested for plotting. Please use the detector names in the data list')
data_keys = new_data_keys
# default settings
# Default is to show the model with steps
step = True
data_cmap = threeML_config['ogip']['data plot cmap'] # plt.cm.rainbow
model_cmap = threeML_config['ogip']['model plot cmap'] # plt.cm.nipy_spectral_r
# Legend is on by default
show_legend = True
show_residuals = True
# Default colors
data_colors = cmap_intervals(len(data_keys), data_cmap)
model_colors = cmap_intervals(len(data_keys), model_cmap)
# Now override defaults according to the optional keywords, if present
if 'show_data' in kwargs:
show_data = bool(kwargs.pop('show_data'))
else:
show_data = True
if 'show_legend' in kwargs:
show_legend = bool(kwargs.pop('show_legend'))
if 'show_residuals' in kwargs:
show_residuals= bool(kwargs.pop('show_residuals'))
if 'step' in kwargs:
step = bool(kwargs.pop('step'))
if 'min_rate' in kwargs:
min_rate = kwargs.pop('min_rate')
# If min_rate is a floating point, use the same for all datasets, otherwise use the provided ones
try:
min_rate = float(min_rate)
min_rates = [min_rate] * len(data_keys)
except TypeError:
min_rates = list(min_rate)
assert len(min_rates) >= len(
data_keys), "If you provide different minimum rates for each data set, you need" \
"to provide an iterable of the same length of the number of datasets"
else:
# This is the default (no rebinning)
min_rates = [NO_REBIN] * len(data_keys)
if 'data_cmap' in kwargs:
data_cmap = plt.get_cmap(kwargs.pop('data_cmap'))
data_colors = cmap_intervals(len(data_keys), data_cmap)
if 'model_cmap' in kwargs:
model_cmap = kwargs.pop('model_cmap')
model_colors = cmap_intervals(len(data_keys), model_cmap)
if 'data_colors' in kwargs:
data_colors = kwargs.pop('data_colors')
assert len(data_colors) >= len(data_keys), "You need to provide at least a number of data colors equal to the " \
"number of datasets"
elif 'data_color' in kwargs:
data_colors = [kwargs.pop('data_color')] * len(data_keys)
if 'model_colors' in kwargs:
model_colors = kwargs.pop('model_colors')
assert len(model_colors) >= len(
data_keys), "You need to provide at least a number of model colors equal to the " \
"number of datasets"
ratio_residuals=False
if 'ratio_residuals' in kwargs:
ratio_residuals = bool(kwargs['ratio_residuals'])
elif 'model_color' in kwargs:
model_colors = [kwargs.pop('model_color')] * len(data_keys)
if 'model_labels' in kwargs:
model_labels = kwargs.pop('model_labels')
assert len(model_labels) == len(data_keys), 'you must have the same number of model labels as data sets'
else:
model_labels = ['%s Model' % analysis.data_list[key]._name for key in data_keys]
#fig, (ax, ax1) = plt.subplots(2, 1, sharex=True, gridspec_kw={'height_ratios': [2, 1]}, **kwargs)
residual_plot = ResidualPlot(show_residuals=show_residuals, **kwargs)
if show_residuals:
axes = [residual_plot.data_axis,residual_plot.residual_axis]
else:
axes = residual_plot.data_axis
# go thru the detectors
for key, data_color, model_color, min_rate, model_label in zip(data_keys, data_colors, model_colors, min_rates, model_labels):
# NOTE: we use the original (unmasked) vectors because we need to rebin ourselves the data later on
data = analysis.data_list[key] # type: threeML.plugins.SpectrumLike.SpectrumLike
data.display_model(data_color=data_color,
model_color=model_color,
min_rate=min_rate,
step=step,
show_residuals=show_residuals,
show_data=show_data,
show_legend=show_legend,
ratio_residuals=ratio_residuals,
model_label=model_label,
model_subplot=axes
)
return residual_plot.figure
def display_photometry_model_magnitudes(analysis, data=(), **kwargs):
"""
Display the fitted model count spectrum of one or more Spectrum plugins
NOTE: all parameters passed as keyword arguments that are not in the list below, will be passed as keyword arguments
to the plt.subplots() constructor. So for example, you can specify the size of the figure using figsize = (20,10)
:param args: one or more instances of Spectrum plugin
:param min_rate: (optional) rebin to keep this minimum rate in each channel (if possible). If one number is
provided, the same minimum rate is used for each dataset, otherwise a list can be provided with the minimum rate
for each dataset
:param data_cmap: (str) (optional) the color map used to extract automatically the colors for the data
:param model_cmap: (str) (optional) the color map used to extract automatically the colors for the models
:param data_colors: (optional) a tuple or list with the color for each dataset
:param model_colors: (optional) a tuple or list with the color for each folded model
:param show_legend: (optional) if True (default), shows a legend
:param step: (optional) if True (default), show the folded model as steps, if False, the folded model is plotted
with linear interpolation between each bin
:return: figure instance
"""
# If the user supplies a subset of the data, we will use that
if not data:
data_keys = list(analysis.data_list.keys())
else:
data_keys = data
# Now we want to make sure that we only grab OGIP plugins
new_data_keys = []
for key in data_keys:
# Make sure it is a valid key
if key in list(analysis.data_list.keys()):
if isinstance(analysis.data_list[key], photolike.PhotometryLike):
new_data_keys.append(key)
else:
custom_warnings.warn(
"Dataset %s is not of the Photometery kind. Cannot be plotted by "
"display_photometry_model_magnitudes" % key)
if not new_data_keys:
RuntimeError(
"There were no valid Photometry data requested for plotting. Please use the detector names in the data list"
)
data_keys = new_data_keys
# Default is to show the model with steps
step = threeML_config.plugins.photo.fit_plot.step
data_cmap = threeML_config.plugins.photo.fit_plot.data_cmap.value # plt.cm.rainbow
model_cmap = threeML_config.plugins.photo.fit_plot.model_cmap.value
# Legend is on by default
show_legend = True
# Default colors
data_colors = cmap_intervals(len(data_keys), data_cmap)
model_colors = cmap_intervals(len(data_keys), model_cmap)
# Now override defaults according to the optional keywords, if present
if "show_legend" in kwargs:
show_legend = bool(kwargs.pop("show_legend"))
if "step" in kwargs:
step = bool(kwargs.pop("step"))
if "data_cmap" in kwargs:
data_cmap = plt.get_cmap(kwargs.pop("data_cmap"))
data_colors = cmap_intervals(len(data_keys), data_cmap)
if "model_cmap" in kwargs:
model_cmap = kwargs.pop("model_cmap")
model_colors = cmap_intervals(len(data_keys), model_cmap)
if "data_colors" in kwargs:
data_colors = kwargs.pop("data_colors")
if len(data_colors) < len(data_keys):
log.error(
"You need to provide at least a number of data colors equal to the "
"number of datasets")
raise ValueError()
if "model_colors" in kwargs:
model_colors = kwargs.pop("model_colors")
if len(model_colors) < len(data_keys):
log.error(
"You need to provide at least a number of model colors equal to the "
"number of datasets")
raise ValueError()
residual_plot = ResidualPlot(**kwargs)
# go thru the detectors
for key, data_color, model_color in zip(data_keys, data_colors,
model_colors):
data = analysis.data_list[key] # type: photolike
# get the expected counts
avg_wave_length = (data._filter_set.effective_wavelength.value
) # type: np.ndarray
# need to sort because filters are not always in order
sort_idx = avg_wave_length.argsort()
expected_model_magnitudes = data._get_total_expectation()[sort_idx]
magnitudes = data.magnitudes[sort_idx]
mag_errors = data.magnitude_errors[sort_idx]
avg_wave_length = avg_wave_length[sort_idx]
residuals = old_div((expected_model_magnitudes - magnitudes),
mag_errors)
widths = data._filter_set.wavelength_bounds.widths[sort_idx]
residual_plot.add_data(
x=avg_wave_length,
y=magnitudes,
xerr=widths,
yerr=mag_errors,
residuals=residuals,
label=data._name,
color=data_color,
)
residual_plot.add_model(
avg_wave_length,
expected_model_magnitudes,
label="%s Model" % data._name,
color=model_color,
)
return residual_plot.finalize(
xlabel="Wavelength\n(%s)" % data._filter_set.waveunits,
ylabel="Magnitudes",
xscale="linear",
yscale="linear",
invert_y=True,
)
def __init__(self):
# Read first the default configuration file
default_configuration_path = get_path_of_data_file(_config_file_name)
assert os.path.exists(default_configuration_path), \
"Default configuration %s does not exist. Re-install 3ML" % default_configuration_path
with open(default_configuration_path) as f:
try:
configuration = yaml.load(f, Loader=yaml.SafeLoader)
except:
raise ConfigurationFileCorrupt("Default configuration file %s cannot be parsed!" %
(default_configuration_path))
# This needs to be here for the _check_configuration to work
self._default_configuration_raw = configuration
# Test the default configuration
try:
self._check_configuration(configuration, default_configuration_path)
except:
raise
else:
self._default_path = default_configuration_path
# Check if the user has a user-supplied config file under .threeML
user_config_path = os.path.join(get_path_of_user_dir(), _config_file_name)
if os.path.exists(user_config_path):
with open(user_config_path) as f:
configuration = yaml.load(f, Loader=yaml.SafeLoader)
# Test if the local/configuration is ok
try:
self._configuration = self._check_configuration(configuration, user_config_path)
except ConfigurationFileCorrupt:
# Probably an old configuration file
custom_warnings.warn("The user configuration file at %s does not appear to be valid. We will "
"substitute it with the default configuration. You will find a copy of the "
"old configuration at %s so you can transfer any customization you might "
"have from there to the new configuration file. We will use the default "
"configuration for this session."
%(user_config_path, "%s.bak" % user_config_path))
# Move the config file to a backup file
shutil.copy2(user_config_path, "%s.bak" % user_config_path)
# Remove old file
os.remove(user_config_path)
# Copy the default configuration
shutil.copy2(self._default_path, user_config_path)
self._configuration = self._check_configuration(self._default_configuration_raw, self._default_path)
self._filename = self._default_path
else:
self._filename = user_config_path
print("Configuration read from %s" % (user_config_path))
else:
custom_warnings.warn("Using default configuration from %s. "
"You might want to copy it to %s to customize it and avoid this warning."
% (self._default_path, user_config_path))
self._configuration = self._check_configuration(self._default_configuration_raw, self._default_path)
self._filename = self._default_path
def fit(self, quiet=False, compute_covariance=True, n_samples=5000):
"""
Perform a fit of the current likelihood model on the datasets
:param quiet: If True, print the results (default), otherwise do not print anything
:param compute_covariance:If True (default), compute and display the errors and the correlation matrix.
:return: a dictionary with the results on the parameters, and the values of the likelihood at the minimum
for each dataset and the total one.
"""
# Update the list of free parameters, to be safe against changes the user might do between
# the creation of this class and the calling of this method
self._update_free_parameters()
# Empty the call recorder
self._record_calls = {}
self._ncalls = 0
# Check if we have free parameters, otherwise simply return the value of the log like
if len(self._free_parameters) == 0:
custom_warnings.warn("There is no free parameter in the current model", RuntimeWarning)
# Create the minimizer anyway because it will be needed by the following code
self._minimizer = self._get_minimizer(self.minus_log_like_profile,
self._free_parameters)
# Store the "minimum", which is just the current value
self._current_minimum = float(self.minus_log_like_profile())
else:
# Instance the minimizer
# If we have a global minimizer, use that first (with no covariance)
if isinstance(self._minimizer_type, minimization.GlobalMinimization):
# Do global minimization first
global_minimizer = self._get_minimizer(self.minus_log_like_profile, self._free_parameters)
xs, global_log_likelihood_minimum = global_minimizer.minimize(compute_covar=False)
# Gather global results
paths = []
values = []
errors = []
units = []
for par in self._free_parameters.values():
paths.append(par.path)
values.append(par.value)
errors.append(0)
units.append(par.unit)
global_results = ResultsTable(paths, values, errors, errors, units)
if not quiet:
print("\n\nResults after global minimizer (before secondary optimization):")
global_results.display()
print("\nTotal log-likelihood minimum: %.3f\n" % global_log_likelihood_minimum)
# Now set up secondary minimizer
self._minimizer = self._minimizer_type.get_second_minimization_instance(self.minus_log_like_profile,
self._free_parameters)
else:
# Only local minimization to be performed
self._minimizer = self._get_minimizer(self.minus_log_like_profile,
self._free_parameters)
# Perform the fit, but first flush stdout (so if we have verbose=True the messages there will follow
# what is already in the buffer)
sys.stdout.flush()
xs, log_likelihood_minimum = self._minimizer.minimize(compute_covar=compute_covariance)
if log_likelihood_minimum == minimization.FIT_FAILED:
raise FitFailed("The fit failed to converge.")
# Store the current minimum for the -log likelihood
self._current_minimum = float(log_likelihood_minimum)
# First restore best fit (to make sure we compute the likelihood at the right point in the following)
self._minimizer.restore_best_fit()
# Now collect the values for the likelihood for the various datasets
# Fill the dictionary with the values of the -log likelihood (dataset by dataset)
minus_log_likelihood_values = collections.OrderedDict()
# Keep track of the total for a double check
total = 0
# sum up the total number of data points
total_number_of_data_points = 0
for dataset in self._data_list.values():
ml = dataset.inner_fit() * (-1)
minus_log_likelihood_values[dataset.name] = ml
total += ml
total_number_of_data_points += dataset.get_number_of_data_points()
assert total == self._current_minimum, "Current minimum stored after fit and current do not correspond!"
# compute additional statistics measures
statistical_measures = collections.OrderedDict()
# for MLE we can only compute the AIC and BIC as they
# are point estimates
statistical_measures['AIC'] = aic(-total,len(self._free_parameters),total_number_of_data_points)
statistical_measures['BIC'] = bic(-total,len(self._free_parameters),total_number_of_data_points)
# Now instance an analysis results class
self._analysis_results = MLEResults(self.likelihood_model, self._minimizer.covariance_matrix,
minus_log_likelihood_values,statistical_measures=statistical_measures, n_samples=n_samples)
# Show the results
if not quiet:
self._analysis_results.display()
return self._analysis_results.get_data_frame(), self._analysis_results.get_statistic_frame()
def __init__(self, matrix_list, exposure_getter, counts_getter, reference_time=0.0):
"""
:param matrix_list:
:type matrix_list : list[InstrumentResponse]
:param exposure_getter : a function returning the exposure between t1 and t2
:param counts_getter : a function returning the number of counts between t1 and t2
:param reference_time : a reference time to be added to the specifications of the intervals used in the
weight_by_* methods. Use this if you want to express the time intervals in time units from the reference_time,
instead of "absolute" time. For GRBs, this is the trigger time. NOTE: if you use a reference time, the
counts_getter and the exposure_getter must accept times relative to the reference time.
"""
# Store list of matrices
self._matrix_list = list(matrix_list) # type: list[InstrumentResponse]
# Create the corresponding list of coverage intervals
self._coverage_intervals = TimeIntervalSet(map(lambda x: x.coverage_interval,
self._matrix_list))
# Make sure that all matrices have coverage interval set
if None in self._coverage_intervals:
raise NoCoverageIntervals("You need to specify the coverage interval for all matrices in the matrix_list")
# Remove from the list matrices that cover intervals of zero duration (yes, the GBM publishes those too,
# one example is in data/ogip_test_gbm_b0.rsp2)
to_be_removed = []
for i, interval in enumerate(self._coverage_intervals):
if interval.duration == 0:
# Remove it
with custom_warnings.catch_warnings():
custom_warnings.simplefilter("always", RuntimeWarning)
custom_warnings.warn("Removing matrix %s (numbering starts at zero) because it has a coverage of "
"zero seconds" % i, RuntimeWarning)
to_be_removed.append(i)
# Actually remove them
if len(to_be_removed) > 0:
[self._matrix_list.pop(index) for index in to_be_removed]
[self._coverage_intervals.pop(index) for index in to_be_removed]
# Order the matrices by time
idx = self._coverage_intervals.argsort()
# It is possible that there is only one coverage interval (these are published by GBM e.g. GRB090819607)
# so we need to be sure that the array is a least 1D
self._coverage_intervals = TimeIntervalSet(np.atleast_1d(itemgetter(*idx)(self._coverage_intervals)))
self._matrix_list = np.atleast_1d(itemgetter(*idx)(self._matrix_list))
# Now make sure that the coverage intervals are contiguous (i.e., there are no gaps)
if not self._coverage_intervals.is_contiguous():
raise NonContiguousCoverageIntervals("The provided responses have coverage intervals which are not contiguous!")
# Apply the reference time shift, if any
self._coverage_intervals -= reference_time
# Store callable
self._exposure_getter = exposure_getter # type: callable
self._counts_getter = counts_getter # type: callable
# Store reference time
self._reference_time = float(reference_time)
def from_root_file(cls, response_file_name):
"""
Build response from a ROOT file. Do not use directly, use the hawc_response_factory function instead.
:param response_file_name:
:return: a HAWCResponse instance
"""
from ..root_handler import open_ROOT_file, get_list_of_keys, tree_to_ndarray
# Make sure file is readable
response_file_name = sanitize_filename(response_file_name)
# Check that they exists and can be read
if not file_existing_and_readable(
response_file_name): # pragma: no cover
raise IOError("Response %s does not exist or is not readable" %
response_file_name)
# Read response
with open_ROOT_file(response_file_name) as root_file:
# Get the name of the trees
object_names = get_list_of_keys(root_file)
# Make sure we have all the things we need
assert 'LogLogSpectrum' in object_names
assert 'DecBins' in object_names
assert 'AnalysisBins' in object_names
# Read spectrum used during the simulation
log_log_spectrum = root_file.Get("LogLogSpectrum")
# Get the analysis bins definition
dec_bins_ = tree_to_ndarray(root_file.Get("DecBins"))
dec_bins_lower_edge = dec_bins_['lowerEdge'] # type: np.ndarray
dec_bins_upper_edge = dec_bins_['upperEdge'] # type: np.ndarray
dec_bins_center = dec_bins_['simdec'] # type: np.ndarray
dec_bins = zip(dec_bins_lower_edge, dec_bins_center,
dec_bins_upper_edge)
# Read in the ids of the response bins ("analysis bins" in LiFF jargon)
try:
response_bins_ids = tree_to_ndarray(
root_file.Get("AnalysisBins"), "name") # type: np.ndarray
except ValueError:
try:
response_bins_ids = tree_to_ndarray(
root_file.Get("AnalysisBins"),
"id") # type: np.ndarray
except ValueError:
# Some old response files (or energy responses) have no "name" branch
custom_warnings.warn(
"Response %s has no AnalysisBins 'id' or 'name' branch. "
"Will try with default names" % response_file_name)
response_bins_ids = None
response_bins_ids = response_bins_ids.astype(str)
# Now we create a dictionary of ResponseBin instances for each dec bin_name
response_bins = collections.OrderedDict()
for dec_id in range(len(dec_bins)):
this_response_bins = collections.OrderedDict()
min_dec, dec_center, max_dec = dec_bins[dec_id]
# If we couldn't get the reponse_bins_ids above, let's use the default names
if response_bins_ids is None:
# Default are just integers. let's read how many nHit bins are from the first dec bin
dec_id_label = "dec_%02i" % dec_id
n_energy_bins = root_file.Get(dec_id_label).GetNkeys()
response_bins_ids = range(n_energy_bins)
for response_bin_id in response_bins_ids:
this_response_bin = ResponseBin.from_ttree(
root_file, dec_id, response_bin_id, log_log_spectrum,
min_dec, dec_center, max_dec)
this_response_bins[response_bin_id] = this_response_bin
response_bins[dec_bins[dec_id][1]] = this_response_bins
# Now the file is closed. Let's explicitly remove f so we are sure it is freed
del root_file
# Instance the class and return it
instance = cls(response_file_name, dec_bins, response_bins)
return instance
def to_spectrumlike(self, from_bins=False, start=None, stop=None, interval_name='_interval', extract_measured_background=False):
"""
Create plugin(s) from either the current active selection or the time bins.
If creating from an event list, the
bins are from create_time_bins. If using a pre-time binned time series, the bins are those
native to the data. Start and stop times can be used to control which bins are used.
:param from_bins: choose to create plugins from the time bins
:param start: optional start time of the bins
:param stop: optional stop time of the bins
:param extract_measured_background: Use the selected background rather than a polynomial fit to the background
:param interval_name: the name of the interval
:return: SpectrumLike plugin(s)
"""
# we can use either the modeled or the measured background. In theory, all the information
# in the background spectrum should propagate to the likelihood
if extract_measured_background:
this_background_spectrum = self._measured_background_spectrum
else:
this_background_spectrum = self._background_spectrum
# this is for a single interval
if not from_bins:
assert self._observed_spectrum is not None, 'Must have selected an active time interval'
assert isinstance(self._observed_spectrum, BinnedSpectrum), 'You are attempting to create a SpectrumLike plugin from the wrong data type'
if this_background_spectrum is None:
custom_warnings.warn('No background selection has been made. This plugin will contain no background!')
if self._response is None:
return SpectrumLike(name=self._name,
observation=self._observed_spectrum,
background=this_background_spectrum,
verbose=self._verbose,
tstart=self._tstart,
tstop=self._tstop)
else:
return DispersionSpectrumLike(name=self._name,
observation=self._observed_spectrum,
background=this_background_spectrum,
verbose=self._verbose,
tstart = self._tstart,
tstop = self._tstop
)
else:
# this is for a set of intervals.
assert self._time_series.bins is not None, 'This time series does not have any bins!'
# save the original interval if there is one
old_interval = copy.copy(self._active_interval)
old_verbose = copy.copy(self._verbose)
# we will keep it quiet to keep from being annoying
self._verbose = False
list_of_speclikes = []
# get the bins from the time series
# for event lists, these are from created bins
# for binned spectra sets, these are the native bines
these_bins = self._time_series.bins # type: TimeIntervalSet
if start is not None:
assert stop is not None, 'must specify a start AND a stop time'
if stop is not None:
assert stop is not None, 'must specify a start AND a stop time'
these_bins = these_bins.containing_interval(start, stop, inner=False)
# loop through the intervals and create spec likes
with progress_bar(len(these_bins), title='Creating plugins') as p:
for i, interval in enumerate(these_bins):
self.set_active_time_interval(interval.to_string())
assert isinstance(self._observed_spectrum, BinnedSpectrum), 'You are attempting to create a SpectrumLike plugin from the wrong data type'
if extract_measured_background:
this_background_spectrum = self._measured_background_spectrum
else:
this_background_spectrum = self._background_spectrum
if this_background_spectrum is None:
custom_warnings.warn(
'No bakckground selection has been made. This plugin will contain no background!')
try:
if self._response is None:
sl = SpectrumLike(name="%s%s%d" % (self._name, interval_name, i),
observation=self._observed_spectrum,
background=this_background_spectrum,
verbose=self._verbose,
tstart=self._tstart,
tstop=self._tstop)
else:
sl = DispersionSpectrumLike(name="%s%s%d" % (self._name, interval_name, i),
observation=self._observed_spectrum,
background=this_background_spectrum,
verbose=self._verbose,
tstart=self._tstart,
tstop=self._tstop)
list_of_speclikes.append(sl)
except(NegativeBackground):
custom_warnings.warn('Something is wrong with interval %s. skipping.' % interval)
p.increase()
# restore the old interval
if old_interval is not None:
self.set_active_time_interval(*old_interval)
else:
self._active_interval = None
self._verbose = old_verbose
return list_of_speclikes
def get_contours(self,
param_1,
param_1_minimum,
param_1_maximum,
param_1_n_steps,
param_2=None,
param_2_minimum=None,
param_2_maximum=None,
param_2_n_steps=None,
progress=True,
**options):
"""
Generate confidence contours for the given parameters by stepping for the given number of steps between
the given boundaries. Call it specifying only source_1, param_1, param_1_minimum and param_1_maximum to
generate the profile of the likelihood for parameter 1. Specify all parameters to obtain instead a 2d
contour of param_1 vs param_2.
NOTE: if using parallel computation, param_1_n_steps must be an integer multiple of the number of running
engines. If that is not the case, the code will reduce the number of steps to match that requirement, and
issue a warning
:param param_1: fully qualified name of the first parameter or parameter instance
:param param_1_minimum: lower bound for the range for the first parameter
:param param_1_maximum: upper bound for the range for the first parameter
:param param_1_n_steps: number of steps for the first parameter
:param param_2: fully qualified name of the second parameter or parameter instance
:param param_2_minimum: lower bound for the range for the second parameter
:param param_2_maximum: upper bound for the range for the second parameter
:param param_2_n_steps: number of steps for the second parameter
:param progress: (True or False) whether to display progress or not
:param log: by default the steps are taken linearly. With this optional parameter you can provide a tuple of
booleans which specify whether the steps are to be taken logarithmically. For example,
'log=(True,False)' specify that the steps for the first parameter are to be taken logarithmically,
while they are linear for the second parameter. If you are generating the profile for only one
parameter, you can specify 'log=(True,)' or 'log=(False,)' (optional)
:return: a tuple containing an array corresponding to the steps for the first parameter, an array corresponding
to the steps for the second parameter (or None if stepping only in one direction), a matrix of size
param_1_steps x param_2_steps containing the value of the function at the corresponding points in the
grid. If param_2_steps is None (only one parameter), then this reduces to an array of
size param_1_steps.
"""
if hasattr(param_1, "value"):
# Substitute with the name
param_1 = param_1.path
if hasattr(param_2, "value"):
param_2 = param_2.path
# Check that the parameters exist
assert param_1 in self._likelihood_model.free_parameters, (
"Parameter %s is not a free parameters of the "
"current model" % param_1)
if param_2 is not None:
assert param_2 in self._likelihood_model.free_parameters, (
"Parameter %s is not a free parameters of the "
"current model" % param_2)
# Check that we have a valid fit
assert (
self._current_minimum is not None
), "You have to run the .fit method before calling get_contours."
# Then restore the best fit
self._minimizer.restore_best_fit()
# Check minimal assumptions about the procedure
assert not (param_1
== param_2), "You have to specify two different parameters"
assert (param_1_minimum <
param_1_maximum), "Minimum larger than maximum for parameter 1"
min1, max1 = self.likelihood_model[param_1].bounds
if min1 is not None:
assert param_1_minimum >= min1, (
"Requested low range for parameter %s (%s) "
"is below parameter minimum (%s)" %
(param_1, param_1_minimum, min1))
if max1 is not None:
assert param_1_maximum <= max1, (
"Requested hi range for parameter %s (%s) "
"is above parameter maximum (%s)" %
(param_1, param_1_maximum, max1))
if param_2 is not None:
min2, max2 = self.likelihood_model[param_2].bounds
if min2 is not None:
assert param_2_minimum >= min2, (
"Requested low range for parameter %s (%s) "
"is below parameter minimum (%s)" %
(param_2, param_2_minimum, min2))
if max2 is not None:
assert param_2_maximum <= max2, (
"Requested hi range for parameter %s (%s) "
"is above parameter maximum (%s)" %
(param_2, param_2_maximum, max2))
# Check whether we are parallelizing or not
if not threeML_config["parallel"]["use-parallel"]:
a, b, cc = self.minimizer.contours(
param_1, param_1_minimum, param_1_maximum, param_1_n_steps,
param_2, param_2_minimum, param_2_maximum, param_2_n_steps,
progress, **options)
# Collapse the second dimension of the results if we are doing a 1d contour
if param_2 is None:
cc = cc[:, 0]
else:
# With parallel computation
# In order to distribute fairly the computation, the strategy is to parallelize the computation
# by assigning to the engines one "line" of the grid at the time
# Connect to the engines
client = ParallelClient(**options)
# Get the number of engines
n_engines = client.get_number_of_engines()
# Check whether the number of threads is larger than the number of steps in the first direction
if n_engines > param_1_n_steps:
n_engines = int(param_1_n_steps)
custom_warnings.warn(
"The number of engines is larger than the number of steps. Using only %s engines."
% n_engines,
ReducingNumberOfThreads,
)
# Check if the number of steps is divisible by the number
# of threads, otherwise issue a warning and make it so
if float(param_1_n_steps) % n_engines != 0:
# Set the number of steps to an integer multiple of the engines
# (note that // is the floor division, also called integer division)
param_1_n_steps = (param_1_n_steps // n_engines) * n_engines
custom_warnings.warn(
"Number of steps is not a multiple of the number of threads. Reducing steps to %s"
% param_1_n_steps,
ReducingNumberOfSteps,
)
# Compute the number of splits, i.e., how many lines in the grid for each engine.
# (note that this is guaranteed to be an integer number after the previous checks)
p1_split_steps = param_1_n_steps // n_engines
# Prepare arrays for results
if param_2 is None:
# One array
pcc = np.zeros(param_1_n_steps)
pa = np.linspace(param_1_minimum, param_1_maximum,
param_1_n_steps)
pb = None
else:
pcc = np.zeros((param_1_n_steps, param_2_n_steps))
# Prepare the two axes of the parameter space
pa = np.linspace(param_1_minimum, param_1_maximum,
param_1_n_steps)
pb = np.linspace(param_2_minimum, param_2_maximum,
param_2_n_steps)
# Define the parallel worker which will go through the computation
# NOTE: I only divide
# on the first parameter axis so that the different
# threads are more or less well mixed for points close and
# far from the best fit
def worker(start_index):
# Re-create the minimizer
backup_freeParameters = [
x.value for x in list(
self._likelihood_model.free_parameters.values())
]
this_minimizer = self._get_minimizer(
self.minus_log_like_profile, self._free_parameters)
this_p1min = pa[start_index * p1_split_steps]
this_p1max = pa[(start_index + 1) * p1_split_steps - 1]
# print("From %s to %s" % (this_p1min, this_p1max))
aa, bb, ccc = this_minimizer.contours(param_1,
this_p1min,
this_p1max,
p1_split_steps,
param_2,
param_2_minimum,
param_2_maximum,
param_2_n_steps,
progress=True,
**options)
# Restore best fit values
for val, par in zip(
backup_freeParameters,
list(self._likelihood_model.free_parameters.values()),
):
par.value = val
return ccc
# Now re-assemble the vector of results taking the different parts from the engines
all_results = client.execute_with_progress_bar(
worker, list(range(n_engines)), chunk_size=1)
for i, these_results in enumerate(all_results):
if param_2 is None:
pcc[i * p1_split_steps:(i + 1) *
p1_split_steps] = these_results[:, 0]
else:
pcc[i * p1_split_steps:(i + 1) *
p1_split_steps, :] = these_results
# Give the results the names that the following code expect. These are kept separate for debugging
# purposes
cc = pcc
a = pa
b = pb
# Here we have done the computation, in parallel computation or not. Let's make the plot
# with the contour
if param_2 is not None:
# 2d contour
fig = self._plot_contours("%s" % (param_1), a, "%s" % (param_2, ),
b, cc)
else:
# 1d contour (i.e., a profile)
fig = self._plot_profile("%s" % (param_1), a, cc)
# Check if we found a better minimum. This shouldn't happen, but in case of very difficult fit
# it might.
if self._current_minimum - cc.min() > 0.1:
if param_2 is not None:
idx = cc.argmin()
aidx, bidx = np.unravel_index(idx, cc.shape)
print(
"\nFound a better minimum: %s with %s = %s and %s = %s. Run again your fit starting from here."
% (cc.min(), param_1, a[aidx], param_2, b[bidx]))
else:
idx = cc.argmin()
print(
"Found a better minimum: %s with %s = %s. Run again your fit starting from here."
% (cc.min(), param_1, a[idx]))
return a, b, cc, fig
def __init__(self, matrix, ebounds, monte_carlo_energies, coverage_interval=None):
"""
Generic response class that accepts a full matrix, detector energy boundaries (ebounds) and monte carlo energies,
and an optional coverage interval which indicates which time interval the matrix applies to.
If there are n_channels in the detector, and the monte carlo energies are n_mc_energies, then the matrix must
be n_channels x n_mc_energies.
Therefore, an OGIP style RSP from a file is not required if the matrix,
ebounds, and mc channels exist.
:param matrix: an n_channels x n_mc_energies response matrix representing both effective area and
energy dispersion effects
:param ebounds: the energy boundaries of the detector channels (size n_channels + 1)
:param monte_carlo_energies: the energy boundaries of the monte carlo channels (size n_mc_energies + 1)
:param coverage_interval: the time interval to which the matrix refers to (if available, None by default)
:type coverage_interval: TimeInterval
"""
# we simply store all the variables to the class
self._matrix = np.array(matrix, float)
# Make sure there are no nans or inf
assert np.all(np.isfinite(self._matrix)), "Infinity or nan in matrix"
self._ebounds = np.array(ebounds, float)
self._mc_energies = np.array(monte_carlo_energies)
self._integral_function = None
# Store the time interval
if coverage_interval is not None:
assert isinstance(coverage_interval, TimeInterval), "The coverage interval must be a TimeInterval instance"
self._coverage_interval = coverage_interval
else:
self._coverage_interval = None
# Safety checks
assert self._matrix.shape == (self._ebounds.shape[0]-1, self._mc_energies.shape[0]-1), \
"Matrix has the wrong shape. Got %s, expecting %s" % (self._matrix.shape,
[self._ebounds.shape[0]-1,
self._mc_energies.shape[0]-1])
if self._mc_energies.max() < self._ebounds.max():
custom_warnings.warn("Maximum MC energy (%s) is smaller "
"than maximum EBOUNDS energy (%s)" % (self._mc_energies.max(), self.ebounds.max()),
RuntimeWarning)
if self._mc_energies.min() > self._ebounds.min():
custom_warnings.warn("Minimum MC energy (%s) is larger than "
"minimum EBOUNDS energy (%s)" % (self._mc_energies.min(), self._ebounds.min()),
RuntimeWarning)
def _compute_covariance_matrix(self, best_fit_values):
"""
This function compute the approximate covariance matrix as the inverse of the Hessian matrix,
which is the matrix of second derivatives of the likelihood function with respect to
the parameters.
The sqrt of the diagonal of the result is an accurate estimate of the errors only if the
log.likelihood is parabolic in the neighborhood of the minimum.
Derivatives are computed numerically.
:return: the covariance matrix
"""
minima = map(lambda parameter:parameter._get_internal_min_value(), self.parameters.values())
maxima = map(lambda parameter: parameter._get_internal_max_value(), self.parameters.values())
# Check whether some of the minima or of the maxima are None. If they are, set them
# to a value 1000 times smaller or larger respectively than the best fit.
# An error of 3 orders of magnitude is not interesting in general, and this is the only
# way to be able to compute a derivative numerically
for i in range(len(minima)):
if minima[i] is None:
minima[i] = best_fit_values[i] / 1000.0
if maxima[i] is None:
maxima[i] = best_fit_values[i] * 1000.0
# Transform them in np.array
minima = np.array(minima)
maxima = np.array(maxima)
try:
hessian_matrix = get_hessian(self.function, best_fit_values, minima, maxima)
except ParameterOnBoundary:
custom_warnings.warn("One or more of the parameters are at their boundaries. Cannot compute covariance and"
" errors", CannotComputeCovariance)
n_dim = len(best_fit_values)
return np.zeros((n_dim,n_dim)) * np.nan
# Invert it to get the covariance matrix
try:
covariance_matrix = np.linalg.inv(hessian_matrix)
except:
custom_warnings.warn("Cannot invert Hessian matrix, looks like the matrix is singular")
n_dim = len(best_fit_values)
return np.zeros((n_dim, n_dim)) * np.nan
# Now check that the covariance matrix is semi-positive definite (it must be unless
# there have been numerical problems, which can happen when some parameter is unconstrained)
# The fastest way is to try and compute the Cholesky decomposition, which
# works only if the matrix is positive definite
try:
_ = np.linalg.cholesky(covariance_matrix)
except:
custom_warnings.warn("Covariance matrix is NOT semi-positive definite. Cannot estimate errors. This can "
"happen for many reasons, the most common being one or more unconstrained parameters",
CannotComputeCovariance)
return covariance_matrix
def to_polarlike(self, from_bins=False, start=None, stop=None, interval_name='_interval', extract_measured_background=False):
assert has_polarpy, 'you must have the polarpy module installed'
assert issubclass(self._container_type, BinnedModulationCurve), 'You are attempting to create a POLARLike plugin from the wrong data type'
if extract_measured_background:
this_background_spectrum = self._measured_background_spectrum
else:
this_background_spectrum = self._background_spectrum
if isinstance(self._response,str):
self._response = PolarResponse(self._response)
if not from_bins:
assert self._observed_spectrum is not None, 'Must have selected an active time interval'
if this_background_spectrum is None:
custom_warnings.warn('No background selection has been made. This plugin will contain no background!')
return PolarLike(name=self._name,
observation=self._observed_spectrum,
background=this_background_spectrum,
response=self._response,
verbose=self._verbose,
# tstart=self._tstart,
# tstop=self._tstop
)
else:
# this is for a set of intervals.
assert self._time_series.bins is not None, 'This time series does not have any bins!'
# save the original interval if there is one
old_interval = copy.copy(self._active_interval)
old_verbose = copy.copy(self._verbose)
# we will keep it quiet to keep from being annoying
self._verbose = False
list_of_polarlikes = []
# now we make one response to save time
# get the bins from the time series
# for event lists, these are from created bins
# for binned spectra sets, these are the native bines
these_bins = self._time_series.bins # type: TimeIntervalSet
if start is not None:
assert stop is not None, 'must specify a start AND a stop time'
if stop is not None:
assert stop is not None, 'must specify a start AND a stop time'
these_bins = these_bins.containing_interval(start, stop, inner=False)
# loop through the intervals and create spec likes
with progress_bar(len(these_bins), title='Creating plugins') as p:
for i, interval in enumerate(these_bins):
self.set_active_time_interval(interval.to_string())
if extract_measured_background:
this_background_spectrum = self._measured_background_spectrum
else:
this_background_spectrum = self._background_spectrum
if this_background_spectrum is None:
custom_warnings.warn(
'No bakckground selection has been made. This plugin will contain no background!')
try:
pl = PolarLike(name="%s%s%d" % (self._name, interval_name, i),
observation=self._observed_spectrum,
background=this_background_spectrum,
response=self._response,
verbose=self._verbose,
# tstart=self._tstart,
# tstop=self._tstop
)
list_of_polarlikes.append(pl)
except(NegativeBackground):
custom_warnings.warn('Something is wrong with interval %s. skipping.' % interval)
p.increase()
# restore the old interval
if old_interval is not None:
self.set_active_time_interval(*old_interval)
else:
self._active_interval = None
self._verbose = old_verbose
return list_of_polarlikes
def __init__(self,
start_time,
stop_time,
n_channels,
native_quality=None,
first_channel=1,
ra=None,
dec=None,
mission=None,
instrument=None,
verbose=True,
edges=None):
"""
The EventList is a container for event data that is tagged in time and in PHA/energy. It handles event selection,
temporal polynomial fitting, temporal binning, and exposure calculations (in subclasses). Once events are selected
and/or polynomials are fit, the selections can be extracted via a PHAContainer which is can be read by an OGIPLike
instance and translated into a PHA instance.
:param n_channels: Number of detector channels
:param start_time: start time of the event list
:param stop_time: stop time of the event list
:param first_channel: where detchans begin indexing
:param rsp_file: the response file corresponding to these events
:param arrival_times: list of event arrival times
:param energies: list of event energies or pha channels
:param native_quality: native pha quality flags
:param edges: The histogram boundaries if not specified by a response
:param mission:
:param instrument:
:param verbose:
:param ra:
:param dec:
"""
self._verbose = verbose
self._n_channels = n_channels
self._first_channel = first_channel
self._native_quality = native_quality
# we haven't made selections yet
self._time_intervals = None
self._poly_intervals = None
self._counts = None
self._exposure = None
self._poly_counts = None
self._poly_count_err = None
self._poly_selected_counts = None
self._poly_exposure = None
# ebounds for objects w/o a response
self._edges = edges
if native_quality is not None:
assert len(
native_quality
) == n_channels, "the native quality has length %d but you specified there were %d channels" % (
len(native_quality), n_channels)
self._start_time = start_time
self._stop_time = stop_time
# name the instrument if there is not one
if instrument is None:
custom_warnings.warn(
'No instrument name is given. Setting to UNKNOWN')
self._instrument = "UNKNOWN"
else:
self._instrument = instrument
if mission is None:
custom_warnings.warn(
'No mission name is given. Setting to UNKNOWN')
self._mission = "UNKNOWN"
else:
self._mission = mission
self._user_poly_order = -1
self._time_selection_exists = False
self._poly_fit_exists = False
self._fit_method_info = {"bin type": None, 'fit method': None}
def __init__(self, name, time_series, response=None,
poly_order=-1, unbinned=True, verbose=True, restore_poly_fit=None, container_type=BinnedSpectrumWithDispersion):
"""
Class for handling generic time series data including binned and event list
series. Depending on the data, this class builds either a SpectrumLike or
DisperisonSpectrumLike plugin
For specific instruments, use the TimeSeries.from() classmethods
:param name: name for the plugin
:param time_series: a TimeSeries instance
:param response: options InstrumentResponse instance
:param poly_order: the polynomial order to use for background fitting
:param unbinned: if the background should be fit unbinned
:param verbose: the verbosity switch
:param restore_poly_fit: file from which to read a prefitted background
"""
assert isinstance(time_series, TimeSeries), "must be a TimeSeries instance"
assert issubclass(container_type,Histogram), 'must be a subclass of Histogram'
self._name = name
self._container_type = container_type
self._time_series = time_series # type: TimeSeries
# make sure we have a proper response
if response is not None:
assert isinstance(response, InstrumentResponse) or isinstance(response,
InstrumentResponseSet) or isinstance(response, str), 'Response must be an instance of InstrumentResponse'
# deal with RSP weighting if need be
if isinstance(response, InstrumentResponseSet):
# we have a weighted response
self._rsp_is_weighted = True
self._weighted_rsp = response
# just get a dummy response for the moment
# it will be corrected when we set the interval
self._response = InstrumentResponse.create_dummy_response(response.ebounds,
response.monte_carlo_energies)
else:
self._rsp_is_weighted = False
self._weighted_rsp = None
self._response = response
self._verbose = verbose
self._active_interval = None
self._observed_spectrum = None
self._background_spectrum = None
self._measured_background_spectrum = None
self._time_series.poly_order = poly_order
self._default_unbinned = unbinned
# try and restore the poly fit if requested
if restore_poly_fit is not None:
if file_existing_and_readable(restore_poly_fit):
self._time_series.restore_fit(restore_poly_fit)
if verbose:
print('Successfully restored fit from %s'%restore_poly_fit)
else:
custom_warnings.warn(
"Could not find saved background %s." % restore_poly_fit)