def display(self, key_formatter=long_path_formatter):
def row_formatter(row):
value = row["value"]
lower_bound = value + row["negative_error"]
upper_bound = value + row["positive_error"]
pretty_string = uncertainty_formatter(value, lower_bound,
upper_bound)
return pretty_string
# Make another data frame with the keys
new_frame = self._data_frame.copy(deep=True) # type: pd.DataFrame
# Add new column which will become the new index
new_frame["parameter"] = [
key_formatter(x) for x in new_frame.index.values
]
# Set it as the index
new_frame.set_index("parameter", drop=True, inplace=True)
# compute the display
new_frame["result"] = new_frame.apply(row_formatter, axis=1)
# Display
display(new_frame[["result", "unit"]])
def display(self, key_formatter=long_path_formatter):
def row_formatter(row):
value = row['value']
lower_bound = value + row['negative_error']
upper_bound = value + row['positive_error']
pretty_string = uncertainty_formatter(value, lower_bound,
upper_bound)
return pretty_string
# Make another data frame with the keys
new_frame = self._data_frame.copy(deep=True) # type: pd.DataFrame
# Add new column which will become the new index
new_frame['parameter'] = map(lambda x: key_formatter(x),
new_frame.index.values)
# Set it as the index
new_frame.set_index('parameter', drop=True, inplace=True)
# compute the display
new_frame['result'] = new_frame.apply(row_formatter, axis=1)
# Display
display(new_frame[['result', 'unit']])
def display(self, key_formatter = long_path_formatter):
def row_formatter(row):
value = row['value']
lower_bound = value + row['negative_error']
upper_bound = value + row['positive_error']
pretty_string = uncertainty_formatter(value, lower_bound, upper_bound)
return pretty_string
# Make another data frame with the keys
new_frame = self._data_frame.copy(deep=True) # type: pd.DataFrame
# Add new column which will become the new index
new_frame['parameter'] = map(lambda x: key_formatter(x), new_frame.index.values)
# Set it as the index
new_frame.set_index('parameter', drop=True, inplace=True)
# compute the display
new_frame['result'] = new_frame.apply(row_formatter, axis=1)
# Display
display(new_frame[['result','unit']])
def _setup(self):
# Setup the widget, which is a bar between 0 and 100
self._bar = FloatProgress(min=0, max=100)
# Set explicitly the bar to 0
self._bar.value = 0
# Setup also an HTML label (which will contain the progress, the elapsed time and the foreseen
# completion time)
self._title_cell = HTML()
if self._title is not None:
self._title_cell.value = "%s : " % self._title
self._label = HTML()
self._vbox = VBox(children=[self._title_cell, self._label, self._bar])
# Display everything
display(self._vbox)
self._animate(0)
def _setup(self):
# Setup the widget, which is a bar between 0 and 100
self._bar = FloatProgress(min=0, max=100)
# Set explicitly the bar to 0
self._bar.value = 0
# Setup also an HTML label (which will contain the progress, the elapsed time and the foreseen
# completion time)
self._title_cell = HTML()
if self._title is not None:
self._title_cell.value = "%s : " % self._title
self._label = HTML()
self._vbox = VBox(children=[self._title_cell, self._label, self._bar])
# Display everything
display(self._vbox)
self._animate(0)
def display(self):
"""
Display the time intervals
:return: None
"""
display(self._create_pandas())
def display(self):
"""
Display the time intervals
:return: None
"""
display(self._create_pandas())
def display(self):
df = pd.DataFrame()
df['Bin'] = list(self._analysis_bins.keys())
df['Nside'] = [
self._analysis_bins[bin_id].nside for bin_id in self._analysis_bins
]
df['Scheme'] = [
self._analysis_bins[bin_id].scheme
for bin_id in self._analysis_bins
]
# Compute observed counts, background counts, how many pixels we have in the ROI and
# the sky area they cover
n_bins = len(self._analysis_bins)
obs_counts = np.zeros(n_bins)
bkg_counts = np.zeros_like(obs_counts)
n_pixels = np.zeros(n_bins, dtype=int)
sky_area = np.zeros_like(obs_counts)
size = 0
for i, bin_id in enumerate(self._analysis_bins):
analysis_bin = self._analysis_bins[bin_id]
sparse_obs = analysis_bin.observation_map.as_partial()
sparse_bkg = analysis_bin.background_map.as_partial()
size += sparse_obs.nbytes
size += sparse_bkg.nbytes
obs_counts[i] = sparse_obs.sum()
bkg_counts[i] = sparse_bkg.sum()
n_pixels[i] = sparse_obs.shape[0]
sky_area[i] = n_pixels[i] * analysis_bin.observation_map.pixel_area
df['Obs counts'] = obs_counts
df['Bkg counts'] = bkg_counts
df['obs/bkg'] = old_div(obs_counts, bkg_counts)
df['Pixels in ROI'] = n_pixels
df['Area (deg^2)'] = sky_area
display(df)
first_bin_id = list(self._analysis_bins.keys())[0]
log.info("This Map Tree contains %.3f transits in the first bin" \
% self._analysis_bins[first_bin_id].n_transits)
log.info("Total data size: %.2f Mb" %
(size * u.byte).to(u.megabyte).value)
def print_fit_results(self):
"""
Display the results of the last minimization.
:return: (none)
"""
# Restore the best fit values, in case something has changed
self._restore_best_fit()
# I do not use the print_param facility in iminuit because
# it does not work well with console output, since it fails
# to autoprobe that it is actually run in a console and uses
# the HTML backend instead
# Create a list of strings to print
data = []
# Also store the maximum length to decide the length for the line
name_length = 0
for k, v in self.parameters.iteritems():
minuit_name = self._parameter_name_to_minuit_name(k)
# Format the value and the error with sensible significant
# numbers
x = uncertainties.ufloat(v.value, self.minuit.errors[minuit_name])
# Add some space around the +/- sign
rep = x.__str__().replace("+/-", " +/- ")
data.append([k, rep, v.unit])
if len(k) > name_length:
name_length = len(k)
table = Table(rows=data, names=["Name", "Value", "Unit"], dtype=("S%i" % name_length, str, str))
display(table)
print("\nNOTE: errors on parameters are approximate. Use get_errors().\n")
def display(self):
df = pd.DataFrame()
df['Bin'] = self._data_bins_labels
df['Nside'] = map(lambda x: x.nside, self._data_analysis_bins)
df['Scheme'] = map(lambda x: x.scheme, self._data_analysis_bins)
# Compute observed counts, background counts, how many pixels we have in the ROI and
# the sky area they cover
n_bins = len(self._data_bins_labels)
obs_counts = np.zeros(n_bins)
bkg_counts = np.zeros_like(obs_counts)
n_pixels = np.zeros(n_bins, dtype=int)
sky_area = np.zeros_like(obs_counts)
size = 0
for i, analysis_bin in enumerate(self._data_analysis_bins):
sparse_obs = analysis_bin.observation_map.as_partial()
sparse_bkg = analysis_bin.background_map.as_partial()
size += sparse_obs.nbytes
size += sparse_bkg.nbytes
obs_counts[i] = sparse_obs.sum()
bkg_counts[i] = sparse_bkg.sum()
n_pixels[i] = sparse_obs.shape[0]
sky_area[i] = n_pixels[i] * analysis_bin.observation_map.pixel_area
df['Obs counts'] = obs_counts
df['Bkg counts'] = bkg_counts
df['obs/bkg'] = obs_counts / bkg_counts
df['Pixels in ROI'] = n_pixels
df['Area (deg^2)'] = sky_area
display(df)
print("This Map Tree contains %.3f transits in the first bin" %
self._data_analysis_bins[0].n_transits)
print("Total data size: %.2f Mb" %
(size * u.byte).to(u.megabyte).value)
def display(self,
display_correlation=False,
error_type="equal tail",
cl=0.68):
best_fit_table = self._get_results_table(error_type, cl)
print("Maximum a posteriori probability (MAP) point:\n")
best_fit_table.display()
if display_correlation:
corr_matrix = NumericMatrix(self.get_correlation_matrix())
for col in corr_matrix.colnames:
corr_matrix[col].format = '2.2f'
print("\nCorrelation matrix:\n")
display(corr_matrix)
print("\nValues of -log(posterior) at the minimum:\n")
display(self.get_statistic_frame())
print("\nValues of statistical measures:\n")
display(self.get_statistic_measure_frame())
def display(self, display_correlation=True, cl=0.68):
best_fit_table = self._get_results_table(
error_type="covariance", cl=cl, covariance=self.covariance_matrix)
print("Best fit values:\n")
best_fit_table.display()
if display_correlation:
corr_matrix = NumericMatrix(self.get_correlation_matrix())
for col in corr_matrix.colnames:
corr_matrix[col].format = '2.2f'
print("\nCorrelation matrix:\n")
display(corr_matrix)
print("\nValues of -log(likelihood) at the minimum:\n")
display(self.get_statistic_frame())
print("\nValues of statistical measures:\n")
display(self.get_statistic_measure_frame())
def get_point_source_flux(self,
ene_min,
ene_max,
sources=(),
confidence_level=0.68,
flux_unit='erg/(s cm2)',
use_components=False,
components_to_use=(),
sum_sources=False):
"""
:param ene_min: minimum energy (an astropy quantity, like 1.0 * u.keV. You can also use a frequency, like
1 * u.Hz)
:param ene_max: maximum energy (an astropy quantity, like 10 * u.keV. You can also use a frequency, like
10 * u.Hz)
:param sources: Use this to specify the name of the source or a tuple/list of source names to be plotted.
If you don't use this, all sources will be plotted.
:param confidence_level: the confidence level for the error (default: 0.68)
:param flux_unit: (optional) astropy flux unit in string form (can be
:param use_components: plot the components of each source (default: False)
:param components_to_use: (optional) list of string names of the components to plot: including 'total'
:param sum_sources: (optional) if True, also the sum of all sources will be plotted
:return:
"""
# Convert the ene_min and ene_max in pure numbers in keV
_ene_min = ene_min.to("keV").value
_ene_max = ene_max.to("keV").value
_params = {
'confidence_level': confidence_level,
'equal_tailed': True, # FIXME: what happens if this is False?
'best_fit': 'median',
'energy_unit': 'keV',
'flux_unit': flux_unit,
'use_components': use_components,
'components_to_use': components_to_use,
'sources_to_use': sources,
'sum_sources': sum_sources,
}
mle_results, bayes_results = _calculate_point_source_flux(
_ene_min, _ene_max, self, **_params)
# The output contains one source per row
def _format_error(row):
rep = uncertainty_formatter(row['flux'].value,
row['low bound'].value,
row['hi bound'].value)
# Represent the unit as a string
unit_rep = str(row['flux'].unit)
return pd.Series({'flux': "%s %s" % (rep, unit_rep)})
if mle_results is not None:
# Format the errors and display the resulting data frame
display(mle_results.apply(_format_error, axis=1))
# Return the dataframe
return mle_results
elif bayes_results is not None:
# Format the errors and display the resulting data frame
display(bayes_results.apply(_format_error, axis=1))
# Return the dataframe
return bayes_results
def display(self):
return display(self)
def get_errors(self):
"""
Compute asymmetric errors using MINOS (slow, but accurate) and print them.
NOTE: this should be called immediately after the minimize() method
:return: a dictionary containing the asymmetric errors for each parameter.
"""
if not self._migrad_has_converged():
raise CannotComputeErrors("MIGRAD results not valid, cannot compute errors. Did you run the fit first ?")
try:
self.minuit.minos()
except:
raise MINOSFailed(
"MINOS has failed. This usually means that the fit is very difficult, for example "
"because of high correlation between parameters. Check the correlation matrix printed"
"in the fit step, and check contour plots with getContours(). If you are using a "
"user-defined model, you can also try to "
"reformulate your model with less correlated parameters."
)
# Make a ordered dict for the results
errors = collections.OrderedDict()
for k, par in self.parameters.iteritems():
minuit_name = self._parameter_name_to_minuit_name(k)
errors[k] = (self.minuit.merrors[(minuit_name, -1)], self.minuit.merrors[(minuit_name, 1)])
# Set the parameters back to the best fit value
self._restore_best_fit()
# Print a table with the errors
data = []
name_length = 0
for k, v in self.parameters.iteritems():
# Format the value and the error with sensible significant
# numbers
# Process the negative error
x = uncertainties.ufloat(v.value, abs(errors[k][0]))
# Split the uncertainty in number, negative error, and exponent (if any)
num, uncm, exponent = re.match(
"\(?(\-?[0-9]+\.?[0-9]+) ([0-9]+\.[0-9]+)\)?(e[\+|\-][0-9]+)?", x.__str__().replace("+/-", " ")
).groups()
# Process the positive error
x = uncertainties.ufloat(v.value, abs(errors[k][1]))
# Split the uncertainty in number, positive error, and exponent (if any)
_, uncp, _ = re.match(
"\(?(\-?[0-9]+\.?[0-9]+) ([0-9]+\.[0-9]+)\)?(e[\+|\-][0-9]+)?", x.__str__().replace("+/-", " ")
).groups()
if exponent is None:
# Number without exponent
pretty_string = "%s -%s +%s" % (num, uncm, uncp)
else:
# Number with exponent
pretty_string = "(%s -%s +%s)%s" % (num, uncm, uncp, exponent)
data.append([k, pretty_string, v.unit])
if len(k) > name_length:
name_length = len(k)
# Create and display the table
table = Table(rows=data, names=["Name", "Value", "Unit"], dtype=("S%i" % name_length, str, str))
display(table)
return errors
def print_correlation_matrix(self):
"""
Display the current correlation matrix
:return: (none)
"""
# Print a custom covariance matrix because iminuit does
# not guess correctly the frontend when 3ML is used
# from terminal
cov = self.minuit.covariance
if cov is None:
raise CannotComputeCovariance(
"Cannot compute covariance numerically. This usually means that there are "
+ " unconstrained parameters. Fix those or reduce their allowed range, or "
+ "use a simpler model."
)
# Get list of parameters
keys = self.parameters.keys()
# Convert them to the format for iminuit
minuit_names = map(lambda k: self._parameter_name_to_minuit_name(k), keys)
# Accumulate rows and compute the maximum length of the names
data = []
length_of_names = 0
for key1, name1 in zip(keys, minuit_names):
if len(name1) > length_of_names:
length_of_names = len(name1)
this_row = []
for key2, name2 in zip(keys, minuit_names):
# Compute correlation between parameter key1 and key2
corr = cov[(name1, name2)] / (math.sqrt(cov[(name1, name1)]) * math.sqrt(cov[(name2, name2)]))
this_row.append(corr)
data.append(this_row)
# Prepare the dtypes for the matrix
dtypes = map(lambda x: float, minuit_names)
# Column names are the parameter names
cols = keys
# Finally generate the matrix with the names
table = NumericMatrix(rows=data, names=cols, dtype=dtypes)
# Customize the format to avoid too many digits
for col in table.colnames:
table[col].format = "2.2f"
display(table)
def get_other_instrument_information(self):
"""
Return the detectors used for spectral analysis as well as their background
intervals. Peak flux and fluence intervals are also returned as well as best fit models
:return: observing information dataframe indexed by source
"""
assert (
self._last_query_results is not None
), "You have to run a query before getting observing information"
sources = {}
for name, row in self._last_query_results.T.items():
obs_instrument = {}
# loop over the observation indices
for obs in range(1, 5):
if obs == 1:
obs_base = "other_obs"
else:
obs_base = "other_obs%d" % obs
obs_ref = "%s_ref" % obs_base
obs = row[obs_base]
# this means that nothing in this column saw the grb
if obs == "":
observed = False
else:
observed = True
if observed:
# if we saw it then lets get the GCN
gcn_number = _gcn_match.search(row[obs_ref]).group(1)
# gcn_number = filter(lambda x: x != '', row[obs_ref].split('.'))[1]
# make the URL
gcn = "https://gcn.gsfc.nasa.gov/gcn3/%s.gcn3" % gcn_number
# just for Fermi GBM, lets get the trigger number
# TODO: add more instruments
if obs == "Fermi-GBM":
info = {
"GCN":
gcn,
"trigger number":
self._get_fermiGBM_trigger_number_from_gcn(
str(gcn)),
}
else:
info = {"GCN": gcn, "trigger number": None}
obs_instrument[obs] = info
sources[name] = obs_instrument
# build the data frame
sources = pd.concat(list(map(pd.DataFrame, list(sources.values()))),
keys=list(sources.keys()))
display(sources)
return sources
def display(self):
return display(self)
def get_credible_intervals(self, probability=95):
"""
Print and returns the (equal-tail) credible intervals for all free parameters in the model
:param probability: the probability for this credible interval (default: 95, corresponding to 95%)
:return: a dictionary with the lower bound and upper bound of the credible intervals, as well as the median
"""
# Gather the credible intervals (percentiles of the posterior)
credible_intervals = collections.OrderedDict()
for i, (parameter_name, parameter) in enumerate(self._free_parameters.iteritems()):
# Get the percentiles from the posterior samples
lower_bound,median,upper_bound = numpy.percentile(self.samples[parameter_name],
(100-probability,50,probability))
# Save them in the dictionary
credible_intervals[parameter_name] = {'lower bound': lower_bound,
'median': median,
'upper bound': upper_bound}
# Print a table with the errors
data = []
name_length = 0
for i, (parameter_name, parameter) in enumerate(self._free_parameters.iteritems()):
# Format the value and the error with sensible significant
# numbers
lower_bound, median, upper_bound = [credible_intervals[parameter_name][key] for key in ('lower bound',
'median',
'upper bound')
]
# Process the negative "error"
x = uncertainties.ufloat(median, abs(lower_bound - median))
# Split the uncertainty in number, negative error, and exponent (if any)
number, unc_lower_bound, exponent = re.match('\(?(\-?[0-9]+\.?[0-9]+) ([0-9]+\.[0-9]+)\)?(e[\+|\-][0-9]+)?',
x.__str__().replace("+/-", " ")).groups()
# Process the positive "error"
x = uncertainties.ufloat(median, abs(upper_bound - median))
# Split the uncertainty in number, positive error, and exponent (if any)
_, unc_upper_bound, _ = re.match('\(?(\-?[0-9]+\.?[0-9]+) ([0-9]+\.[0-9]+)\)?(e[\+|\-][0-9]+)?',
x.__str__().replace("+/-", " ")).groups()
if exponent is None:
# Number without exponent
pretty_string = "%s -%s +%s" % (number, unc_lower_bound, unc_upper_bound)
else:
# Number with exponent
pretty_string = "(%s -%s +%s)%s" % (number, unc_lower_bound, unc_upper_bound, exponent)
unit = self._free_parameters[parameter_name].unit
data.append([parameter_name, pretty_string, unit])
if len(parameter_name) > name_length:
name_length = len(parameter_name)
# Create and display the table
table = Table(rows=data,
names=["Name", "Value", "Unit"],
dtype=('S%i' % name_length, str, 'S15'))
display(table)
return credible_intervals
def fit(self):
"""
Perform a fit of the current likelihood model on the datasets
:param pre_fit: (True or False) If True, perform a pre-fit with only normalizations free (experimental)
:return: a dictionary with the results on the parameters, and the values of the likelihood at the minimum
for each dataset and the total one.
"""
# Update the list of free parameters, to be safe against changes the user might do between
# the creation of this class and the calling of this method
self._update_free_parameters()
# Now check and fix if needed all the deltas of the parameters
# to 10% of their value (otherwise the fit will be super-slow)
for k, v in self._free_parameters.iteritems():
if abs(v.delta) < abs(v.value) * 0.1:
v.delta = abs(v.value) * 0.1
# Instance the minimizer
self._minimizer = self.Minimizer(self.minus_log_like_profile, self._free_parameters)
# Perform the fit
xs, log_likelihood_minimum = self._minimizer.minimize()
# Store the current minimum for the -log likelihood
self._current_minimum = float(log_likelihood_minimum)
# Print results
print("Best fit values:\n")
self._minimizer.print_fit_results()
print("Nuisance parameters:\n")
nuisance_parameters = collections.OrderedDict()
for dataset in self._data_list.values():
for pName in dataset.get_nuisance_parameters():
nuisance_parameters[(dataset.get_name(), pName)] = dataset.nuisanceParameters[pName]
nuisance_parameters_table = self._get_table_of_parameters(nuisance_parameters)
display(nuisance_parameters_table)
print("\nCorrelation matrix:\n")
self._minimizer.print_correlation_matrix()
print("\nMinimum of -logLikelihood is: %s\n" % log_likelihood_minimum)
print("Contributions to the -logLikelihood at the minimum:\n")
# Fill the dictionary with the values of the -log likelihood (total, and dataset by dataset)
minus_log_likelihood_values = collections.OrderedDict()
minus_log_likelihood_values["total"] = log_likelihood_minimum
data = []
name_length = 0
for dataset in self._data_list.values():
ml = dataset.inner_fit() * (-1)
minus_log_likelihood_values[dataset.get_name()] = ml
name_length = max(name_length, len(dataset.get_name()) + 1)
data.append([dataset.get_name(), ml])
log_like_values_table = Table(rows=data, names=["Detector", "-LogL"], dtype=("S%i" % name_length, float))
log_like_values_table["-LogL"].format = "2.2f"
display(log_like_values_table)
# Prepare the dictionary with the results
results = collections.OrderedDict()
results["parameters"] = xs
results["minusLogLike"] = minus_log_likelihood_values
return results