def display(self, key_formatter=long_path_formatter): def row_formatter(row): value = row["value"] lower_bound = value + row["negative_error"] upper_bound = value + row["positive_error"] pretty_string = uncertainty_formatter(value, lower_bound, upper_bound) return pretty_string # Make another data frame with the keys new_frame = self._data_frame.copy(deep=True) # type: pd.DataFrame # Add new column which will become the new index new_frame["parameter"] = [ key_formatter(x) for x in new_frame.index.values ] # Set it as the index new_frame.set_index("parameter", drop=True, inplace=True) # compute the display new_frame["result"] = new_frame.apply(row_formatter, axis=1) # Display display(new_frame[["result", "unit"]])
def display(self, key_formatter=long_path_formatter): def row_formatter(row): value = row['value'] lower_bound = value + row['negative_error'] upper_bound = value + row['positive_error'] pretty_string = uncertainty_formatter(value, lower_bound, upper_bound) return pretty_string # Make another data frame with the keys new_frame = self._data_frame.copy(deep=True) # type: pd.DataFrame # Add new column which will become the new index new_frame['parameter'] = map(lambda x: key_formatter(x), new_frame.index.values) # Set it as the index new_frame.set_index('parameter', drop=True, inplace=True) # compute the display new_frame['result'] = new_frame.apply(row_formatter, axis=1) # Display display(new_frame[['result', 'unit']])
def display(self, key_formatter = long_path_formatter): def row_formatter(row): value = row['value'] lower_bound = value + row['negative_error'] upper_bound = value + row['positive_error'] pretty_string = uncertainty_formatter(value, lower_bound, upper_bound) return pretty_string # Make another data frame with the keys new_frame = self._data_frame.copy(deep=True) # type: pd.DataFrame # Add new column which will become the new index new_frame['parameter'] = map(lambda x: key_formatter(x), new_frame.index.values) # Set it as the index new_frame.set_index('parameter', drop=True, inplace=True) # compute the display new_frame['result'] = new_frame.apply(row_formatter, axis=1) # Display display(new_frame[['result','unit']])
def _setup(self): # Setup the widget, which is a bar between 0 and 100 self._bar = FloatProgress(min=0, max=100) # Set explicitly the bar to 0 self._bar.value = 0 # Setup also an HTML label (which will contain the progress, the elapsed time and the foreseen # completion time) self._title_cell = HTML() if self._title is not None: self._title_cell.value = "%s : " % self._title self._label = HTML() self._vbox = VBox(children=[self._title_cell, self._label, self._bar]) # Display everything display(self._vbox) self._animate(0)
def display(self): """ Display the time intervals :return: None """ display(self._create_pandas())
def display(self): df = pd.DataFrame() df['Bin'] = list(self._analysis_bins.keys()) df['Nside'] = [ self._analysis_bins[bin_id].nside for bin_id in self._analysis_bins ] df['Scheme'] = [ self._analysis_bins[bin_id].scheme for bin_id in self._analysis_bins ] # Compute observed counts, background counts, how many pixels we have in the ROI and # the sky area they cover n_bins = len(self._analysis_bins) obs_counts = np.zeros(n_bins) bkg_counts = np.zeros_like(obs_counts) n_pixels = np.zeros(n_bins, dtype=int) sky_area = np.zeros_like(obs_counts) size = 0 for i, bin_id in enumerate(self._analysis_bins): analysis_bin = self._analysis_bins[bin_id] sparse_obs = analysis_bin.observation_map.as_partial() sparse_bkg = analysis_bin.background_map.as_partial() size += sparse_obs.nbytes size += sparse_bkg.nbytes obs_counts[i] = sparse_obs.sum() bkg_counts[i] = sparse_bkg.sum() n_pixels[i] = sparse_obs.shape[0] sky_area[i] = n_pixels[i] * analysis_bin.observation_map.pixel_area df['Obs counts'] = obs_counts df['Bkg counts'] = bkg_counts df['obs/bkg'] = old_div(obs_counts, bkg_counts) df['Pixels in ROI'] = n_pixels df['Area (deg^2)'] = sky_area display(df) first_bin_id = list(self._analysis_bins.keys())[0] log.info("This Map Tree contains %.3f transits in the first bin" \ % self._analysis_bins[first_bin_id].n_transits) log.info("Total data size: %.2f Mb" % (size * u.byte).to(u.megabyte).value)
def print_fit_results(self): """ Display the results of the last minimization. :return: (none) """ # Restore the best fit values, in case something has changed self._restore_best_fit() # I do not use the print_param facility in iminuit because # it does not work well with console output, since it fails # to autoprobe that it is actually run in a console and uses # the HTML backend instead # Create a list of strings to print data = [] # Also store the maximum length to decide the length for the line name_length = 0 for k, v in self.parameters.iteritems(): minuit_name = self._parameter_name_to_minuit_name(k) # Format the value and the error with sensible significant # numbers x = uncertainties.ufloat(v.value, self.minuit.errors[minuit_name]) # Add some space around the +/- sign rep = x.__str__().replace("+/-", " +/- ") data.append([k, rep, v.unit]) if len(k) > name_length: name_length = len(k) table = Table(rows=data, names=["Name", "Value", "Unit"], dtype=("S%i" % name_length, str, str)) display(table) print("\nNOTE: errors on parameters are approximate. Use get_errors().\n")
def display(self): df = pd.DataFrame() df['Bin'] = self._data_bins_labels df['Nside'] = map(lambda x: x.nside, self._data_analysis_bins) df['Scheme'] = map(lambda x: x.scheme, self._data_analysis_bins) # Compute observed counts, background counts, how many pixels we have in the ROI and # the sky area they cover n_bins = len(self._data_bins_labels) obs_counts = np.zeros(n_bins) bkg_counts = np.zeros_like(obs_counts) n_pixels = np.zeros(n_bins, dtype=int) sky_area = np.zeros_like(obs_counts) size = 0 for i, analysis_bin in enumerate(self._data_analysis_bins): sparse_obs = analysis_bin.observation_map.as_partial() sparse_bkg = analysis_bin.background_map.as_partial() size += sparse_obs.nbytes size += sparse_bkg.nbytes obs_counts[i] = sparse_obs.sum() bkg_counts[i] = sparse_bkg.sum() n_pixels[i] = sparse_obs.shape[0] sky_area[i] = n_pixels[i] * analysis_bin.observation_map.pixel_area df['Obs counts'] = obs_counts df['Bkg counts'] = bkg_counts df['obs/bkg'] = obs_counts / bkg_counts df['Pixels in ROI'] = n_pixels df['Area (deg^2)'] = sky_area display(df) print("This Map Tree contains %.3f transits in the first bin" % self._data_analysis_bins[0].n_transits) print("Total data size: %.2f Mb" % (size * u.byte).to(u.megabyte).value)
def display(self, display_correlation=False, error_type="equal tail", cl=0.68): best_fit_table = self._get_results_table(error_type, cl) print("Maximum a posteriori probability (MAP) point:\n") best_fit_table.display() if display_correlation: corr_matrix = NumericMatrix(self.get_correlation_matrix()) for col in corr_matrix.colnames: corr_matrix[col].format = '2.2f' print("\nCorrelation matrix:\n") display(corr_matrix) print("\nValues of -log(posterior) at the minimum:\n") display(self.get_statistic_frame()) print("\nValues of statistical measures:\n") display(self.get_statistic_measure_frame())
def display(self, display_correlation=True, cl=0.68): best_fit_table = self._get_results_table( error_type="covariance", cl=cl, covariance=self.covariance_matrix) print("Best fit values:\n") best_fit_table.display() if display_correlation: corr_matrix = NumericMatrix(self.get_correlation_matrix()) for col in corr_matrix.colnames: corr_matrix[col].format = '2.2f' print("\nCorrelation matrix:\n") display(corr_matrix) print("\nValues of -log(likelihood) at the minimum:\n") display(self.get_statistic_frame()) print("\nValues of statistical measures:\n") display(self.get_statistic_measure_frame())
def get_point_source_flux(self, ene_min, ene_max, sources=(), confidence_level=0.68, flux_unit='erg/(s cm2)', use_components=False, components_to_use=(), sum_sources=False): """ :param ene_min: minimum energy (an astropy quantity, like 1.0 * u.keV. You can also use a frequency, like 1 * u.Hz) :param ene_max: maximum energy (an astropy quantity, like 10 * u.keV. You can also use a frequency, like 10 * u.Hz) :param sources: Use this to specify the name of the source or a tuple/list of source names to be plotted. If you don't use this, all sources will be plotted. :param confidence_level: the confidence level for the error (default: 0.68) :param flux_unit: (optional) astropy flux unit in string form (can be :param use_components: plot the components of each source (default: False) :param components_to_use: (optional) list of string names of the components to plot: including 'total' :param sum_sources: (optional) if True, also the sum of all sources will be plotted :return: """ # Convert the ene_min and ene_max in pure numbers in keV _ene_min = ene_min.to("keV").value _ene_max = ene_max.to("keV").value _params = { 'confidence_level': confidence_level, 'equal_tailed': True, # FIXME: what happens if this is False? 'best_fit': 'median', 'energy_unit': 'keV', 'flux_unit': flux_unit, 'use_components': use_components, 'components_to_use': components_to_use, 'sources_to_use': sources, 'sum_sources': sum_sources, } mle_results, bayes_results = _calculate_point_source_flux( _ene_min, _ene_max, self, **_params) # The output contains one source per row def _format_error(row): rep = uncertainty_formatter(row['flux'].value, row['low bound'].value, row['hi bound'].value) # Represent the unit as a string unit_rep = str(row['flux'].unit) return pd.Series({'flux': "%s %s" % (rep, unit_rep)}) if mle_results is not None: # Format the errors and display the resulting data frame display(mle_results.apply(_format_error, axis=1)) # Return the dataframe return mle_results elif bayes_results is not None: # Format the errors and display the resulting data frame display(bayes_results.apply(_format_error, axis=1)) # Return the dataframe return bayes_results
def display(self): return display(self)
def get_errors(self): """ Compute asymmetric errors using MINOS (slow, but accurate) and print them. NOTE: this should be called immediately after the minimize() method :return: a dictionary containing the asymmetric errors for each parameter. """ if not self._migrad_has_converged(): raise CannotComputeErrors("MIGRAD results not valid, cannot compute errors. Did you run the fit first ?") try: self.minuit.minos() except: raise MINOSFailed( "MINOS has failed. This usually means that the fit is very difficult, for example " "because of high correlation between parameters. Check the correlation matrix printed" "in the fit step, and check contour plots with getContours(). If you are using a " "user-defined model, you can also try to " "reformulate your model with less correlated parameters." ) # Make a ordered dict for the results errors = collections.OrderedDict() for k, par in self.parameters.iteritems(): minuit_name = self._parameter_name_to_minuit_name(k) errors[k] = (self.minuit.merrors[(minuit_name, -1)], self.minuit.merrors[(minuit_name, 1)]) # Set the parameters back to the best fit value self._restore_best_fit() # Print a table with the errors data = [] name_length = 0 for k, v in self.parameters.iteritems(): # Format the value and the error with sensible significant # numbers # Process the negative error x = uncertainties.ufloat(v.value, abs(errors[k][0])) # Split the uncertainty in number, negative error, and exponent (if any) num, uncm, exponent = re.match( "\(?(\-?[0-9]+\.?[0-9]+) ([0-9]+\.[0-9]+)\)?(e[\+|\-][0-9]+)?", x.__str__().replace("+/-", " ") ).groups() # Process the positive error x = uncertainties.ufloat(v.value, abs(errors[k][1])) # Split the uncertainty in number, positive error, and exponent (if any) _, uncp, _ = re.match( "\(?(\-?[0-9]+\.?[0-9]+) ([0-9]+\.[0-9]+)\)?(e[\+|\-][0-9]+)?", x.__str__().replace("+/-", " ") ).groups() if exponent is None: # Number without exponent pretty_string = "%s -%s +%s" % (num, uncm, uncp) else: # Number with exponent pretty_string = "(%s -%s +%s)%s" % (num, uncm, uncp, exponent) data.append([k, pretty_string, v.unit]) if len(k) > name_length: name_length = len(k) # Create and display the table table = Table(rows=data, names=["Name", "Value", "Unit"], dtype=("S%i" % name_length, str, str)) display(table) return errors
def print_correlation_matrix(self): """ Display the current correlation matrix :return: (none) """ # Print a custom covariance matrix because iminuit does # not guess correctly the frontend when 3ML is used # from terminal cov = self.minuit.covariance if cov is None: raise CannotComputeCovariance( "Cannot compute covariance numerically. This usually means that there are " + " unconstrained parameters. Fix those or reduce their allowed range, or " + "use a simpler model." ) # Get list of parameters keys = self.parameters.keys() # Convert them to the format for iminuit minuit_names = map(lambda k: self._parameter_name_to_minuit_name(k), keys) # Accumulate rows and compute the maximum length of the names data = [] length_of_names = 0 for key1, name1 in zip(keys, minuit_names): if len(name1) > length_of_names: length_of_names = len(name1) this_row = [] for key2, name2 in zip(keys, minuit_names): # Compute correlation between parameter key1 and key2 corr = cov[(name1, name2)] / (math.sqrt(cov[(name1, name1)]) * math.sqrt(cov[(name2, name2)])) this_row.append(corr) data.append(this_row) # Prepare the dtypes for the matrix dtypes = map(lambda x: float, minuit_names) # Column names are the parameter names cols = keys # Finally generate the matrix with the names table = NumericMatrix(rows=data, names=cols, dtype=dtypes) # Customize the format to avoid too many digits for col in table.colnames: table[col].format = "2.2f" display(table)
def get_other_instrument_information(self): """ Return the detectors used for spectral analysis as well as their background intervals. Peak flux and fluence intervals are also returned as well as best fit models :return: observing information dataframe indexed by source """ assert ( self._last_query_results is not None ), "You have to run a query before getting observing information" sources = {} for name, row in self._last_query_results.T.items(): obs_instrument = {} # loop over the observation indices for obs in range(1, 5): if obs == 1: obs_base = "other_obs" else: obs_base = "other_obs%d" % obs obs_ref = "%s_ref" % obs_base obs = row[obs_base] # this means that nothing in this column saw the grb if obs == "": observed = False else: observed = True if observed: # if we saw it then lets get the GCN gcn_number = _gcn_match.search(row[obs_ref]).group(1) # gcn_number = filter(lambda x: x != '', row[obs_ref].split('.'))[1] # make the URL gcn = "https://gcn.gsfc.nasa.gov/gcn3/%s.gcn3" % gcn_number # just for Fermi GBM, lets get the trigger number # TODO: add more instruments if obs == "Fermi-GBM": info = { "GCN": gcn, "trigger number": self._get_fermiGBM_trigger_number_from_gcn( str(gcn)), } else: info = {"GCN": gcn, "trigger number": None} obs_instrument[obs] = info sources[name] = obs_instrument # build the data frame sources = pd.concat(list(map(pd.DataFrame, list(sources.values()))), keys=list(sources.keys())) display(sources) return sources
def get_credible_intervals(self, probability=95): """ Print and returns the (equal-tail) credible intervals for all free parameters in the model :param probability: the probability for this credible interval (default: 95, corresponding to 95%) :return: a dictionary with the lower bound and upper bound of the credible intervals, as well as the median """ # Gather the credible intervals (percentiles of the posterior) credible_intervals = collections.OrderedDict() for i, (parameter_name, parameter) in enumerate(self._free_parameters.iteritems()): # Get the percentiles from the posterior samples lower_bound,median,upper_bound = numpy.percentile(self.samples[parameter_name], (100-probability,50,probability)) # Save them in the dictionary credible_intervals[parameter_name] = {'lower bound': lower_bound, 'median': median, 'upper bound': upper_bound} # Print a table with the errors data = [] name_length = 0 for i, (parameter_name, parameter) in enumerate(self._free_parameters.iteritems()): # Format the value and the error with sensible significant # numbers lower_bound, median, upper_bound = [credible_intervals[parameter_name][key] for key in ('lower bound', 'median', 'upper bound') ] # Process the negative "error" x = uncertainties.ufloat(median, abs(lower_bound - median)) # Split the uncertainty in number, negative error, and exponent (if any) number, unc_lower_bound, exponent = re.match('\(?(\-?[0-9]+\.?[0-9]+) ([0-9]+\.[0-9]+)\)?(e[\+|\-][0-9]+)?', x.__str__().replace("+/-", " ")).groups() # Process the positive "error" x = uncertainties.ufloat(median, abs(upper_bound - median)) # Split the uncertainty in number, positive error, and exponent (if any) _, unc_upper_bound, _ = re.match('\(?(\-?[0-9]+\.?[0-9]+) ([0-9]+\.[0-9]+)\)?(e[\+|\-][0-9]+)?', x.__str__().replace("+/-", " ")).groups() if exponent is None: # Number without exponent pretty_string = "%s -%s +%s" % (number, unc_lower_bound, unc_upper_bound) else: # Number with exponent pretty_string = "(%s -%s +%s)%s" % (number, unc_lower_bound, unc_upper_bound, exponent) unit = self._free_parameters[parameter_name].unit data.append([parameter_name, pretty_string, unit]) if len(parameter_name) > name_length: name_length = len(parameter_name) # Create and display the table table = Table(rows=data, names=["Name", "Value", "Unit"], dtype=('S%i' % name_length, str, 'S15')) display(table) return credible_intervals
def fit(self): """ Perform a fit of the current likelihood model on the datasets :param pre_fit: (True or False) If True, perform a pre-fit with only normalizations free (experimental) :return: a dictionary with the results on the parameters, and the values of the likelihood at the minimum for each dataset and the total one. """ # Update the list of free parameters, to be safe against changes the user might do between # the creation of this class and the calling of this method self._update_free_parameters() # Now check and fix if needed all the deltas of the parameters # to 10% of their value (otherwise the fit will be super-slow) for k, v in self._free_parameters.iteritems(): if abs(v.delta) < abs(v.value) * 0.1: v.delta = abs(v.value) * 0.1 # Instance the minimizer self._minimizer = self.Minimizer(self.minus_log_like_profile, self._free_parameters) # Perform the fit xs, log_likelihood_minimum = self._minimizer.minimize() # Store the current minimum for the -log likelihood self._current_minimum = float(log_likelihood_minimum) # Print results print("Best fit values:\n") self._minimizer.print_fit_results() print("Nuisance parameters:\n") nuisance_parameters = collections.OrderedDict() for dataset in self._data_list.values(): for pName in dataset.get_nuisance_parameters(): nuisance_parameters[(dataset.get_name(), pName)] = dataset.nuisanceParameters[pName] nuisance_parameters_table = self._get_table_of_parameters(nuisance_parameters) display(nuisance_parameters_table) print("\nCorrelation matrix:\n") self._minimizer.print_correlation_matrix() print("\nMinimum of -logLikelihood is: %s\n" % log_likelihood_minimum) print("Contributions to the -logLikelihood at the minimum:\n") # Fill the dictionary with the values of the -log likelihood (total, and dataset by dataset) minus_log_likelihood_values = collections.OrderedDict() minus_log_likelihood_values["total"] = log_likelihood_minimum data = [] name_length = 0 for dataset in self._data_list.values(): ml = dataset.inner_fit() * (-1) minus_log_likelihood_values[dataset.get_name()] = ml name_length = max(name_length, len(dataset.get_name()) + 1) data.append([dataset.get_name(), ml]) log_like_values_table = Table(rows=data, names=["Detector", "-LogL"], dtype=("S%i" % name_length, float)) log_like_values_table["-LogL"].format = "2.2f" display(log_like_values_table) # Prepare the dictionary with the results results = collections.OrderedDict() results["parameters"] = xs results["minusLogLike"] = minus_log_likelihood_values return results