def worker(grade):
polynomial, log_like = polyfit(bins,
cnts,
grade,
exposure,
bayes=bayes)
return log_like
def worker(counts):
with silence_console_log():
polynomial, _ = polyfit(
selected_midpoints,
counts,
self._optimal_polynomial_grade,
selected_exposure,
bayes=bayes,
)
return polynomial
def _fit_global_and_determine_optimum_grade(self, cnts, bins, exposure):
"""
Provides the ability to find the optimum polynomial grade for *binned* counts by fitting the
total (all channels) to 0-4 order polynomials and then comparing them via a likelihood ratio test.
:param cnts: counts per bin
:param bins: the bins used
:param exposure: exposure per bin
:return: polynomial grade
"""
min_grade = 0
max_grade = 4
log_likelihoods = []
for grade in range(min_grade, max_grade + 1):
polynomial, log_like = polyfit(bins, cnts, grade, exposure)
log_likelihoods.append(log_like)
# Found the best one
delta_loglike = np.array(
map(lambda x: 2 * (x[0] - x[1]),
zip(log_likelihoods[:-1], log_likelihoods[1:])))
# print("\ndelta log-likelihoods:")
# for i in range(max_grade):
# print("%s -> %s: delta Log-likelihood = %s" % (i, i + 1, deltaLoglike[i]))
# print("")
delta_threshold = 9.0
mask = (delta_loglike >= delta_threshold)
if (len(mask.nonzero()[0]) == 0):
# best grade is zero!
best_grade = 0
else:
best_grade = mask.nonzero()[0][-1] + 1
return best_grade
def _fit_global_and_determine_optimum_grade(self, cnts, bins, exposure):
"""
Provides the ability to find the optimum polynomial grade for *binned* counts by fitting the
total (all channels) to 0-4 order polynomials and then comparing them via a likelihood ratio test.
:param cnts: counts per bin
:param bins: the bins used
:param exposure: exposure per bin
:return: polynomial grade
"""
min_grade = 0
max_grade = 4
log_likelihoods = []
for grade in range(min_grade, max_grade + 1):
polynomial, log_like = polyfit(bins, cnts, grade, exposure)
log_likelihoods.append(log_like)
# Found the best one
delta_loglike = np.array(map(lambda x: 2 * (x[0] - x[1]), zip(log_likelihoods[:-1], log_likelihoods[1:])))
# print("\ndelta log-likelihoods:")
# for i in range(max_grade):
# print("%s -> %s: delta Log-likelihood = %s" % (i, i + 1, deltaLoglike[i]))
# print("")
delta_threshold = 9.0
mask = (delta_loglike >= delta_threshold)
if (len(mask.nonzero()[0]) == 0):
# best grade is zero!
best_grade = 0
else:
best_grade = mask.nonzero()[0][-1] + 1
return best_grade
def fit_backgrounds(i):
update_logging_level("CRITICAL")
# selections
s1 = x < np.max([-i * .1, x.min() + 10])
s2 = x > i
idx = s1 | s2
# fit background
_, ll = polyfit(x[idx],
y[idx],
grade=3,
exposure=exposure[idx])
return ll
def worker(channel):
channel_mask = total_poly_energies == channel
# Mask background events and current channel
# poly_chan_mask = np.logical_and(poly_mask, channel_mask)
# Select the masked events
current_events = total_poly_events[channel_mask]
cnts, bins = np.histogram(current_events, bins=these_bins)
polynomial, _ = polyfit(mean_time[non_zero_mask],
cnts[non_zero_mask],
self._optimal_polynomial_grade,
exposure_per_bin[non_zero_mask],
bayes=bayes)
return polynomial
def _fit_polynomials(self, bayes=False):
"""
fits a polynomial to all channels over the input time intervals
:param fit_intervals: str input intervals
:return:
"""
# mark that we have fit a poly now
self._poly_fit_exists = True
# we need to adjust the selection to the true intervals of the time-binned spectra
tmp_poly_intervals = self._poly_intervals
poly_intervals = self._adjust_to_true_intervals(tmp_poly_intervals)
self._poly_intervals = poly_intervals
# now lets get all the counts, exposure and midpoints for the
# selection
selected_counts = []
selected_exposure = []
selected_midpoints = []
for selection in poly_intervals:
# get the mask of these bins
mask = self._select_bins(selection.start_time, selection.stop_time)
# the counts will be (time, channel) here,
# so the mask is selecting time.
# a sum along axis=0 is a sum in time, while axis=1 is a sum in energy
selected_counts.extend(
self._binned_spectrum_set.counts_per_bin[mask])
selected_exposure.extend(
self._binned_spectrum_set.exposure_per_bin[mask])
selected_midpoints.extend(
self._binned_spectrum_set.time_intervals.mid_points[mask]
)
selected_counts = np.array(selected_counts)
selected_midpoints = np.array(selected_midpoints)
selected_exposure = np.array(selected_exposure)
# Now we will find the the best poly order unless the use specified one
# The total cnts (over channels) is binned
if self._user_poly_order == -1:
self._optimal_polynomial_grade = (
self._fit_global_and_determine_optimum_grade(
selected_counts.sum(axis=1),
selected_midpoints,
selected_exposure,
bayes=bayes,
)
)
log.info(
"Auto-determined polynomial order: %d"
% self._optimal_polynomial_grade
)
else:
self._optimal_polynomial_grade = self._user_poly_order
if threeML_config["parallel"]["use_parallel"]:
def worker(counts):
with silence_console_log():
polynomial, _ = polyfit(
selected_midpoints,
counts,
self._optimal_polynomial_grade,
selected_exposure,
bayes=bayes,
)
return polynomial
client = ParallelClient()
polynomials = client.execute_with_progress_bar(
worker, selected_counts.T, name=f"Fitting {self._instrument} background")
else:
polynomials = []
# now fit the light curve of each channel
# and save the estimated polynomial
for counts in tqdm(
selected_counts.T, desc=f"Fitting {self._instrument} background"
):
with silence_console_log():
polynomial, _ = polyfit(
selected_midpoints,
counts,
self._optimal_polynomial_grade,
selected_exposure,
bayes=bayes,
)
polynomials.append(polynomial)
self._polynomials = polynomials
def _fit_polynomials(self):
"""
Binned fit to each channel. Sets the polynomial array that will be used to compute
counts over an interval
:return:
"""
self._poly_fit_exists = True
self._fit_method_info['bin type'] = 'Binned'
self._fit_method_info['fit method'] = threeML_config['event list'][
'binned fit method']
# Select all the events that are in the background regions
# and make a mask
all_bkg_masks = []
for selection in self._poly_intervals:
all_bkg_masks.append(
np.logical_and(self._arrival_times >= selection.start_time,
self._arrival_times <= selection.stop_time))
poly_mask = all_bkg_masks[0]
# If there are multiple masks:
if len(all_bkg_masks) > 1:
for mask in all_bkg_masks[1:]:
poly_mask = np.logical_or(poly_mask, mask)
# Select the all the events in the poly selections
# We only need to do this once
total_poly_events = self._arrival_times[poly_mask]
# For the channel energies we will need to down select again.
# We can go ahead and do this to avoid repeated computations
total_poly_energies = self._measurement[poly_mask]
# This calculation removes the unselected portion of the light curve
# so that we are not fitting zero counts. It will be used in the channel calculations
# as well
bin_width = 1. # seconds
these_bins = np.arange(self._start_time, self._stop_time, bin_width)
cnts, bins = np.histogram(total_poly_events, bins=these_bins)
# Find the mean time of the bins and calculate the exposure in each bin
mean_time = []
exposure_per_bin = []
for i in xrange(len(bins) - 1):
m = np.mean((bins[i], bins[i + 1]))
mean_time.append(m)
exposure_per_bin.append(
self.exposure_over_interval(bins[i], bins[i + 1]))
mean_time = np.array(mean_time)
exposure_per_bin = np.array(exposure_per_bin)
# Remove bins with zero counts
all_non_zero_mask = []
for selection in self._poly_intervals:
all_non_zero_mask.append(
np.logical_and(mean_time >= selection.start_time,
mean_time <= selection.stop_time))
non_zero_mask = all_non_zero_mask[0]
if len(all_non_zero_mask) > 1:
for mask in all_non_zero_mask[1:]:
non_zero_mask = np.logical_or(mask, non_zero_mask)
# Now we will find the the best poly order unless the use specified one
# The total cnts (over channels) is binned to .1 sec intervals
if self._user_poly_order == -1:
self._optimal_polynomial_grade = self._fit_global_and_determine_optimum_grade(
cnts[non_zero_mask], mean_time[non_zero_mask],
exposure_per_bin[non_zero_mask])
if self._verbose:
print("Auto-determined polynomial order: %d" %
self._optimal_polynomial_grade)
print('\n')
else:
self._optimal_polynomial_grade = self._user_poly_order
channels = range(self._first_channel,
self._n_channels + self._first_channel)
polynomials = []
with progress_bar(self._n_channels,
title="Fitting %s background" %
self._instrument) as p:
for channel in channels:
channel_mask = total_poly_energies == channel
# Mask background events and current channel
# poly_chan_mask = np.logical_and(poly_mask, channel_mask)
# Select the masked events
current_events = total_poly_events[channel_mask]
# now bin the selected channel counts
cnts, bins = np.histogram(current_events, bins=these_bins)
# Put data to fit in an x vector and y vector
polynomial, _ = polyfit(mean_time[non_zero_mask],
cnts[non_zero_mask],
self._optimal_polynomial_grade,
exposure_per_bin[non_zero_mask])
polynomials.append(polynomial)
p.increase()
# We are now ready to return the polynomials
self._polynomials = polynomials
def _fit_polynomials(self):
"""
fits a polynomial to all channels over the input time intervals
:param fit_intervals: str input intervals
:return:
"""
# mark that we have fit a poly now
self._poly_fit_exists = True
# set the fit method
self._fit_method_info['bin type'] = 'Binned'
self._fit_method_info['fit method'] = threeML_config['event list']['binned fit method']
# we need to adjust the selection to the true intervals of the time-binned spectra
tmp_poly_intervals = self._poly_intervals
poly_intervals = self._adjust_to_true_intervals(tmp_poly_intervals)
self._poly_intervals = poly_intervals
# now lets get all the counts, exposure and midpoints for the
# selection
selected_counts = []
selected_exposure = []
selected_midpoints = []
for selection in poly_intervals:
# get the mask of these bins
mask = self._select_bins(selection.start_time,selection.stop_time)
# the counts will be (time, channel) here,
# so the mask is selecting time.
# a sum along axis=0 is a sum in time, while axis=1 is a sum in energy
selected_counts.extend(self._binned_spectrum_set.counts_per_bin[mask])
selected_exposure.extend(self._binned_spectrum_set.exposure_per_bin[mask])
selected_midpoints.extend(self._binned_spectrum_set.time_intervals.mid_points[mask])
selected_counts = np.array(selected_counts)
selected_midpoints = np.array(selected_midpoints)
selected_exposure = np.array(selected_exposure)
# Now we will find the the best poly order unless the use specified one
# The total cnts (over channels) is binned
if self._user_poly_order == -1:
self._optimal_polynomial_grade = self._fit_global_and_determine_optimum_grade(selected_counts.sum(axis=1),
selected_midpoints,
selected_exposure)
if self._verbose:
print("Auto-determined polynomial order: %d" % self._optimal_polynomial_grade)
print('\n')
else:
self._optimal_polynomial_grade = self._user_poly_order
polynomials = []
# now fit the light curve of each channel
# and save the estimated polynomial
with progress_bar(self._n_channels, title="Fitting background") as p:
for counts in selected_counts.T:
polynomial, _ = polyfit(selected_midpoints,
counts,
self._optimal_polynomial_grade,
selected_exposure)
polynomials.append(polynomial)
p.increase()
self._polynomials = polynomials
def _compute_primary_bkg_selection(self):
# number of sub divisions
n_trials = 100
# the end points of the background fit
end_points = np.linspace(1, self._max_time, n_trials)
log_likes = np.empty((self._n_light_curves, n_trials))
# loop through each light curve
for j, lc in enumerate(self._light_curves):
# extract the light curve info
_log_likes = np.empty(n_trials)
y = lc.counts
x = lc.mean_times
exposure = lc.exposure
# define a closure for this light curve
def fit_backgrounds(i):
update_logging_level("CRITICAL")
# selections
s1 = x < np.max([-i * .1, x.min() + 10])
s2 = x > i
idx = s1 | s2
# fit background
_, ll = polyfit(x[idx],
y[idx],
grade=3,
exposure=exposure[idx])
return ll
# continuosly increase the starting of the bkg
# selections and store the log likelihood
_log_likes = Parallel(n_jobs=8)(delayed(fit_backgrounds)(i)
for i in end_points)
log_likes[j, ...] = np.array(_log_likes)
# now difference them all
# the idea here is that the ordered log likes
# will flatten out once a good background is
# found and then we can identify that via its
# change points
delta = []
for ll in log_likes:
delta.append(np.diff(ll))
delta = np.vstack(delta).T
delta = delta.reshape(delta.shape[0], -1)
delta = (delta - np.min(delta, axis=0).reshape((1, -1)) + 1)
angles = angle_mapping(delta)
dist = distance_mapping(delta)
penalty = 2 * np.log(len(angles))
algo = rpt.Pelt().fit(angles)
cpts_seg = algo.predict(pen=penalty)
# algo = rpt.Pelt().fit(dist)
# cpts_seg2 = algo.predict(pen=penalty)
algo = rpt.Pelt().fit(dist / dist.max())
cpts_seg2 = algo.predict(pen=.01)
tol = 1E-2
best_range = len(dist)
while (best_range >= len(dist) - 1) and (tol < 1):
for i in cpts_seg2:
best_range = i
if np.alltrue(np.abs(np.diff(dist / dist.max())[i:]) < tol):
break
tol += 1e-2
time = np.linspace(1, self._max_time, n_trials)[best_range + 1]
# now save all of this
# and fit a polynomial to
# each light curve and save it
pre = np.max([-time * .1, x.min() + 10])
post = time
# create polys
self._polys = []
for j, lc in enumerate(self._light_curves):
_log_likes = np.empty(n_trials)
y = lc.counts
x = lc.mean_times
exposure = lc.exposure
idx = (x < pre) | (x > post)
p, ll = polyfit(x[idx], y[idx], grade=3, exposure=exposure[idx])
self._polys.append(p)
self._pre = pre
self._post = post
def _fit_global_and_determine_optimum_grade(self,
cnts,
bins,
exposure,
bayes=False):
"""
Provides the ability to find the optimum polynomial grade for *binned* counts by fitting the
total (all channels) to 0-4 order polynomials and then comparing them via a likelihood ratio test.
:param cnts: counts per bin
:param bins: the bins used
:param exposure: exposure per bin
:param bayes:
:return: polynomial grade
"""
min_grade = 0
max_grade = 4
log_likelihoods = []
log.debug("attempting to find best poly with binned data")
if threeML_config["parallel"]["use_parallel"]:
def worker(grade):
polynomial, log_like = polyfit(bins,
cnts,
grade,
exposure,
bayes=bayes)
return log_like
client = ParallelClient()
log_likelihoods = client.execute_with_progress_bar(
worker,
list(range(min_grade, max_grade + 1)),
name="Finding best polynomial Order")
else:
for grade in trange(min_grade,
max_grade + 1,
desc="Finding best polynomial Order"):
polynomial, log_like = polyfit(bins,
cnts,
grade,
exposure,
bayes=bayes)
log_likelihoods.append(log_like)
# Found the best one
delta_loglike = np.array([
2 * (x[0] - x[1])
for x in zip(log_likelihoods[:-1], log_likelihoods[1:])
])
log.debug(f"log likes {log_likelihoods}")
log.debug(f" delta loglikes {delta_loglike}")
delta_threshold = 9.0
mask = delta_loglike >= delta_threshold
if len(mask.nonzero()[0]) == 0:
# best grade is zero!
best_grade = 0
else:
best_grade = mask.nonzero()[0][-1] + 1
return best_grade
