def worker(grade): polynomial, log_like = polyfit(bins, cnts, grade, exposure, bayes=bayes) return log_like
def worker(counts): with silence_console_log(): polynomial, _ = polyfit( selected_midpoints, counts, self._optimal_polynomial_grade, selected_exposure, bayes=bayes, ) return polynomial
def _fit_global_and_determine_optimum_grade(self, cnts, bins, exposure): """ Provides the ability to find the optimum polynomial grade for *binned* counts by fitting the total (all channels) to 0-4 order polynomials and then comparing them via a likelihood ratio test. :param cnts: counts per bin :param bins: the bins used :param exposure: exposure per bin :return: polynomial grade """ min_grade = 0 max_grade = 4 log_likelihoods = [] for grade in range(min_grade, max_grade + 1): polynomial, log_like = polyfit(bins, cnts, grade, exposure) log_likelihoods.append(log_like) # Found the best one delta_loglike = np.array( map(lambda x: 2 * (x[0] - x[1]), zip(log_likelihoods[:-1], log_likelihoods[1:]))) # print("\ndelta log-likelihoods:") # for i in range(max_grade): # print("%s -> %s: delta Log-likelihood = %s" % (i, i + 1, deltaLoglike[i])) # print("") delta_threshold = 9.0 mask = (delta_loglike >= delta_threshold) if (len(mask.nonzero()[0]) == 0): # best grade is zero! best_grade = 0 else: best_grade = mask.nonzero()[0][-1] + 1 return best_grade
def _fit_global_and_determine_optimum_grade(self, cnts, bins, exposure): """ Provides the ability to find the optimum polynomial grade for *binned* counts by fitting the total (all channels) to 0-4 order polynomials and then comparing them via a likelihood ratio test. :param cnts: counts per bin :param bins: the bins used :param exposure: exposure per bin :return: polynomial grade """ min_grade = 0 max_grade = 4 log_likelihoods = [] for grade in range(min_grade, max_grade + 1): polynomial, log_like = polyfit(bins, cnts, grade, exposure) log_likelihoods.append(log_like) # Found the best one delta_loglike = np.array(map(lambda x: 2 * (x[0] - x[1]), zip(log_likelihoods[:-1], log_likelihoods[1:]))) # print("\ndelta log-likelihoods:") # for i in range(max_grade): # print("%s -> %s: delta Log-likelihood = %s" % (i, i + 1, deltaLoglike[i])) # print("") delta_threshold = 9.0 mask = (delta_loglike >= delta_threshold) if (len(mask.nonzero()[0]) == 0): # best grade is zero! best_grade = 0 else: best_grade = mask.nonzero()[0][-1] + 1 return best_grade
def fit_backgrounds(i): update_logging_level("CRITICAL") # selections s1 = x < np.max([-i * .1, x.min() + 10]) s2 = x > i idx = s1 | s2 # fit background _, ll = polyfit(x[idx], y[idx], grade=3, exposure=exposure[idx]) return ll
def worker(channel): channel_mask = total_poly_energies == channel # Mask background events and current channel # poly_chan_mask = np.logical_and(poly_mask, channel_mask) # Select the masked events current_events = total_poly_events[channel_mask] cnts, bins = np.histogram(current_events, bins=these_bins) polynomial, _ = polyfit(mean_time[non_zero_mask], cnts[non_zero_mask], self._optimal_polynomial_grade, exposure_per_bin[non_zero_mask], bayes=bayes) return polynomial
def _fit_polynomials(self, bayes=False): """ fits a polynomial to all channels over the input time intervals :param fit_intervals: str input intervals :return: """ # mark that we have fit a poly now self._poly_fit_exists = True # we need to adjust the selection to the true intervals of the time-binned spectra tmp_poly_intervals = self._poly_intervals poly_intervals = self._adjust_to_true_intervals(tmp_poly_intervals) self._poly_intervals = poly_intervals # now lets get all the counts, exposure and midpoints for the # selection selected_counts = [] selected_exposure = [] selected_midpoints = [] for selection in poly_intervals: # get the mask of these bins mask = self._select_bins(selection.start_time, selection.stop_time) # the counts will be (time, channel) here, # so the mask is selecting time. # a sum along axis=0 is a sum in time, while axis=1 is a sum in energy selected_counts.extend( self._binned_spectrum_set.counts_per_bin[mask]) selected_exposure.extend( self._binned_spectrum_set.exposure_per_bin[mask]) selected_midpoints.extend( self._binned_spectrum_set.time_intervals.mid_points[mask] ) selected_counts = np.array(selected_counts) selected_midpoints = np.array(selected_midpoints) selected_exposure = np.array(selected_exposure) # Now we will find the the best poly order unless the use specified one # The total cnts (over channels) is binned if self._user_poly_order == -1: self._optimal_polynomial_grade = ( self._fit_global_and_determine_optimum_grade( selected_counts.sum(axis=1), selected_midpoints, selected_exposure, bayes=bayes, ) ) log.info( "Auto-determined polynomial order: %d" % self._optimal_polynomial_grade ) else: self._optimal_polynomial_grade = self._user_poly_order if threeML_config["parallel"]["use_parallel"]: def worker(counts): with silence_console_log(): polynomial, _ = polyfit( selected_midpoints, counts, self._optimal_polynomial_grade, selected_exposure, bayes=bayes, ) return polynomial client = ParallelClient() polynomials = client.execute_with_progress_bar( worker, selected_counts.T, name=f"Fitting {self._instrument} background") else: polynomials = [] # now fit the light curve of each channel # and save the estimated polynomial for counts in tqdm( selected_counts.T, desc=f"Fitting {self._instrument} background" ): with silence_console_log(): polynomial, _ = polyfit( selected_midpoints, counts, self._optimal_polynomial_grade, selected_exposure, bayes=bayes, ) polynomials.append(polynomial) self._polynomials = polynomials
def _fit_polynomials(self): """ Binned fit to each channel. Sets the polynomial array that will be used to compute counts over an interval :return: """ self._poly_fit_exists = True self._fit_method_info['bin type'] = 'Binned' self._fit_method_info['fit method'] = threeML_config['event list'][ 'binned fit method'] # Select all the events that are in the background regions # and make a mask all_bkg_masks = [] for selection in self._poly_intervals: all_bkg_masks.append( np.logical_and(self._arrival_times >= selection.start_time, self._arrival_times <= selection.stop_time)) poly_mask = all_bkg_masks[0] # If there are multiple masks: if len(all_bkg_masks) > 1: for mask in all_bkg_masks[1:]: poly_mask = np.logical_or(poly_mask, mask) # Select the all the events in the poly selections # We only need to do this once total_poly_events = self._arrival_times[poly_mask] # For the channel energies we will need to down select again. # We can go ahead and do this to avoid repeated computations total_poly_energies = self._measurement[poly_mask] # This calculation removes the unselected portion of the light curve # so that we are not fitting zero counts. It will be used in the channel calculations # as well bin_width = 1. # seconds these_bins = np.arange(self._start_time, self._stop_time, bin_width) cnts, bins = np.histogram(total_poly_events, bins=these_bins) # Find the mean time of the bins and calculate the exposure in each bin mean_time = [] exposure_per_bin = [] for i in xrange(len(bins) - 1): m = np.mean((bins[i], bins[i + 1])) mean_time.append(m) exposure_per_bin.append( self.exposure_over_interval(bins[i], bins[i + 1])) mean_time = np.array(mean_time) exposure_per_bin = np.array(exposure_per_bin) # Remove bins with zero counts all_non_zero_mask = [] for selection in self._poly_intervals: all_non_zero_mask.append( np.logical_and(mean_time >= selection.start_time, mean_time <= selection.stop_time)) non_zero_mask = all_non_zero_mask[0] if len(all_non_zero_mask) > 1: for mask in all_non_zero_mask[1:]: non_zero_mask = np.logical_or(mask, non_zero_mask) # Now we will find the the best poly order unless the use specified one # The total cnts (over channels) is binned to .1 sec intervals if self._user_poly_order == -1: self._optimal_polynomial_grade = self._fit_global_and_determine_optimum_grade( cnts[non_zero_mask], mean_time[non_zero_mask], exposure_per_bin[non_zero_mask]) if self._verbose: print("Auto-determined polynomial order: %d" % self._optimal_polynomial_grade) print('\n') else: self._optimal_polynomial_grade = self._user_poly_order channels = range(self._first_channel, self._n_channels + self._first_channel) polynomials = [] with progress_bar(self._n_channels, title="Fitting %s background" % self._instrument) as p: for channel in channels: channel_mask = total_poly_energies == channel # Mask background events and current channel # poly_chan_mask = np.logical_and(poly_mask, channel_mask) # Select the masked events current_events = total_poly_events[channel_mask] # now bin the selected channel counts cnts, bins = np.histogram(current_events, bins=these_bins) # Put data to fit in an x vector and y vector polynomial, _ = polyfit(mean_time[non_zero_mask], cnts[non_zero_mask], self._optimal_polynomial_grade, exposure_per_bin[non_zero_mask]) polynomials.append(polynomial) p.increase() # We are now ready to return the polynomials self._polynomials = polynomials
def _fit_polynomials(self): """ fits a polynomial to all channels over the input time intervals :param fit_intervals: str input intervals :return: """ # mark that we have fit a poly now self._poly_fit_exists = True # set the fit method self._fit_method_info['bin type'] = 'Binned' self._fit_method_info['fit method'] = threeML_config['event list']['binned fit method'] # we need to adjust the selection to the true intervals of the time-binned spectra tmp_poly_intervals = self._poly_intervals poly_intervals = self._adjust_to_true_intervals(tmp_poly_intervals) self._poly_intervals = poly_intervals # now lets get all the counts, exposure and midpoints for the # selection selected_counts = [] selected_exposure = [] selected_midpoints = [] for selection in poly_intervals: # get the mask of these bins mask = self._select_bins(selection.start_time,selection.stop_time) # the counts will be (time, channel) here, # so the mask is selecting time. # a sum along axis=0 is a sum in time, while axis=1 is a sum in energy selected_counts.extend(self._binned_spectrum_set.counts_per_bin[mask]) selected_exposure.extend(self._binned_spectrum_set.exposure_per_bin[mask]) selected_midpoints.extend(self._binned_spectrum_set.time_intervals.mid_points[mask]) selected_counts = np.array(selected_counts) selected_midpoints = np.array(selected_midpoints) selected_exposure = np.array(selected_exposure) # Now we will find the the best poly order unless the use specified one # The total cnts (over channels) is binned if self._user_poly_order == -1: self._optimal_polynomial_grade = self._fit_global_and_determine_optimum_grade(selected_counts.sum(axis=1), selected_midpoints, selected_exposure) if self._verbose: print("Auto-determined polynomial order: %d" % self._optimal_polynomial_grade) print('\n') else: self._optimal_polynomial_grade = self._user_poly_order polynomials = [] # now fit the light curve of each channel # and save the estimated polynomial with progress_bar(self._n_channels, title="Fitting background") as p: for counts in selected_counts.T: polynomial, _ = polyfit(selected_midpoints, counts, self._optimal_polynomial_grade, selected_exposure) polynomials.append(polynomial) p.increase() self._polynomials = polynomials
def _compute_primary_bkg_selection(self): # number of sub divisions n_trials = 100 # the end points of the background fit end_points = np.linspace(1, self._max_time, n_trials) log_likes = np.empty((self._n_light_curves, n_trials)) # loop through each light curve for j, lc in enumerate(self._light_curves): # extract the light curve info _log_likes = np.empty(n_trials) y = lc.counts x = lc.mean_times exposure = lc.exposure # define a closure for this light curve def fit_backgrounds(i): update_logging_level("CRITICAL") # selections s1 = x < np.max([-i * .1, x.min() + 10]) s2 = x > i idx = s1 | s2 # fit background _, ll = polyfit(x[idx], y[idx], grade=3, exposure=exposure[idx]) return ll # continuosly increase the starting of the bkg # selections and store the log likelihood _log_likes = Parallel(n_jobs=8)(delayed(fit_backgrounds)(i) for i in end_points) log_likes[j, ...] = np.array(_log_likes) # now difference them all # the idea here is that the ordered log likes # will flatten out once a good background is # found and then we can identify that via its # change points delta = [] for ll in log_likes: delta.append(np.diff(ll)) delta = np.vstack(delta).T delta = delta.reshape(delta.shape[0], -1) delta = (delta - np.min(delta, axis=0).reshape((1, -1)) + 1) angles = angle_mapping(delta) dist = distance_mapping(delta) penalty = 2 * np.log(len(angles)) algo = rpt.Pelt().fit(angles) cpts_seg = algo.predict(pen=penalty) # algo = rpt.Pelt().fit(dist) # cpts_seg2 = algo.predict(pen=penalty) algo = rpt.Pelt().fit(dist / dist.max()) cpts_seg2 = algo.predict(pen=.01) tol = 1E-2 best_range = len(dist) while (best_range >= len(dist) - 1) and (tol < 1): for i in cpts_seg2: best_range = i if np.alltrue(np.abs(np.diff(dist / dist.max())[i:]) < tol): break tol += 1e-2 time = np.linspace(1, self._max_time, n_trials)[best_range + 1] # now save all of this # and fit a polynomial to # each light curve and save it pre = np.max([-time * .1, x.min() + 10]) post = time # create polys self._polys = [] for j, lc in enumerate(self._light_curves): _log_likes = np.empty(n_trials) y = lc.counts x = lc.mean_times exposure = lc.exposure idx = (x < pre) | (x > post) p, ll = polyfit(x[idx], y[idx], grade=3, exposure=exposure[idx]) self._polys.append(p) self._pre = pre self._post = post
def _fit_global_and_determine_optimum_grade(self, cnts, bins, exposure, bayes=False): """ Provides the ability to find the optimum polynomial grade for *binned* counts by fitting the total (all channels) to 0-4 order polynomials and then comparing them via a likelihood ratio test. :param cnts: counts per bin :param bins: the bins used :param exposure: exposure per bin :param bayes: :return: polynomial grade """ min_grade = 0 max_grade = 4 log_likelihoods = [] log.debug("attempting to find best poly with binned data") if threeML_config["parallel"]["use_parallel"]: def worker(grade): polynomial, log_like = polyfit(bins, cnts, grade, exposure, bayes=bayes) return log_like client = ParallelClient() log_likelihoods = client.execute_with_progress_bar( worker, list(range(min_grade, max_grade + 1)), name="Finding best polynomial Order") else: for grade in trange(min_grade, max_grade + 1, desc="Finding best polynomial Order"): polynomial, log_like = polyfit(bins, cnts, grade, exposure, bayes=bayes) log_likelihoods.append(log_like) # Found the best one delta_loglike = np.array([ 2 * (x[0] - x[1]) for x in zip(log_likelihoods[:-1], log_likelihoods[1:]) ]) log.debug(f"log likes {log_likelihoods}") log.debug(f" delta loglikes {delta_loglike}") delta_threshold = 9.0 mask = delta_loglike >= delta_threshold if len(mask.nonzero()[0]) == 0: # best grade is zero! best_grade = 0 else: best_grade = mask.nonzero()[0][-1] + 1 return best_grade