def test_errors(self):
with self.assertRaises(ValueError):
snr.periodogram_snr([1., 2., 3., 4., 5.], [3., 4., 5.], 1, 1, 'LS')
with self.assertRaises(AttributeError):
snr.periodogram_snr([1, 2], [1, 2], [1, 2], 23, 'LS')
with self.assertRaises(ValueError):
snr.periodogram_snr([1, np.inf], [1, 2], 1, 4, 'PDM')
with self.assertRaises(ValueError):
snr.periodogram_snr([1, np.nan], [1, 2], 1, 5, 'PDM')
with self.assertRaises(AttributeError):
snr.periodogram_snr([1, 2], [1, 2], [1, 2], 2, 'bob__')
def test_pdm_snr(self):
snr_val = snr.periodogram_snr(self.pdm_periodogram, self.periods,
self.index, 80., 'PDM')
self.assertAlmostEqual(snr_val, (1. - self.pdm_per_amplitude) /
(self.per_pdm_delta / 2.),
places=8)
def test_bls_snr(self):
snr_val = snr.periodogram_snr(self.periodogram, self.periods,
self.index, 80., 'BLS')
self.assertAlmostEqual(snr_val,
self.per_amplitude / (self.per_delta / 2.),
places=8)
def iterative_deblend(t,
y,
dy,
neighbors,
period_finding_func,
results_storage_container,
which_method,
function_params=None,
nharmonics_fit=5,
nharmonics_resid=10,
ID=None,
medianfilter=False,
freq_window_epsilon_mf=1.,
freq_window_epsilon_snr=1.,
window_size_mf=40,
window_size_snr=40,
snr_threshold=0.,
spn_threshold=None,
fap_baluev_threshold=None,
neighbor_peaks_tocheck=8,
max_blend_recursion=8,
recursion_level=0,
nworkers=1,
max_neighbor_amp=3.):
"""
Iteratively deblend a lightcurve against neighbors
Parameters
----------
t: array_like
Time coordinates of lightcurve
y: array_like
Brightness measurements (in mag)
dy: array_like
Uncertainties for each brightness measurement
neighbors: list
List of (t, y, dy) lightcurves for each
neighbor (NOT including the lightcurve we are deblending)
period_finding_func: function
Function used to find the period. Output format assumed to
be the same as that used in astrobase.periodbase
results_storage_container: instance of data_processing.periodsearch_results
Used to store the results
which_method: string
Which period search method is being used
(optional from here below):
function_params: dictionary
A dictionary containing parameters for the function in
period_finding_func
nharmonics_fit: int
Number of harmonics to use in the fit (used to estimate
flux amplitude)
nharmonics_resid: int
Number of harmonics to use to obtain the residual if we
find that the signal is a blend of a neighboring signal
ID: string
ID of the object
medianfilter: boolean
whether to median filter the periodogram
freq_window_epsilon_mf: int
sets the size of the exclusion area
in the periodogram for the median filter calculation
freq_window_epsilon_snr: int
sets the size of the exclusion area
in the periodogram for the SNR calculation
window_size_mf: int
number of points to include in
calculating the median value for median filter
window_size_snr: int
number of points to include in
calculating the standard deviation for the SNR
snr_threshold: float, array_like, or callable
threshold value or function for
counting a signal as robust, can be:
single value -- applies to all objects and periods
iterable -- length of number of objects, applies
each value to each object
callable -- function of period
spn_threshold: float or callable
threshold value or function for
counting a signal as robust, can be:
single value -- applies to all objects and periods
callable -- function of period
neighbor_peaks_tocheck: int
when blend is determined, number of neighbor peak periods
to check that the blended period is actually in the neighbor
max_blend_recursion: int
maximum number of blends to try and fit
out before giving up
recursion_level: int
current recursion level
nworkers: int
number of child workers
max_neighbor_amp: float
the maximum delta F / F0 amplitude a neighbor can have to
still be counted as a legitimate amplitude; not considered
for blend checking otherwise
"""
# Use the period finding function to find the best period
lsp_dict = period_finding_func(t, y, dy, **function_params)
# If no period is found at all, quit
if np.isnan(lsp_dict['bestperiod']):
if ID:
print(ID + "\n -> " + which_method + " found no period, for " +
ID)
else:
print(" -> " + which_method + " found no period.", flush=True)
return None
# Now median filter the periodogram if selected
if medianfilter:
pdgm_values = median_filtering(lsp_dict['lspvals'],
lsp_dict['periods'],
freq_window_epsilon_mf,
window_size_mf,
t[-1] - t[0],
which_method,
nworkers=nworkers)
lsp_dict['medianfilter'] = True
lsp_dict['lspvalsmf'] = pdgm_values
# Otherwise just copy periodogram values over
else:
pdgm_values = lsp_dict['lspvals']
# Check that the best period matches what period_finding_func says is best period
lsp_dict['medianfilter'] = False
if which_method == 'PDM':
per_to_comp = lsp_dict['periods'][np.argmin(pdgm_values)]
elif which_method == 'BLS': # Special handling of inf for BLS
per_to_comp = lsp_dict['periods'][np.where(np.isinf(pdgm_values),
-np.inf,
pdgm_values).argmax()]
else:
per_to_comp = lsp_dict['periods'][np.argmax(pdgm_values)]
if abs(per_to_comp -
lsp_dict['bestperiod']) / lsp_dict['bestperiod'] > 1e-7:
print(" Periods: " + str(per_to_comp) + " " +\
str(lsp_dict['bestperiod']))
raise ValueError(
"The bestperiod does not match the actual best period w/o median filtering, "
+ which_method)
# Get index for the best periodogram value
if which_method == 'PDM':
best_pdgm_index = np.argmin(pdgm_values)
elif which_method == 'BLS': # Special handling of inf for BLS
best_pdgm_index = np.where(np.isinf(pdgm_values), -np.inf,
pdgm_values).argmax()
else:
best_pdgm_index = np.argmax(pdgm_values)
# Set some values
freq_window_size = freq_window_epsilon_snr / (max(t) - min(t))
delta_frequency = abs(1. / lsp_dict['periods'][1] -
1. / lsp_dict['periods'][0])
freq_window_index_size = int(round(freq_window_size / delta_frequency))
best_freq = 1. / lsp_dict['periods'][best_pdgm_index]
# Compute periodogram SNR, compare to threshold
per_snr = snr.periodogram_snr(pdgm_values,
lsp_dict['periods'],
best_pdgm_index,
max(t) - min(t),
which_method,
freq_window_epsilon=freq_window_epsilon_snr,
rms_window_bin_size=window_size_snr)
# Print out results
if ID:
print(
"%s\n %s PERIOD: %.5e days; pSNR: %.5e" %
(ID, which_method, lsp_dict['periods'][best_pdgm_index], per_snr))
else:
print("%s PERIOD: %.5e days; pSNR: %.5e" %
(which_method, lsp_dict['periods'][best_pdgm_index], per_snr))
# Compare to the threshold, if below quit
if per_snr < snr_threshold_tocomp(
snr_threshold,
period=lsp_dict['periods'][best_pdgm_index]) or np.isnan(
lsp_dict['periods'][best_pdgm_index]):
if ID:
print(" -> pSNR not significant enough, for " + ID, flush=True)
else:
print(" -> pSNR not significant enough.", flush=True)
return None
# Check the signal-to-pink-noise if a threshold is provided
if spn_threshold is not None:
if abs(1./lsp_dict['periods'][best_pdgm_index] - 1./lsp_dict['nbestperiods'][0]) >\
.5*delta_frequency: # If the periods are different, rerun period finding to get stats
function_params_temp = copy.deepcopy(function_params)
function_params_temp['startp'] = .999999 * lsp_dict['periods'][
best_pdgm_index + 1]
function_params_temp['endp'] = lsp_dict['periods'][best_pdgm_index
- 1]
lsp_dict_temp = period_finding_func(t, y, dy,
**function_params_temp)
bls_stats_touse = lsp_dict_temp['stats'][0]
per_temp = lsp_dict_temp['nbestperiods'][0]
else:
per_temp = lsp_dict['periods'][best_pdgm_index]
bls_stats_touse = lsp_dict['stats'][0]
spn_val = pinknoise.pinknoise_calc(
t, y, dy, per_temp, bls_stats_touse['transitduration'],
bls_stats_touse['transitdepth'],
bls_stats_touse['npoints_in_transit'], bls_stats_touse['epoch'],
pinknoise.ntransits(t.min(), t.max(), bls_stats_touse['epoch'],
per_temp))
print(" SPN: %.5e" % spn_val)
if spn_val < snr_threshold_tocomp(spn_threshold, period=per_temp):
if ID:
print(" -> S/PN not significant enough, for " + ID,
flush=True)
else:
print(" -> S/PN not significant enough.", flush=True)
return None
else:
spn_val = None
# Check the Baluev FAP is a threshold is provided
if fap_baluev_threshold is not None:
fap_baluev_val = fap_baluev(t, dy,
lsp_dict['lspvals'][best_pdgm_index],
1. / lsp_dict['periods'].min())
print(" B. FAP: %.5e" % fap_baluev_val, flush=True)
if fap_baluev_val > snr_threshold_tocomp(
fap_baluev_threshold,
period=lsp_dict['periods'][best_pdgm_index]):
if ID:
print(" -> B. FAP too large, for " + ID, flush=True)
else:
print(" -> B. FAP too large.", flush=True)
return None
else:
fap_baluev_val = None
# Fit truncated Fourier series at this frequency
ff = (FourierFit(nharmonics=nharmonics_fit).fit(t, y, dy, best_freq))
this_flux_amplitude = ff.flux_amplitude()
# Fit another truncated Fourier series with more harmonics
ffr = (FourierFit(nharmonics=nharmonics_resid).fit(t, y, dy, best_freq))
# Now fit Fourier series to all the neighbor light curves
ffn_all = {}
for n_ID in neighbors.keys():
# fit neighbor's lightcurve at this frequency
ffn = (FourierFit(nharmonics=nharmonics_fit).fit(
neighbors[n_ID][0], neighbors[n_ID][1], neighbors[n_ID][2],
best_freq))
ffn_all[n_ID] = ffn
# Figure out which has maximum amplitude
max_amp = 0.
max_ffn_ID = None
significant_neighbor_blends = []
toolargeamp_neighbor_blends = []
for n_ID in ffn_all.keys():
n_flux_amp, n_df_f = ffn_all[n_ID].flux_amplitude(return_df_f0=True)
if n_df_f > max_neighbor_amp or np.isnan(n_df_f):
# Amplitude is too large, don't consider this neighbor
print(" this neighbor's flux amplitude too large: " + n_ID +
" " + str(n_flux_amp) + " " + str(n_df_f))
toolargeamp_neighbor_blends.append(n_ID)
continue
if n_flux_amp >\
results_storage_container.count_neighbor_threshold*this_flux_amplitude:
significant_neighbor_blends.append(n_ID)
if n_flux_amp > max_amp:
max_amp = n_flux_amp
max_ffn_ID = n_ID
# If neighbor has larger flux amplitude,
# then we consider this signal to be a blend.
# subtract off model signal to get residual
# lightcurve, and try again
notmax = False
if max_ffn_ID:
print(" checking blends")
print(" " + max_ffn_ID)
print(" n: " + str(ffn_all[max_ffn_ID].flux_amplitude()) +
" vs. " + str(this_flux_amplitude),
flush=True)
if ffn_all[max_ffn_ID].flux_amplitude() > this_flux_amplitude:
if this_flux_amplitude < results_storage_container.stillcount_blend_factor * ffn_all[
max_ffn_ID].flux_amplitude():
# Check that the neighbor actually has this period
if neighbor_peaks_tocheck > 0:
function_params_neighbor = copy.deepcopy(function_params)
function_params_neighbor[
'nbestpeaks'] = neighbor_peaks_tocheck
n_lsp_dict = period_finding_func(
neighbors[max_ffn_ID][0], neighbors[max_ffn_ID][1],
neighbors[max_ffn_ID][2], **function_params_neighbor)
if not any(
np.isclose(n_lsp_dict['nbestperiods'],
lsp_dict['periods'][best_pdgm_index],
rtol=1e-2,
atol=1e-5)):
# If the highest-amp blended neighbor doesn't have this period as one of its top periods
# Count as a good period
print(
" -> this isn't a peak period for the neighbor, so ignoring blend.",
flush=True)
results_storage_container.add_good_period(
lsp_dict,
t,
y,
dy,
lsp_dict['periods'][best_pdgm_index],
per_snr,
this_flux_amplitude,
significant_neighbor_blends,
ffr.params,
notmax=notmax,
s_pinknoise=spn_val,
fap_baluev=fap_baluev_val,
ignore_blend=max_ffn_ID,
toolargeamp_neighbors=toolargeamp_neighbor_blends)
return y - ffr(t)
print(" -> blended! Trying again. " + str(
ffn_all[max_ffn_ID].flux_amplitude(return_df_f0=True)[1]),
flush=True)
results_storage_container.add_blend(
lsp_dict,
t,
y,
dy,
max_ffn_ID,
lsp_dict['periods'][best_pdgm_index],
per_snr,
this_flux_amplitude,
ffn_all[max_ffn_ID].flux_amplitude(),
ffr.params,
s_pinknoise=spn_val,
fap_baluev=fap_baluev_val,
toolargeamp_neighbors=toolargeamp_neighbor_blends)
if recursion_level >= max_blend_recursion:
print(
" Reached the blend recursion level, no longer checking",
flush=True)
return None
return iterative_deblend(
t,
y - ffr(t),
dy,
neighbors,
period_finding_func,
results_storage_container,
which_method,
function_params=function_params,
nharmonics_fit=nharmonics_fit,
nharmonics_resid=nharmonics_resid,
ID=ID,
medianfilter=medianfilter,
freq_window_epsilon_mf=freq_window_epsilon_mf,
freq_window_epsilon_snr=freq_window_epsilon_snr,
window_size_mf=window_size_mf,
window_size_snr=window_size_snr,
snr_threshold=snr_threshold,
spn_threshold=spn_threshold,
fap_baluev_threshold=fap_baluev_threshold,
neighbor_peaks_tocheck=neighbor_peaks_tocheck,
max_blend_recursion=max_blend_recursion,
recursion_level=recursion_level + 1,
nworkers=nworkers,
max_neighbor_amp=max_neighbor_amp)
else:
notmax = True
# Save the period info and return the pre-whitened light curve
results_storage_container.add_good_period(
lsp_dict,
t,
y,
dy,
lsp_dict['periods'][best_pdgm_index],
per_snr,
this_flux_amplitude,
significant_neighbor_blends,
ffr.params,
notmax=notmax,
s_pinknoise=spn_val,
fap_baluev=fap_baluev_val,
toolargeamp_neighbors=toolargeamp_neighbor_blends)
return y - ffr(t)
def iterative_deblend(t,
y,
dy,
neighbors,
period_finding_func,
results_storage_container,
which_method,
function_params=None,
nharmonics_fit=5,
nharmonics_resid=10,
ID=None,
medianfilter=False,
freq_window_epsilon_mf=1.,
freq_window_epsilon_snr=1.,
window_size_mf=40,
window_size_snr=40,
snr_threshold=0.,
max_blend_recursion=8,
recursion_level=0,
nworkers=1):
"""
Iteratively deblend a lightcurve against neighbors
Parameters
----------
t: array_like
Time coordinates of lightcurve
y: array_like
Brightness measurements (in mag)
dy: array_like
Uncertainties for each brightness measurement
neighbors: list
List of (t, y, dy) lightcurves for each
neighbor (NOT including the lightcurve we are deblending)
period_finding_func: function
Function used to find the period. Output format assumed to
be the same as that used in astrobase.periodbase
results_storage_container: instance of data_processing.periodsearch_results
Used to store the results
which_method: string
Which period search method is being used
(optional from here below):
function_params: dictionary
A dictionary containing parameters for the function in
period_finding_func
nharmonics_fit: int
Number of harmonics to use in the fit (used to estimate
flux amplitude)
nharmonics_resid: int
Number of harmonics to use to obtain the residual if we
find that the signal is a blend of a neighboring signal
ID: string
ID of the object
medianfilter: boolean
whether to median filter the periodogram
freq_window_epsilon_mf: int
sets the size of the exclusion area
in the periodogram for the median filter calculation
freq_window_epsilon_snr: int
sets the size of the exclusion area
in the periodogram for the SNR calculation
window_size_mf: int
number of points to include in
calculating the median value for median filter
window_size_snr: int
number of points to include in
calculating the standard deviation for the SNR
snr_threshold=0: float, array_like, or callable
threshold value or function for
counting a signal as robust, can be:
single value -- applies to all objects and periods
iterable -- length of number of objects, applies
each value to each object
callable -- function of period
max_blend_recursion: int
maximum number of blends to try and fit
out before giving up
recursion_level: int
current recursion level
nworkers: int
number of child workers
"""
# Use the period finding function to find the best period
lsp_dict = period_finding_func(t, y, dy, **function_params)
# If no period is found at all, quit
if np.isnan(lsp_dict['bestperiod']):
if ID:
print(ID + "\n -> " + which_method + " found no period, for " +
ID)
else:
print(" -> " + which_method + " found no period.")
return None
# Now median filter the periodogram if selected
if medianfilter:
pdgm_values = median_filtering(lsp_dict['lspvals'],
lsp_dict['periods'],
freq_window_epsilon_mf,
window_size_mf,
t[-1] - t[0],
which_method,
nworkers=nworkers)
lsp_dict['medianfilter'] = True
lsp_dict['lspvalsmf'] = pdgm_values
# Otherwise just copy periodogram values over
else:
pdgm_values = lsp_dict['lspvals']
# Check that the best period matches what period_finding_func says is best period
lsp_dict['medianfilter'] = False
if which_method == 'PDM':
per_to_comp = lsp_dict['periods'][np.argmin(pdgm_values)]
else:
per_to_comp = lsp_dict['periods'][np.argmax(pdgm_values)]
if abs(per_to_comp -
lsp_dict['bestperiod']) / lsp_dict['bestperiod'] > 1e-7:
print(" Periods: " + str(per_to_comp) + " " +\
str(lsp_dict['bestperiod']))
raise ValueError(
"The bestperiod does not match the actual best period w/o median filtering, "
+ which_method)
# Get index for the best periodogram value
if which_method == 'PDM':
best_pdgm_index = np.argmin(pdgm_values)
else:
best_pdgm_index = np.argmax(pdgm_values)
# Set some values
freq_window_size = freq_window_epsilon_snr / (max(t) - min(t))
delta_frequency = abs(1. / lsp_dict['periods'][1] -
1. / lsp_dict['periods'][0])
freq_window_index_size = int(round(freq_window_size / delta_frequency))
best_freq = 1. / lsp_dict['periods'][best_pdgm_index]
# Compute periodogram SNR, compare to threshold
per_snr = snr.periodogram_snr(pdgm_values,
lsp_dict['periods'],
best_pdgm_index,
max(t) - min(t),
which_method,
freq_window_epsilon=freq_window_epsilon_snr,
rms_window_bin_size=window_size_snr)
# Print out results
if ID:
print(
"%s\n %s PERIOD: %.5e days; pSNR: %.5e" %
(ID, which_method, lsp_dict['periods'][best_pdgm_index], per_snr))
else:
print("%s PERIOD: %.5e days; pSNR: %.5e" %
(which_method, lsp_dict['periods'][best_pdgm_index], per_snr))
# Compare to the threshold, if below quit
if per_snr < snr_threshold_tocomp(
snr_threshold,
period=lsp_dict['periods'][best_pdgm_index]) or np.isnan(
lsp_dict['periods'][best_pdgm_index]):
if ID:
print(" -> not significant enough, for " + ID)
else:
print(" -> not significant enough.")
return None
# Fit truncated Fourier series at this frequency
ff = (FourierFit(nharmonics=nharmonics_fit).fit(t, y, dy, best_freq))
this_flux_amplitude = ff.flux_amplitude
# Fit another truncated Fourier series with more harmonics
ffr = (FourierFit(nharmonics=nharmonics_resid).fit(t, y, dy, best_freq))
# Now fit Fourier series to all the neighbor light curves
ffn_all = {}
for n_ID in neighbors.keys():
# fit neighbor's lightcurve at this frequency
ffn = (FourierFit(nharmonics=nharmonics_fit).fit(
neighbors[n_ID][0], neighbors[n_ID][1], neighbors[n_ID][2],
best_freq))
ffn_all[n_ID] = ffn
# Figure out which has maximum amplitude
max_amp = 0.
max_ffn_ID = None
significant_neighbor_blends = []
for n_ID in ffn_all.keys():
if ffn_all[n_ID].flux_amplitude >\
results_storage_container.count_neighbor_threshold*this_flux_amplitude:
significant_neighbor_blends.append(n_ID)
if ffn_all[n_ID].flux_amplitude > max_amp:
max_amp = ffn_all[n_ID].flux_amplitude
max_ffn_ID = n_ID
# If neighbor has larger flux amplitude,
# then we consider this signal to be a blend.
# subtract off model signal to get residual
# lightcurve, and try again
notmax = False
if max_ffn_ID:
print(" checking blends")
print(" " + max_ffn_ID)
print(" n: " + str(ffn_all[max_ffn_ID].flux_amplitude) + " vs. " +
str(this_flux_amplitude))
if ffn_all[max_ffn_ID].flux_amplitude > this_flux_amplitude:
if this_flux_amplitude < results_storage_container.stillcount_blend_factor * ffn_all[
max_ffn_ID].flux_amplitude:
print(" -> blended! Trying again.")
results_storage_container.add_blend(
lsp_dict, t, y, dy, max_ffn_ID,
snr_threshold_tocomp(
snr_threshold,
period=lsp_dict['periods'][best_pdgm_index]),
this_flux_amplitude)
if recursion_level >= max_blend_recursion:
print(
" Reached the blend recursion level, no longer checking"
)
return None
return iterative_deblend(
t,
y - ffr(t),
dy,
neighbors,
period_finding_func,
results_storage_container,
which_method,
function_params=function_params,
nharmonics_fit=nharmonics_fit,
nharmonics_resid=nharmonics_resid,
ID=ID,
medianfilter=medianfilter,
freq_window_epsilon_mf=freq_window_epsilon_mf,
freq_window_epsilon_snr=freq_window_epsilon_snr,
window_size_mf=window_size_mf,
window_size_snr=window_size_snr,
snr_threshold=snr_threshold_tocomp(
snr_threshold,
period=lsp_dict['periods'][best_pdgm_index]),
max_blend_recursion=max_blend_recursion,
recursion_level=recursion_level + 1,
nworkers=nworkers)
else:
notmax = True
# Save the period info and return the pre-whitened light curve
results_storage_container.add_good_period(
lsp_dict,
t,
y,
dy,
snr_threshold_tocomp(snr_threshold,
period=lsp_dict['periods'][best_pdgm_index]),
this_flux_amplitude,
significant_neighbor_blends,
notmax=notmax)
return y - ffr(t)