def model(params):
a = params[0] ** 2 + T_small ** 2
i = -(params[1] ** 2 + min_index)
t = params[2] ** 2 + T_small ** 2
spec = npow.overf_power_spectrum(a, i, f_0, dt, n_time)
spec += t
spec = npow.convolve_power(spec, window)
spec = npow.prune_power(spec)
spec = spec[1:].real
return spec
def model(params):
a = params[0]**2 + T_small**2
i = -(params[1]**2 + min_index)
t = params[2]**2 + T_small**2
spec = npow.overf_power_spectrum(a, i, f_0, dt, n_time)
spec += t
spec = npow.convolve_power(spec, window)
spec = npow.prune_power(spec)
spec = spec[1:].real
return spec
def get_freq_modes_over_f(power_mat, window_function, frequency, n_modes, plots=False):
"""Fines the most correlated frequency modes and fits thier noise."""
n_f = len(frequency)
d_f = sp.mean(sp.diff(frequency))
dt = 1.0 / 2.0 / frequency[-1]
n_chan = power_mat.shape[-1]
n_time = window_function.shape[0]
# The threshold for assuming there isn't enough data to measure anything.
no_data_thres = 10.0 / n_time
# Initialize the dictionary that will hold all the parameters.
output_params = {}
# First take the low frequency part of the spetrum matrix and average over
# enough bins to get a well conditioned matrix.
low_f_mat = sp.mean(power_mat[: 4 * n_chan, :, :].real, 0)
# Factor the matrix to get the most correlated modes.
e, v = linalg.eigh(low_f_mat)
# Make sure they are sorted.
if not sp.alltrue(sp.diff(e) >= 0):
raise RuntimeError("Eigenvalues not sorted")
# Power matrix striped of the biggest modes.
reduced_power = sp.copy(power_mat)
mode_list = []
# Solve for the spectra of these modes.
for ii in range(n_modes):
this_mode_params = {}
# Get power spectrum and window function for this mode.
mode = v[:, -1 - ii]
mode_power = sp.sum(mode * power_mat.real, -1)
mode_power = sp.sum(mode * mode_power, -1)
mode_window = sp.sum(mode[:, None] ** 2 * window_function, 1)
mode_window = sp.sum(mode_window * mode[None, :] ** 2, 1)
# Protect against no data.
if sp.mean(mode_window).real < no_data_thres:
this_mode_params["amplitude"] = 0.0
this_mode_params["index"] = 0.0
this_mode_params["f_0"] = 1.0
this_mode_params["thermal"] = T_infinity ** 2 * dt
else:
# Fit the spectrum.
p = fit_overf_const(mode_power, mode_window, frequency)
# Put all the parameters we measured into the output.
this_mode_params["amplitude"] = p[0]
this_mode_params["index"] = p[1]
this_mode_params["f_0"] = p[2]
this_mode_params["thermal"] = p[3]
this_mode_params["mode"] = mode
output_params["over_f_mode_" + str(ii)] = this_mode_params
# Remove the mode from the power matrix.
tmp_amp = sp.sum(reduced_power * mode, -1)
tmp_amp2 = sp.sum(reduced_power * mode[:, None], -2)
tmp_amp3 = sp.sum(tmp_amp2 * mode, -1)
reduced_power -= tmp_amp[:, :, None] * mode
reduced_power -= tmp_amp2[:, None, :] * mode[:, None]
reduced_power += tmp_amp3[:, None, None] * mode[:, None] * mode
mode_list.append(mode)
# Initialize the compensation matrix, that will be used to restore thermal
# noise that gets subtracted out. See Jan 29, Feb 17th, 2012 of Kiyo's
# notes.
compensation = sp.eye(n_chan, dtype=float)
for mode1 in mode_list:
compensation.flat[:: n_chan + 1] -= 2 * mode1 ** 2
for mode2 in mode_list:
mode_prod = mode1 * mode2
compensation += mode_prod[:, None] * mode_prod[None, :]
# Now that we've striped the noisiest modes, measure the auto power
# spectrum, averaged over channels.
auto_spec_mean = reduced_power.view()
auto_spec_mean.shape = (n_f, n_chan ** 2)
auto_spec_mean = auto_spec_mean[:, :: n_chan + 1].real
auto_spec_mean = sp.mean(auto_spec_mean, -1)
diag_window = window_function.view()
diag_window.shape = (n_time, n_chan ** 2)
diag_window = diag_window[:, :: n_chan + 1]
auto_spec_window = sp.mean(diag_window, -1)
if sp.mean(auto_spec_window).real < no_data_thres:
auto_cross_over = 0.0
auto_index = 0.0
auto_thermal = 0
else:
auto_spec_params = fit_overf_const(auto_spec_mean, auto_spec_window, frequency)
auto_thermal = auto_spec_params[3]
if auto_spec_params[0] <= 0 or auto_spec_params[3] <= 0 or auto_spec_params[1] > -0.599:
auto_cross_over = 0.0
auto_index = 0.0
else:
auto_index = auto_spec_params[1]
auto_cross_over = auto_spec_params[2] * (auto_spec_params[0] / auto_spec_params[3]) ** (-1.0 / auto_index)
# if auto_cross_over < d_f:
# auto_index = 0.
# auto_cross_over = 0.
# Plot the mean auto spectrum if desired.
if plots:
h = plt.gcf()
a = h.add_subplot(*h.current_subplot)
norm = sp.mean(auto_spec_window).real
auto_plot = auto_spec_mean / norm
plotable = auto_plot > 0
lines = a.loglog(frequency[plotable], auto_plot[plotable])
c = lines[-1].get_color()
# And plot the fit in a light color.
if auto_cross_over > d_f / 4.0:
spec = npow.overf_power_spectrum(auto_thermal, auto_index, auto_cross_over, dt, n_time)
else:
spec = sp.zeros(n_time, dtype=float)
spec += auto_thermal
spec[0] = 0
spec = npow.convolve_power(spec, auto_spec_window)
spec = npow.prune_power(spec)
spec = spec[1:].real
if norm > no_data_thres:
spec /= norm
plotable = spec > 0
a.loglog(frequency[plotable], spec[plotable], c=c, alpha=0.4, linestyle=":")
output_params["all_channel_index"] = auto_index
output_params["all_channel_corner_f"] = auto_cross_over
# Finally measure the thermal part of the noise in each channel.
cross_over_ind = sp.digitize([auto_cross_over * 4], frequency)[0]
cross_over_ind = max(cross_over_ind, n_f // 2)
cross_over_ind = min(cross_over_ind, int(9.0 * n_f / 10.0))
thermal = reduced_power[cross_over_ind:, :, :].real
n_high_f = thermal.shape[0]
thermal.shape = (n_high_f, n_chan ** 2)
thermal = sp.mean(thermal[:, :: n_chan + 1], 0)
thermal_norms = sp.mean(diag_window, 0).real
bad_inds = thermal_norms < no_data_thres
thermal_norms[bad_inds] = 1.0
# Compensate for power lost in mode subtraction.
compensation[:, bad_inds] = 0
compensation[bad_inds, :] = 0
for ii in xrange(n_chan):
if bad_inds[ii]:
compensation[ii, ii] = 1.0
thermal = linalg.solve(compensation, thermal)
# Normalize
thermal /= thermal_norms
thermal[bad_inds] = T_infinity ** 2 * dt
# Occationally the compensation fails horribly on a few channels.
# When this happens, zero out the offending indices.
thermal[thermal < 0] = 0
output_params["thermal"] = thermal
# Now that we know what thermal is, we can subtract it out of the modes we
# already measured.
for ii in range(n_modes):
mode_params = output_params["over_f_mode_" + str(ii)]
thermal_contribution = sp.sum(mode_params["mode"] ** 2 * thermal)
# Subtract a maximum of 90% of the white noise to keep things positive
# definate.
new_white = max(mode_params["thermal"] - thermal_contribution, 0.1 * mode_params["thermal"])
if mode_params["thermal"] < 0.5 * T_infinity ** 2 * dt:
mode_params["thermal"] = new_white
return output_params
def get_freq_modes_over_f(power_mat,
window_function,
frequency,
n_modes,
plots=False):
"""Fines the most correlated frequency modes and fits thier noise."""
n_f = len(frequency)
d_f = sp.mean(sp.diff(frequency))
dt = 1. / 2. / frequency[-1]
n_chan = power_mat.shape[-1]
n_time = window_function.shape[0]
# The threshold for assuming there isn't enough data to measure anything.
no_data_thres = 10. / n_time
# Initialize the dictionary that will hold all the parameters.
output_params = {}
# First take the low frequency part of the spetrum matrix and average over
# enough bins to get a well conditioned matrix.
low_f_mat = sp.mean(power_mat[:4 * n_chan, :, :].real, 0)
# Factor the matrix to get the most correlated modes.
e, v = linalg.eigh(low_f_mat)
# Make sure they are sorted.
if not sp.alltrue(sp.diff(e) >= 0):
raise RuntimeError("Eigenvalues not sorted")
# Power matrix striped of the biggest modes.
reduced_power = sp.copy(power_mat)
mode_list = []
# Solve for the spectra of these modes.
for ii in range(n_modes):
this_mode_params = {}
# Get power spectrum and window function for this mode.
mode = v[:, -1 - ii]
mode_power = sp.sum(mode * power_mat.real, -1)
mode_power = sp.sum(mode * mode_power, -1)
mode_window = sp.sum(mode[:, None]**2 * window_function, 1)
mode_window = sp.sum(mode_window * mode[None, :]**2, 1)
# Protect against no data.
if sp.mean(mode_window).real < no_data_thres:
this_mode_params['amplitude'] = 0.
this_mode_params['index'] = 0.
this_mode_params['f_0'] = 1.
this_mode_params['thermal'] = T_infinity**2 * dt
else:
# Fit the spectrum.
p = fit_overf_const(mode_power, mode_window, frequency)
# Put all the parameters we measured into the output.
this_mode_params['amplitude'] = p[0]
this_mode_params['index'] = p[1]
this_mode_params['f_0'] = p[2]
this_mode_params['thermal'] = p[3]
this_mode_params['mode'] = mode
output_params['over_f_mode_' + str(ii)] = this_mode_params
# Remove the mode from the power matrix.
tmp_amp = sp.sum(reduced_power * mode, -1)
tmp_amp2 = sp.sum(reduced_power * mode[:, None], -2)
tmp_amp3 = sp.sum(tmp_amp2 * mode, -1)
reduced_power -= tmp_amp[:, :, None] * mode
reduced_power -= tmp_amp2[:, None, :] * mode[:, None]
reduced_power += tmp_amp3[:, None, None] * mode[:, None] * mode
mode_list.append(mode)
# Initialize the compensation matrix, that will be used to restore thermal
# noise that gets subtracted out. See Jan 29, Feb 17th, 2012 of Kiyo's
# notes.
compensation = sp.eye(n_chan, dtype=float)
for mode1 in mode_list:
compensation.flat[::n_chan + 1] -= 2 * mode1**2
for mode2 in mode_list:
mode_prod = mode1 * mode2
compensation += mode_prod[:, None] * mode_prod[None, :]
# Now that we've striped the noisiest modes, measure the auto power
# spectrum, averaged over channels.
auto_spec_mean = reduced_power.view()
auto_spec_mean.shape = (n_f, n_chan**2)
auto_spec_mean = auto_spec_mean[:, ::n_chan + 1].real
auto_spec_mean = sp.mean(auto_spec_mean, -1)
diag_window = window_function.view()
diag_window.shape = (n_time, n_chan**2)
diag_window = diag_window[:, ::n_chan + 1]
auto_spec_window = sp.mean(diag_window, -1)
if sp.mean(auto_spec_window).real < no_data_thres:
auto_cross_over = 0.
auto_index = 0.
auto_thermal = 0
else:
auto_spec_params = fit_overf_const(auto_spec_mean, auto_spec_window,
frequency)
auto_thermal = auto_spec_params[3]
if (auto_spec_params[0] <= 0 or auto_spec_params[3] <= 0
or auto_spec_params[1] > -0.599):
auto_cross_over = 0.
auto_index = 0.
else:
auto_index = auto_spec_params[1]
auto_cross_over = auto_spec_params[2] * (
auto_spec_params[0] / auto_spec_params[3])**(-1. / auto_index)
#if auto_cross_over < d_f:
# auto_index = 0.
# auto_cross_over = 0.
# Plot the mean auto spectrum if desired.
if plots:
h = plt.gcf()
a = h.add_subplot(*h.current_subplot)
norm = sp.mean(auto_spec_window).real
auto_plot = auto_spec_mean / norm
plotable = auto_plot > 0
lines = a.loglog(frequency[plotable], auto_plot[plotable])
c = lines[-1].get_color()
# And plot the fit in a light color.
if auto_cross_over > d_f / 4.:
spec = npow.overf_power_spectrum(auto_thermal, auto_index,
auto_cross_over, dt, n_time)
else:
spec = sp.zeros(n_time, dtype=float)
spec += auto_thermal
spec[0] = 0
spec = npow.convolve_power(spec, auto_spec_window)
spec = npow.prune_power(spec)
spec = spec[1:].real
if norm > no_data_thres:
spec /= norm
plotable = spec > 0
a.loglog(frequency[plotable],
spec[plotable],
c=c,
alpha=0.4,
linestyle=':')
output_params['all_channel_index'] = auto_index
output_params['all_channel_corner_f'] = auto_cross_over
# Finally measure the thermal part of the noise in each channel.
cross_over_ind = sp.digitize([auto_cross_over * 4], frequency)[0]
cross_over_ind = max(cross_over_ind, n_f // 2)
cross_over_ind = min(cross_over_ind, int(9. * n_f / 10.))
thermal = reduced_power[cross_over_ind:, :, :].real
n_high_f = thermal.shape[0]
thermal.shape = (n_high_f, n_chan**2)
thermal = sp.mean(thermal[:, ::n_chan + 1], 0)
thermal_norms = sp.mean(diag_window, 0).real
bad_inds = thermal_norms < no_data_thres
thermal_norms[bad_inds] = 1.
# Compensate for power lost in mode subtraction.
compensation[:, bad_inds] = 0
compensation[bad_inds, :] = 0
for ii in xrange(n_chan):
if bad_inds[ii]:
compensation[ii, ii] = 1.
thermal = linalg.solve(compensation, thermal)
# Normalize
thermal /= thermal_norms
thermal[bad_inds] = T_infinity**2 * dt
# Occationally the compensation fails horribly on a few channels.
# When this happens, zero out the offending indices.
thermal[thermal < 0] = 0
output_params['thermal'] = thermal
# Now that we know what thermal is, we can subtract it out of the modes we
# already measured.
for ii in range(n_modes):
mode_params = output_params['over_f_mode_' + str(ii)]
thermal_contribution = sp.sum(mode_params['mode']**2 * thermal)
# Subtract a maximum of 90% of the white noise to keep things positive
# definate.
new_white = max(mode_params['thermal'] - thermal_contribution,
0.1 * mode_params['thermal'])
if mode_params['thermal'] < 0.5 * T_infinity**2 * dt:
mode_params['thermal'] = new_white
return output_params