def fit_single_line(self, x, y, zero_lev, err_continuum, fitting_parameters, bootstrap_iterations = 1000):
#Simple fit
if self.fit_dict['MC_iterations'] == 1:
fit_output = lmfit_minimize(residual_gauss, fitting_parameters, args=(x, y, zero_lev, err_continuum))
self.fit_dict['area_intg'] = simps(y, x) - simps(zero_lev, x)
self.fit_dict['area_intg_err'] = 0.0
#Bootstrap
else:
mini_posterior = Minimizer(lnprob_gaussCurve, fitting_parameters, fcn_args = ([x, y, zero_lev, err_continuum]))
fit_output = mini_posterior.emcee(steps=200, params = fitting_parameters)
#Bootstrap for the area of the lines
area_array = empty(bootstrap_iterations)
len_x_array = len(x)
for i in range(bootstrap_iterations):
y_new = y + np_normal_dist(0.0, err_continuum, len_x_array)
area_array[i] = simps(y_new, x) - simps(zero_lev, x)
self.fit_dict['area_intg'] = mean(area_array)
self.fit_dict['area_intg_err'] = std(area_array)
#Store the fitting parameters
output_params = fit_output.params
for key in self.fit_dict['parameters_list']:
self.fit_dict[key + '_norm'] = output_params[key].value
self.fit_dict[key + '_norm_er'] = output_params[key].stderr
return
def fit_parameters(self, error_func):
assert error_func == residuals_error, "right now we only know about lmfit miniumize, so we need to use residuals error"
gamma1 = (1 / 10 + 1 / 5) / 2
gamma2 = 0.02
gamma3 = 0.02
beta = 0.14
num_cumulative_positiv = np.sum(self._observations[:, 0])
print(f"num_cumulative_positiv {num_cumulative_positiv}")
print(self._observations[:, 0])
tau = 28 / num_cumulative_positiv
delta = 5 / 18
params = Parameters()
params.add('gamma1', value=gamma1, min=1 / 10, max=1 / 5)
params.add('gamma2', value=gamma2, min=0, max=5)
params.add('gamma3', value=gamma3, min=0, max=5)
params.add('beta', value=beta, min=0.01, max=0.5)
params.add('tau', value=tau, min=tau * 0.8, max=tau * 1.2)
params.add('delta', value=delta, min=0.6 * delta, max=1.4 * delta)
# Attention ! We use leastsq
# so the objective function must
# return an array ith the residuals.
result = lmfit_minimize(self._plumb,
params,
args=(len(self._observations), error_func),
method='leastsq')
report_fit(result)
self._fit_params = result.params
def fit_blended_line_BS_lmfit(self, x, y, zero_lev, err_continuum, Ncomps, fitting_parameters, add_wide_component, fitting_parameters_wide, bootstrap_iterations = 500, MC_iterations = 200):
#Prepare some data in advance
n_points = len(x)
n_parameters = len(self.fit_dict['parameters_list'])
params_list = self.fit_dict['parameters_list']
range_param = range(n_parameters)
range_boots = range(bootstrap_iterations)
area_array = empty(bootstrap_iterations)
idcs_components = map(str, range(Ncomps))
params_matrix = empty((n_parameters, bootstrap_iterations))
errors_matrix = empty((n_parameters, bootstrap_iterations))
Ncomps_wide = ['3']
#Loop through the iterations:
for i in range_boots:
y_new = y + np_normal_dist(0.0, err_continuum, n_points)
area_array[i] = simps(y_new, x) - simps(zero_lev, x)
fit_Output = lmfit_minimize(residual_gaussMix, fitting_parameters, args=(x, y_new, zero_lev, err_continuum, idcs_components))
output_params = fit_Output.params
#Case with a wide component
if add_wide_component:
#Only works for Halpha
sigma_limit = output_params['sigma1'].value
limit_0, limit_1 = 6548.05 - self.fit_dict['x_scaler'] - sigma_limit * 1.5, 6548.05 - self.fit_dict['x_scaler'] + sigma_limit * 1.5
limit_2, limit_3 = 0 - sigma_limit * 4, 0 + sigma_limit * 4
limit_4, limit_5 = 6583.46 - self.fit_dict['x_scaler'] - sigma_limit * 3, 6583.46 - self.fit_dict['x_scaler'] + sigma_limit * 3
indeces = ((x >= limit_0) & (x <= limit_1)) + ((x >= limit_2) & (x <= limit_3)) + ((x >= limit_4) & (x <= limit_5))
mask = invert(indeces)
x_wide, y_wide, zero_wide = x[mask], y[mask], zero_lev[mask]
#Fit the wide component without narrow component
fit_output_wide = lmfit_minimize(residual_gaussMix, fitting_parameters_wide, args=(x_wide, y_wide, zero_wide, err_continuum, Ncomps_wide))
output_params_wide = fit_output_wide.params
#Get wide curve
y_wide = gaussian_mixture(output_params_wide.valuesdict(), x, zero_lev, Ncomps_wide)
#Calculate emission line region again
y_pure_narrow = y - y_wide + zero_lev
#Fit narrow components again
fit_Output = lmfit_minimize(residual_gaussMix, fitting_parameters, args=(x, y_pure_narrow, zero_lev, err_continuum, idcs_components))
out_params_narrow = fit_Output.params
#Combine the results from both fits
output_params = out_params_narrow + output_params_wide
#save parameters to array
for j in range_param:
params_matrix[j,i] = output_params[params_list[j]].value
errors_matrix[j,i] = output_params[params_list[j]].stderr
#Calculate mean and std deviation values
mean_area, std_area = mean(area_array), std(area_array)
mean_params_array, stdev_params_array = params_matrix.mean(1), params_matrix.std(1)
errorsmean_array = errors_matrix.mean(1)
#Return the data to a dictionary format
self.fit_dict['area_intg'], self.fit_dict['area_intg_er'] = mean_area, std_area
for j in range(len(self.fit_dict['parameters_list'])):
param = self.fit_dict['parameters_list'][j]
self.fit_dict[param+'_norm'] = mean_params_array[j]
self.fit_dict[param+'_norm_er'] = errorsmean_array[j]
#Increase the number of components if wide observed
self.fit_dict.line_number = self.fit_dict.line_number + 1 if add_wide_component else self.fit_dict.line_number
return
def AGD_double(
vel,
data,
vel_em,
data_em,
errors,
errors_em,
alpha1=None,
alpha2=None,
alpha_em=None,
wiggle=0.1,
drop_width=3,
min_dv=0,
mode="python",
verbose=False,
SNR_thresh=5.0,
BLFrac=0.1,
SNR2_thresh=5.0,
SNR_em=5.0,
deblend=True,
perform_final_fit=True,
phase="two",
):
""" Autonomous Gaussian Decomposition "hybrid" Method
Allows for simultaneous decomposition of 21cm absorption and emission
New free parameters, in addition to the alpha parameters and
signal to noise ratios for standard one- or two-phase AGD fit, include:
alpha_em: regularization paramter for the fit to emission (either
the residuals of the absorption ,or the full emission
spectrum in the absence of detected absorption components)
Default: None
wiggle: factor by which parameters from the absorption fit (i.e.,
the Gaussian parameters of components detected in
absorption) are allowed to vary in the fit to emission.
Default: 10%
drop_width: if a component is found in the fit to emission and its mean
position is within +/- drop_width (defind in channels)
from the mean position of any absorption component, the
emission component is dropped from the fit.
Default: 3 channels
min_dv: minimum width of components fit in the emission-only fit.
Limits AGD from fitting unrealistically narrow components
in emission which, if real, shoud have been detected in
absorption.
Default: 0 (i.e., no constraint)
"""
if type(SNR2_thresh) != type([]):
SNR2_thresh = [SNR2_thresh, SNR2_thresh]
if type(SNR_thresh) != type([]):
SNR_thresh = [SNR_thresh, SNR_thresh]
say("\n --> AGD() \n", verbose)
if (not alpha2) and (phase == "two"):
print("alpha2 value required")
return
dv = np.abs(vel[1] - vel[0])
v_to_i = interp1d(vel, np.arange(len(vel)))
# --------------------------------------#
# Find phase-one guesses #
# --------------------------------------#
agd1 = initialGuess(
vel,
data,
errors=None,
alpha=alpha1,
# plot=plot,
mode=mode,
verbose=verbose,
SNR_thresh=SNR_thresh[0],
BLFrac=BLFrac,
SNR2_thresh=SNR2_thresh[0],
deblend=deblend,
)
amps_g1, widths_g1, offsets_g1, u2 = (
agd1["amps"],
agd1["FWHMs"],
agd1["means"],
agd1["u2"],
)
params_g1 = np.append(np.append(amps_g1, widths_g1), offsets_g1)
ncomps_g1 = len(params_g1) // 3
ncomps_g2 = 0 # Default
ncomps_f1 = 0 # Default
# ----------------------------#
# Find phase-two guesses #
# ----------------------------#
if phase == "two":
say("Beginning phase-two AGD... ", verbose)
ncomps_g2 = 0
# ----------------------------------------------------------#
# Produce the residual signal #
# -- Either the original data, or intermediate subtraction #
# ----------------------------------------------------------#
if ncomps_g1 == 0:
say(
"Phase 2 with no narrow comps -> No intermediate subtration... ",
verbose,
)
residuals = data
else:
# "Else" Narrow components were found, and Phase == 2, so perform intermediate subtraction...
# The "fitmask" is a collection of windows around the a list of phase-one components
fitmask, fitmaskw = create_fitmask(len(vel), v_to_i(offsets_g1),
widths_g1 / dv / 2.355 * 0.9)
notfitmask = 1 - fitmask
# Error function for intermediate optimization
def objectiveD2_leastsq(paramslm):
params = vals_vec_from_lmfit(paramslm)
model0 = func(vel, *params)
model2 = np.diff(np.diff(model0.ravel())) / dv / dv
resids1 = fitmask[1:-1] * (model2 - u2[1:-1]) / errors[1:-1]
resids2 = notfitmask * (model0 - data) / errors / 10.0
return np.append(resids1, resids2)
# Perform the intermediate fit using LMFIT
t0 = time.time()
say("Running LMFIT on initial narrow components...", verbose)
lmfit_params = paramvec_to_lmfit(params_g1)
result = lmfit_minimize(objectiveD2_leastsq,
lmfit_params,
method="leastsq")
params_f1 = vals_vec_from_lmfit(result.params)
ncomps_f1 = len(params_f1) // 3
# # Make "FWHMS" positive
# params_f1[0:ncomps_f1][params_f1[0:ncomps_f1] < 0.0] = (
# -1 * params_f1[0:ncomps_f1][params_f1[0:ncomps_f1] < 0.0]
# )
del lmfit_params
say("LMFIT fit took {0} seconds.".format(time.time() - t0))
if result.success:
# Compute intermediate residuals
# Median filter on 2x effective scale to remove poor subtractions of strong components
intermediate_model = func(
vel, *params_f1).ravel() # Explicit final (narrow) model
median_window = 2.0 * 10**((np.log10(alpha1) + 2.187) / 3.859)
residuals = median_filter(data - intermediate_model,
np.int(median_window))
else:
residuals = data
# Finished producing residual signal # ---------------------------
# Search for phase-two guesses
agd2 = initialGuess(
vel,
residuals,
errors=None,
alpha=alpha2,
mode=mode,
verbose=verbose,
SNR_thresh=SNR_thresh[1],
BLFrac=BLFrac,
SNR2_thresh=SNR2_thresh[1], # June 9 2014, change
deblend=deblend,
# plot=plot,
)
ncomps_g2 = agd2["N_components"]
if ncomps_g2 > 0:
params_g2 = np.concatenate(
[agd2["amps"], agd2["FWHMs"], agd2["means"]])
else:
params_g2 = []
u22 = agd2["u2"]
# END PHASE 2 <<<
# Check for phase two components, make final guess list
# ------------------------------------------------------
if phase == "two" and (ncomps_g2 > 0):
amps_gf = np.append(params_g1[0:ncomps_g1], params_g2[0:ncomps_g2])
widths_gf = np.append(params_g1[ncomps_g1:2 * ncomps_g1],
params_g2[ncomps_g2:2 * ncomps_g2])
offsets_gf = np.append(
params_g1[2 * ncomps_g1:3 * ncomps_g1],
params_g2[2 * ncomps_g2:3 * ncomps_g2],
)
params_gf = np.concatenate([amps_gf, widths_gf, offsets_gf])
ncomps_gf = len(params_gf) // 3
else:
params_gf = params_g1
ncomps_gf = len(params_gf) // 3
# Sort final guess list by amplitude
# ----------------------------------
say("N final parameter guesses: " + str(ncomps_gf))
amps_temp = params_gf[0:ncomps_gf]
widths_temp = params_gf[ncomps_gf:2 * ncomps_gf]
offsets_temp = params_gf[2 * ncomps_gf:3 * ncomps_gf]
w_sort_amp = np.argsort(amps_temp)[::-1]
params_gf = np.concatenate([
amps_temp[w_sort_amp], widths_temp[w_sort_amp],
offsets_temp[w_sort_amp]
])
ncomps_fit = ncomps_gf
params_fit = params_gf
if perform_final_fit:
say("\n\n --> Final Fitting... \n", verbose)
if ncomps_gf > 0:
# Objective functions for final fit
def objective_leastsq(paramslm):
params = vals_vec_from_lmfit(paramslm)
resids = (func(vel, *params).ravel() - data.ravel()) / errors
return resids
# Final fit using unconstrained parameters
t0 = time.time()
lmfit_params = paramvec_to_lmfit(params_gf)
result2 = lmfit_minimize(objective_leastsq,
lmfit_params,
method="leastsq")
params_fit = vals_vec_from_lmfit(result2.params)
params_errs = errs_vec_from_lmfit(result2.params)
ncomps_fit = len(params_fit) // 3
del lmfit_params
say("Final fit took {0} seconds.".format(time.time() - t0),
verbose)
# # Make "FWHMS" positive
# params_fit[0:ncomps_fit][params_fit[0:ncomps_fit] < 0.0] = (
# -1 * params_fit[0:ncomps_fit][params_fit[0:ncomps_fit] < 0.0]
# )
best_fit_final = func(vel, *params_fit).ravel()
rchi2 = np.sum((data - best_fit_final)**2 / errors**2) / len(data)
# Check if any amplitudes are identically zero, if so, remove them.
amps_fit = np.array(params_fit[0:ncomps_fit], dtype=float)
fwhms_fit = np.array(params_fit[ncomps_fit:2 * ncomps_fit],
dtype=float)
offsets_fit = np.array(params_fit[2 * ncomps_fit:3 * ncomps_fit],
dtype=float)
w_keep = amps_fit > 0.0
params_fit = np.concatenate(
[amps_fit[w_keep], fwhms_fit[w_keep], offsets_fit[w_keep]])
ncomps_fit = len(params_fit) // 3
amps_fit = np.array(params_fit[0:ncomps_fit], dtype=float)
fwhms_fit = np.array(params_fit[ncomps_fit:2 * ncomps_fit],
dtype=float)
offsets_fit = np.array(params_fit[2 * ncomps_fit:3 * ncomps_fit],
dtype=float)
w_keep = offsets_fit < np.max(vel)
params_fit = np.concatenate(
[amps_fit[w_keep], fwhms_fit[w_keep], offsets_fit[w_keep]])
fwhms_fit = np.abs(fwhms_fit[w_keep])
ncomps_fit = len(params_fit) // 3
amps_fit = np.array(params_fit[0:ncomps_fit], dtype=float)
fwhms_fit = np.array(params_fit[ncomps_fit:2 * ncomps_fit],
dtype=float)
offsets_fit = np.array(params_fit[2 * ncomps_fit:3 * ncomps_fit],
dtype=float)
w_keep = offsets_fit > np.min(vel)
params_fit = np.concatenate(
[amps_fit[w_keep], fwhms_fit[w_keep], offsets_fit[w_keep]])
fwhms_fit = np.abs(fwhms_fit[w_keep])
ncomps_fit = len(params_fit) // 3
amps_fit = np.array(params_fit[0:ncomps_fit], dtype=float)
fwhms_fit = np.array(params_fit[ncomps_fit:2 * ncomps_fit],
dtype=float)
offsets_fit = np.array(params_fit[2 * ncomps_fit:3 * ncomps_fit],
dtype=float)
w_keep = fwhms_fit > dv
params_fit = np.concatenate(
[amps_fit[w_keep], fwhms_fit[w_keep], offsets_fit[w_keep]])
ncomps_fit = len(params_fit) // 3
else:
best_fit_final = np.zeros(len(vel))
rchi2 = -999.0
ncomps_fit = ncomps_gf
# Construct output dictionary (odict)
# -----------------------------------
odict = {}
odict["initial_parameters"] = params_gf
odict["N_components"] = ncomps_fit
if (perform_final_fit) and (ncomps_fit > 0):
odict["best_fit_parameters"] = params_fit
odict["best_fit_errors"] = params_errs
odict["rchi2"] = rchi2
# ------ ADDING SUBSEQUENT FIT FOR EMISSION-ONLY COMPONENTS ---------
# -- Find initial guesses for fit of absorption components to emission
# -- Based on simple least-squares fit of absorption lines to emission
# -- **Holding width and mean velocity constant for absorption lines
# -- **Fitting only for amplitude (i.e., spin temperature)
vel = vel_em
data = data_em
errors = errors_em
dv = np.abs(vel[1] - vel[0])
v_to_i = interp1d(vel, np.arange(len(vel)))
params_em = []
params_em_errs = []
ncomps_em = 0
if ncomps_fit > 0:
# Objective functions for fit
def objective_leastsq(paramslm):
params = vals_vec_from_lmfit(paramslm)
resids = (func(vel, *params).ravel() - data.ravel()) / errors
return resids
params_full = np.concatenate(
[params_fit,
np.ones(ncomps_fit), params_fit[0:ncomps_fit]])
# Final fit using constrained parameters
t0 = time.time()
lmfit_params = paramvec_p3_to_lmfit(params_full, wiggle, min_dv)
result_em = lmfit_minimize(objective_leastsq,
lmfit_params,
method="leastsq")
params_em = vals_vec_from_lmfit(result_em.params)
params_em_errs = errs_vec_from_lmfit(result_em.params)
ncomps_em = len(params_em) // 3
# The "fitmask" is a collection of windows around the a list of two-phase absorption components
fitmask, fitmaskw = create_fitmask(
len(vel),
v_to_i(params_em[2 * ncomps_em:3 * ncomps_em]),
params_em[ncomps_em:2 * ncomps_em] / dv / 2.355 * 0.9,
)
notfitmask = 1 - fitmask
# notfitmaskw = np.logical_not(fitmaskw)
# Compute intermediate residuals
# Median filter on 2 x effective scale to remove poor subtractions of strong components
intermediate_model = func(
vel, *params_em).ravel() # Explicit final (narrow) model
median_window = 2.0 * 10**((np.log10(alpha_em) + 2.187) / 3.859)
residuals = median_filter(data - intermediate_model,
np.int(median_window))
else:
residuals = data
# Finished producing residual emission signal # ---------------------------
# Search for phase-three guesses in residual emission spectrum
agd3 = initialGuess(
vel,
residuals,
errors,
alpha=alpha_em,
mode=mode,
verbose=verbose,
SNR_thresh=SNR_em,
BLFrac=BLFrac,
SNR2_thresh=SNR_thresh[0],
deblend=deblend,
)
ncomps_g3 = agd3["N_components"]
if ncomps_g3 > 0:
params_g3 = np.concatenate(
[agd3["amps"], agd3["FWHMs"], agd3["means"]])
else:
params_g3 = []
u22 = agd3["u2"]
# Check for phase three components, make final guess list
# ------------------------------------------------------
if ncomps_g3 > 0:
abs_offsets = np.array(params_em[2 * ncomps_em:3 * ncomps_em],
dtype=float)
em_widths = np.array(params_g3[ncomps_g3:2 * ncomps_g3], dtype=float)
em_amps = np.array(params_g3[0:ncomps_g3], dtype=float)
em_offsets = np.array(params_g3[2 * ncomps_g3:3 * ncomps_g3],
dtype=float)
indices = []
# Check if any emission components are within 3 channels of an absorption component
if ncomps_fit > 0:
for i, offset in enumerate(em_offsets):
drop_comp = False
for abs_offset in abs_offsets:
if np.abs(abs_offset - offset) < drop_width * dv:
# print(abs_offset, offset, 3.0 * dv)
drop_comp = True
if not drop_comp:
indices.append(i)
em_offsets = em_offsets[indices]
em_amps = em_amps[indices]
em_widths = em_widths[indices]
ncomps_g3 = len(em_amps)
# print("ncomps_em", ncomps_em)
amps_emf = np.append(params_em[0:ncomps_em], em_amps)
widths_emf = np.append(params_em[ncomps_em:2 * ncomps_em],
em_widths)
offsets_emf = np.append(params_em[2 * ncomps_em:3 * ncomps_em],
em_offsets)
tau_emf = np.append(params_fit[0:ncomps_em], np.zeros(ncomps_g3))
labels_emf = np.append(np.ones(ncomps_em), np.zeros(ncomps_g3))
# print("labels", labels_emf)
else:
amps_emf = em_amps
widths_emf = em_widths
offsets_emf = em_offsets
tau_emf = np.zeros(ncomps_g3)
labels_emf = np.zeros(ncomps_g3)
params_emf = np.concatenate([amps_emf, widths_emf, offsets_emf])
ncomps_emf = len(params_emf) // 3
else:
params_emf = params_em
ncomps_emf = ncomps_em
if ncomps_em > 0:
tau_emf = params_fit[0:ncomps_em]
labels_emf = np.ones(ncomps_em)
else:
tau_emf = []
labels_emf = []
params_emfit = []
params_emfit_errs = []
if ncomps_emf > 0:
say("\n\n --> Final Fitting... \n", verbose)
# Objective functions for final fit
def objective_leastsq(paramslm):
params = vals_vec_from_lmfit(paramslm)
resids = (func(vel, *params).ravel() - data.ravel()) / errors
return resids
# Compile parameters, labels, and original optical depths for final fit:
params_full = np.concatenate([params_emf, labels_emf, tau_emf])
# Final fit using constrained parameters
t0 = time.time()
lmfit_params = paramvec_p3_to_lmfit(params_full, wiggle, min_dv)
result3 = lmfit_minimize(objective_leastsq,
lmfit_params,
method="leastsq")
params_emfit = vals_vec_from_lmfit(result3.params)
params_emfit_errs = errs_vec_from_lmfit(result3.params)
ncomps_emfit = len(params_emfit) // 3
del lmfit_params
say("Final fit took {0} seconds.".format(time.time() - t0), verbose)
best_fit_final = func(vel, *params_emfit).ravel()
rchi2 = np.sum((data - best_fit_final)**2 / errors**2) / len(data)
else:
ncomps_emfit = ncomps_emf
# Construct output dictionary (odict)
# -----------------------------------
# print(
# "NUMBER ABS",
# ncomps_fit,
# " NUMBER EM",
# len(labels_emf[labels_emf == 1]),
# len(params_emfit) // 3,
# ncomps_emfit,
# )
odict["N_components_em"] = ncomps_emfit
if ncomps_emfit >= ncomps_fit:
odict["best_fit_parameters_em"] = params_emfit
odict["best_fit_errors_em"] = params_emfit_errs
odict["fit_labels"] = labels_emf
return (1, odict)
def AGD(
vel,
data,
errors,
alpha1=None,
alpha2=None,
# plot=False,
mode="python",
verbose=False,
SNR_thresh=5.0,
BLFrac=0.1,
SNR2_thresh=5.0,
deblend=True,
perform_final_fit=True,
phase="one",
):
""" Autonomous Gaussian Decomposition
"""
if type(SNR2_thresh) != type([]):
SNR2_thresh = [SNR2_thresh, SNR2_thresh]
if type(SNR_thresh) != type([]):
SNR_thresh = [SNR_thresh, SNR_thresh]
say("\n --> AGD() \n", verbose)
if (not alpha2) and (phase == "two"):
print("alpha2 value required")
return
dv = np.abs(vel[1] - vel[0])
v_to_i = interp1d(vel, np.arange(len(vel)))
# --------------------------------------#
# Find phase-one guesses #
# --------------------------------------#
agd1 = initialGuess(
vel,
data,
errors=None,
alpha=alpha1,
# plot=plot,
mode=mode,
verbose=verbose,
SNR_thresh=SNR_thresh[0],
BLFrac=BLFrac,
SNR2_thresh=SNR2_thresh[0],
deblend=deblend,
)
amps_g1, widths_g1, offsets_g1, u2 = (
agd1["amps"],
agd1["FWHMs"],
agd1["means"],
agd1["u2"],
)
params_g1 = np.append(np.append(amps_g1, widths_g1), offsets_g1)
ncomps_g1 = len(params_g1) // 3
ncomps_g2 = 0 # Default
ncomps_f1 = 0 # Default
# ----------------------------#
# Find phase-two guesses #
# ----------------------------#
if phase == "two":
say("Beginning phase-two AGD... ", verbose)
ncomps_g2 = 0
# ----------------------------------------------------------#
# Produce the residual signal #
# -- Either the original data, or intermediate subtraction #
# ----------------------------------------------------------#
if ncomps_g1 == 0:
say(
"Phase 2 with no narrow comps -> No intermediate subtration... ",
verbose,
)
residuals = data
else:
# "Else" Narrow components were found, and Phase == 2, so perform intermediate subtraction...
# The "fitmask" is a collection of windows around the a list of phase-one components
fitmask, fitmaskw = create_fitmask(len(vel), v_to_i(offsets_g1),
widths_g1 / dv / 2.355 * 0.9)
notfitmask = 1 - fitmask
# Error function for intermediate optimization
def objectiveD2_leastsq(paramslm):
params = vals_vec_from_lmfit(paramslm)
model0 = func(vel, *params)
model2 = np.diff(np.diff(model0.ravel())) / dv / dv
resids1 = fitmask[1:-1] * (model2 - u2[1:-1]) / errors[1:-1]
resids2 = notfitmask * (model0 - data) / errors / 10.0
return np.append(resids1, resids2)
# Perform the intermediate fit using LMFIT
t0 = time.time()
say("Running LMFIT on initial narrow components...", verbose)
lmfit_params = paramvec_to_lmfit(params_g1)
result = lmfit_minimize(objectiveD2_leastsq,
lmfit_params,
method="leastsq")
params_f1 = vals_vec_from_lmfit(result.params)
ncomps_f1 = len(params_f1) // 3
# # Make "FWHMS" positive
# params_f1[0:ncomps_f1][params_f1[0:ncomps_f1] < 0.0] = (
# -1 * params_f1[0:ncomps_f1][params_f1[0:ncomps_f1] < 0.0]
# )
del lmfit_params
say("LMFIT fit took {0} seconds.".format(time.time() - t0))
if result.success:
# Compute intermediate residuals
# Median filter on 2x effective scale to remove poor subtractions of strong components
intermediate_model = func(
vel, *params_f1).ravel() # Explicit final (narrow) model
median_window = 2.0 * 10**((np.log10(alpha1) + 2.187) / 3.859)
residuals = median_filter(data - intermediate_model,
np.int(median_window))
else:
residuals = data
# Finished producing residual signal # ---------------------------
# Search for phase-two guesses
agd2 = initialGuess(
vel,
residuals,
errors=None,
alpha=alpha2,
mode=mode,
verbose=verbose,
SNR_thresh=SNR_thresh[1],
BLFrac=BLFrac,
SNR2_thresh=SNR2_thresh[1], # June 9 2014, change
deblend=deblend,
# plot=plot,
)
ncomps_g2 = agd2["N_components"]
if ncomps_g2 > 0:
params_g2 = np.concatenate(
[agd2["amps"], agd2["FWHMs"], agd2["means"]])
else:
params_g2 = []
u22 = agd2["u2"]
# END PHASE 2 <<<
# Check for phase two components, make final guess list
# ------------------------------------------------------
if phase == "two" and (ncomps_g2 > 0):
amps_gf = np.append(params_g1[0:ncomps_g1], params_g2[0:ncomps_g2])
widths_gf = np.append(params_g1[ncomps_g1:2 * ncomps_g1],
params_g2[ncomps_g2:2 * ncomps_g2])
offsets_gf = np.append(
params_g1[2 * ncomps_g1:3 * ncomps_g1],
params_g2[2 * ncomps_g2:3 * ncomps_g2],
)
params_gf = np.concatenate([amps_gf, widths_gf, offsets_gf])
ncomps_gf = len(params_gf) // 3
else:
params_gf = params_g1
ncomps_gf = len(params_gf) // 3
# Sort final guess list by amplitude
# ----------------------------------
say("N final parameter guesses: " + str(ncomps_gf))
amps_temp = params_gf[0:ncomps_gf]
widths_temp = params_gf[ncomps_gf:2 * ncomps_gf]
offsets_temp = params_gf[2 * ncomps_gf:3 * ncomps_gf]
w_sort_amp = np.argsort(amps_temp)[::-1]
params_gf = np.concatenate([
amps_temp[w_sort_amp], widths_temp[w_sort_amp],
offsets_temp[w_sort_amp]
])
if perform_final_fit:
say("\n\n --> Final Fitting... \n", verbose)
if ncomps_gf > 0:
# Objective functions for final fit
def objective_leastsq(paramslm):
params = vals_vec_from_lmfit(paramslm)
resids = (func(vel, *params).ravel() - data.ravel()) / errors
return resids
# Final fit using unconstrained parameters
t0 = time.time()
lmfit_params = paramvec_to_lmfit(params_gf)
result2 = lmfit_minimize(objective_leastsq,
lmfit_params,
method="leastsq")
params_fit = vals_vec_from_lmfit(result2.params)
params_errs = errs_vec_from_lmfit(result2.params)
ncomps_fit = len(params_fit) // 3
del lmfit_params
say("Final fit took {0} seconds.".format(time.time() - t0),
verbose)
# # Make "FWHMS" positive
# params_fit[0:ncomps_fit][params_fit[0:ncomps_fit] < 0.0] = (
# -1 * params_fit[0:ncomps_fit][params_fit[0:ncomps_fit] < 0.0]
# )
best_fit_final = func(vel, *params_fit).ravel()
rchi2 = np.sum((data - best_fit_final)**2 / errors**2) / len(data)
# Check if any amplitudes are identically zero, if so, remove them.
if np.any(params_fit[0:ncomps_gf] == 0):
amps_fit = params_fit[0:ncomps_gf]
fwhms_fit = params_fit[ncomps_gf:2 * ncomps_gf]
offsets_fit = params_fit[2 * ncomps_gf:3 * ncomps_gf]
w_keep = amps_fit > 0.0
params_fit = np.concatenate(
[amps_fit[w_keep], fwhms_fit[w_keep], offsets_fit[w_keep]])
ncomps_fit = len(params_fit) // 3
else:
best_fit_final = np.zeros(len(vel))
rchi2 = -999.0
ncomps_fit = ncomps_gf
# if plot:
# # P L O T T I N G
# datamax = np.max(data)
#
# # Set up figure
# fig = plt.figure("AGD results", [12, 12])
# ax1 = fig.add_axes([0.1, 0.5, 0.4, 0.4]) # Initial guesses (alpha1)
# ax2 = fig.add_axes([0.5, 0.5, 0.4, 0.4]) # D2 fit to peaks(alpha2)
# ax3 = fig.add_axes([0.1, 0.1, 0.4, 0.4]) # Initial guesses (alpha2)
# ax4 = fig.add_axes([0.5, 0.1, 0.4, 0.4]) # Final fit
#
# # Decorations
# plt.figtext(0.52, 0.47, "Final fit")
# # if perform_final_fit:
# # plt.figtext(0.52, 0.45, "Reduced Chi2: {0:3.1f}".format(rchi2))
# # plt.figtext(0.52, 0.43, "N components: {0}".format(ncomps_fit))
#
# plt.figtext(0.12, 0.47, "Phase-two initial guess")
# plt.figtext(0.12, 0.45, "N components: {0}".format(ncomps_g2))
#
# plt.figtext(0.12, 0.87, "Phase-one initial guess")
# plt.figtext(0.12, 0.85, "N components: {0}".format(ncomps_g1))
#
# plt.figtext(0.52, 0.87, "Intermediate fit")
#
# # Initial Guesses (Panel 1)
# # -------------------------
# ax1.xaxis.tick_top()
# u2_scale = 1.0 / np.max(np.abs(u2)) * datamax * 0.5
# ax1.plot(vel, data, "-k")
# ax1.plot(vel, u2 * u2_scale, "-r")
# ax1.plot(vel, vel / vel * agd1["thresh"], "-k")
# ax1.plot(vel, vel / vel * agd1["thresh2"] * u2_scale, "--r")
#
# for i in range(ncomps_g1):
# one_component = gaussian(
# params_g1[i], params_g1[i + ncomps_g1], params_g1[i + 2 * ncomps_g1]
# )(vel)
# ax1.plot(vel, one_component, "-g")
#
# # Plot intermediate fit components (Panel 2)
# # ------------------------------------------
# ax2.xaxis.tick_top()
# ax2.plot(vel, data, "-k")
# ax2.yaxis.tick_right()
# for i in range(ncomps_f1):
# one_component = gaussian(
# params_f1[i], params_f1[i + ncomps_f1], params_f1[i + 2 * ncomps_f1]
# )(vel)
# ax2.plot(vel, one_component, "-", color="blue")
#
# # Residual spectrum (Panel 3)
# # -----------------------------
# if phase == "two":
# u22_scale = 1.0 / np.abs(u22).max() * np.max(residuals) * 0.5
# ax3.plot(vel, residuals, "-k")
# ax3.plot(vel, vel / vel * agd2["thresh"], "--k")
# ax3.plot(vel, vel / vel * agd2["thresh2"] * u22_scale, "--r")
# ax3.plot(vel, u22 * u22_scale, "-r")
# for i in range(ncomps_g2):
# one_component = gaussian(
# params_g2[i], params_g2[i + ncomps_g2], params_g2[i + 2 * ncomps_g2]
# )(vel)
# ax3.plot(vel, one_component, "-g")
#
# # Plot best-fit model (Panel 4)
# # -----------------------------
# if perform_final_fit:
# ax4.yaxis.tick_right()
# ax4.plot(vel, best_fit_final, label="final model", color="purple")
# ax4.plot(vel, data, label="data", color="black")
# for i in range(ncomps_fit):
# one_component = gaussian(
# params_fit[i],
# params_fit[i + ncomps_fit],
# params_fit[i + 2 * ncomps_fit],
# )(vel)
# ax4.plot(vel, one_component, "-", color="purple")
# ax4.plot(vel, best_fit_final, "-", color="purple")
#
# plt.show()
# plt.close()
# Construct output dictionary (odict)
# -----------------------------------
odict = {}
odict["initial_parameters"] = params_gf
odict["N_components"] = ncomps_gf
if (perform_final_fit) and (ncomps_gf > 0):
odict["best_fit_parameters"] = params_fit
odict["best_fit_errors"] = params_errs
odict["rchi2"] = rchi2
return (1, odict)
def AGD(vel,
data,
errors,
idx=None,
signal_ranges=None,
noise_spike_ranges=None,
improve_fitting_dict=None,
alpha1=None,
alpha2=None,
plot=False,
mode='conv',
verbose=False,
SNR_thresh=5.0,
BLFrac=0.1,
SNR2_thresh=5.0,
deblend=True,
perform_final_fit=True,
phase='one'):
""" Autonomous Gaussian Decomposition."""
dct = {}
if improve_fitting_dict is not None:
# TODO: check if max_amp causes problems
# dct['improve_fitting'] = improve_fitting_dict['improve_fitting']
# dct['min_fwhm'] = improve_fitting_dict['min_fwhm']
# dct['max_fwhm'] = improve_fitting_dict['max_fwhm']
# dct['snr_fit'] = improve_fitting_dict['snr_fit']
# dct['significance'] = improve_fitting_dict['significance']
# dct['min_offset'] = improve_fitting_dict['min_offset']
# dct['max_amp_factor'] = improve_fitting_dict['max_amp_factor']
# dct['max_amp'] = dct['max_amp_factor']*np.max(data)
# dct['rchi2_limit'] = improve_fitting_dict['rchi2_limit']
# dct['snr_negative'] = improve_fitting_dict['snr_negative']
# dct['snr'] = improve_fitting_dict['snr']
dct = improve_fitting_dict
dct['max_amp'] = dct['max_amp_factor'] * np.max(data)
nChannels = len(data)
else:
dct['improve_fitting'] = False
dct['max_amp'] = None
dct['max_fwhm'] = None
if not isinstance(SNR_thresh, list):
SNR_thresh = [SNR_thresh, SNR_thresh]
if not isinstance(SNR2_thresh, list):
SNR2_thresh = [SNR2_thresh, SNR2_thresh]
say('\n --> AGD() \n', verbose)
if (not alpha2) and (phase == 'two'):
print('alpha2 value required')
return
dv = np.abs(vel[1] - vel[0])
v_to_i = interp1d(vel, np.arange(len(vel)))
# -------------------------------------- #
# Find phase-one guesses #
# -------------------------------------- #
agd1 = initialGuess(vel,
data,
errors=errors[0],
alpha=alpha1,
plot=plot,
mode=mode,
verbose=verbose,
SNR_thresh=SNR_thresh[0],
BLFrac=BLFrac,
SNR2_thresh=SNR2_thresh[0],
deblend=deblend)
amps_g1, widths_g1, offsets_g1, u2 = agd1['amps'], agd1['FWHMs'], agd1[
'means'], agd1['u2']
params_g1 = np.append(np.append(amps_g1, widths_g1), offsets_g1)
ncomps_g1 = int(len(params_g1) / 3)
ncomps_g2 = 0 # Default
ncomps_f1 = 0 # Default
# ----------------------------#
# Find phase-two guesses #
# ----------------------------#
if phase == 'two':
say('Beginning phase-two AGD... ', verbose)
ncomps_g2 = 0
# ----------------------------------------------------------#
# Produce the residual signal #
# -- Either the original data, or intermediate subtraction #
# ----------------------------------------------------------#
if ncomps_g1 == 0:
say(
'Phase 2 with no narrow comps -> No intermediate subtration... ',
verbose)
residuals = data
else:
# "Else" Narrow components were found, and Phase == 2, so perform intermediate subtraction...
# The "fitmask" is a collection of windows around the a list of phase-one components
fitmask, fitmaskw = create_fitmask(len(vel), v_to_i(offsets_g1),
widths_g1 / dv / 2.355 * 0.9)
notfitmask = 1 - fitmask
notfitmaskw = np.logical_not(fitmaskw)
# Error function for intermediate optimization
def objectiveD2_leastsq(paramslm):
params = vals_vec_from_lmfit(paramslm)
model0 = func(vel, *params)
model2 = np.diff(np.diff(model0.ravel())) / dv / dv
resids1 = fitmask[1:-1] * (model2 - u2[1:-1]) / errors[1:-1]
resids2 = notfitmask * (model0 - data) / errors / 10.
return np.append(resids1, resids2)
# Perform the intermediate fit using LMFIT
t0 = time.time()
say('Running LMFIT on initial narrow components...', verbose)
lmfit_params = paramvec_to_lmfit(params_g1, dct['max_amp'],
dct['max_fwhm'])
result = lmfit_minimize(objectiveD2_leastsq,
lmfit_params,
method='leastsq')
params_f1 = vals_vec_from_lmfit(result.params)
ncomps_f1 = int(len(params_f1) / 3)
# Make "FWHMS" positive
# params_f1[0:ncomps_f1][np.array(params_f1[0:ncomps_f1]) < 0.0] =\
# -1 * params_f1[0:ncomps_f1][np.array(params_f1[0:ncomps_f1]) < 0.0]
del lmfit_params
say('LMFIT fit took {0} seconds.'.format(time.time() - t0))
if result.success:
# Compute intermediate residuals
# Median filter on 2x effective scale to remove poor subtractions of strong components
intermediate_model = func(
vel, *params_f1).ravel() # Explicit final (narrow) model
median_window = 2. * 10**((np.log10(alpha1) + 2.187) / 3.859)
residuals = median_filter(data - intermediate_model,
np.int(median_window))
else:
residuals = data
# Finished producing residual signal # ---------------------------
# Search for phase-two guesses
agd2 = initialGuess(
vel,
residuals,
errors=errors[0],
alpha=alpha2,
mode=mode,
verbose=verbose,
SNR_thresh=SNR_thresh[1],
BLFrac=BLFrac,
SNR2_thresh=SNR2_thresh[1], # June 9 2014, change
deblend=deblend,
plot=plot)
ncomps_g2 = agd2['N_components']
if ncomps_g2 > 0:
params_g2 = np.concatenate(
[agd2['amps'], agd2['FWHMs'], agd2['means']])
else:
params_g2 = []
u22 = agd2['u2']
# END PHASE 2 <<<
# Check for phase two components, make final guess list
# ------------------------------------------------------
if phase == 'two' and (ncomps_g2 > 0):
amps_gf = np.append(params_g1[0:ncomps_g1], params_g2[0:ncomps_g2])
widths_gf = np.append(params_g1[ncomps_g1:2 * ncomps_g1],
params_g2[ncomps_g2:2 * ncomps_g2])
offsets_gf = np.append(params_g1[2 * ncomps_g1:3 * ncomps_g1],
params_g2[2 * ncomps_g2:3 * ncomps_g2])
params_gf = np.concatenate([amps_gf, widths_gf, offsets_gf])
ncomps_gf = int(len(params_gf) / 3)
else:
params_gf = params_g1
ncomps_gf = int(len(params_gf) / 3)
# Sort final guess list by amplitude
# ----------------------------------
say('N final parameter guesses: ' + str(ncomps_gf))
amps_temp = params_gf[0:ncomps_gf]
widths_temp = params_gf[ncomps_gf:2 * ncomps_gf]
offsets_temp = params_gf[2 * ncomps_gf:3 * ncomps_gf]
w_sort_amp = np.argsort(amps_temp)[::-1]
params_gf = np.concatenate([
amps_temp[w_sort_amp], widths_temp[w_sort_amp],
offsets_temp[w_sort_amp]
])
if (perform_final_fit is True) and (ncomps_gf > 0):
say('\n\n --> Final Fitting... \n', verbose)
# Objective functions for final fit
def objective_leastsq(paramslm):
params = vals_vec_from_lmfit(paramslm)
resids = (func(vel, *params).ravel() - data.ravel()) / errors
return resids
# Final fit using unconstrained parameters
t0 = time.time()
lmfit_params = paramvec_to_lmfit(params_gf, dct['max_amp'], None)
result2 = lmfit_minimize(objective_leastsq,
lmfit_params,
method='leastsq')
params_fit = vals_vec_from_lmfit(result2.params)
params_errs = errs_vec_from_lmfit(result2.params)
del lmfit_params
say('Final fit took {0} seconds.'.format(time.time() - t0), verbose)
ncomps_fit = int(len(params_fit) / 3)
# Make "FWHMS" positive
# params_fit[0:ncomps_fit][np.array(params_fit[0:ncomps_fit]) < 0.0] =\
# -1 * params_fit[0:ncomps_fit][np.array(params_fit[0:ncomps_fit]) < 0.0]
best_fit_final = func(vel, *params_fit).ravel()
# Try to improve the fit
# ----------------------
if dct['improve_fitting']:
if ncomps_gf == 0:
ncomps_fit = 0
params_fit = []
# TODO: check if ncomps_fit should be ncomps_gf
best_fit_list, N_neg_res_peak, N_blended, log_gplus =\
try_to_improve_fitting(
vel, data, errors, params_fit, ncomps_fit, dct,
signal_ranges=signal_ranges, noise_spike_ranges=noise_spike_ranges)
params_fit, params_errs, ncomps_fit, best_fit_final, residual,\
rchi2, aicc, new_fit, params_min, params_max, pvalue, quality_control = best_fit_list
ncomps_gf = ncomps_fit
if plot:
# P L O T T I N G
datamax = np.max(data)
print(("params_fit:", params_fit))
if ncomps_gf == 0:
ncomps_fit = 0
best_fit_final = data * 0
if dct['improve_fitting']:
rchi2 = best_fit_list[5]
else:
# TODO: define mask from signal_ranges
rchi2 = goodness_of_fit(data, best_fit_final, errors, ncomps_fit)
# Set up figure
fig = plt.figure('AGD results', [16, 12])
ax1 = fig.add_axes([0.1, 0.5, 0.4, 0.4]) # Initial guesses (alpha1)
ax2 = fig.add_axes([0.5, 0.5, 0.4, 0.4]) # D2 fit to peaks(alpha2)
ax3 = fig.add_axes([0.1, 0.1, 0.4, 0.4]) # Initial guesses (alpha2)
ax4 = fig.add_axes([0.5, 0.1, 0.4, 0.4]) # Final fit
# Decorations
if dct['improve_fitting']:
plt.figtext(0.52, 0.47, 'Final fit (GaussPy+)')
else:
plt.figtext(0.52, 0.47, 'Final fit (GaussPy)')
if perform_final_fit:
plt.figtext(0.52, 0.45, 'Reduced Chi2: {0:3.3f}'.format(rchi2))
plt.figtext(0.52, 0.43, 'N components: {0}'.format(ncomps_fit))
plt.figtext(0.12, 0.47, 'Phase-two initial guess')
plt.figtext(0.12, 0.45, 'N components: {0}'.format(ncomps_g2))
plt.figtext(0.12, 0.87, 'Phase-one initial guess')
plt.figtext(0.12, 0.85, 'N components: {0}'.format(ncomps_g1))
plt.figtext(0.52, 0.87, 'Intermediate fit')
# Initial Guesses (Panel 1)
# -------------------------
ax1.xaxis.tick_top()
u2_scale = 1. / np.max(np.abs(u2)) * datamax * 0.5
ax1.axhline(color='black', linewidth=0.5)
ax1.plot(vel, data, '-k')
ax1.plot(vel, u2 * u2_scale, '-r')
ax1.plot(vel, np.ones(len(vel)) * agd1['thresh'], '--k')
ax1.plot(vel, np.ones(len(vel)) * agd1['thresh2'] * u2_scale, '--r')
for i in range(ncomps_g1):
one_component = gaussian(params_g1[i], params_g1[i + ncomps_g1],
params_g1[i + 2 * ncomps_g1])(vel)
ax1.plot(vel, one_component, '-g')
# Plot intermediate fit components (Panel 2)
# ------------------------------------------
ax2.xaxis.tick_top()
ax2.axhline(color='black', linewidth=0.5)
ax2.plot(vel, data, '-k')
ax2.yaxis.tick_right()
for i in range(ncomps_f1):
one_component = gaussian(params_f1[i], params_f1[i + ncomps_f1],
params_f1[i + 2 * ncomps_f1])(vel)
ax2.plot(vel, one_component, '-', color='blue')
# Residual spectrum (Panel 3)
# -----------------------------
if phase == 'two':
u22_scale = 1. / np.abs(u22).max() * np.max(residuals) * 0.5
ax3.axhline(color='black', linewidth=0.5)
ax3.plot(vel, residuals, '-k')
ax3.plot(vel, np.ones(len(vel)) * agd2['thresh'], '--k')
ax3.plot(vel,
np.ones(len(vel)) * agd2['thresh2'] * u22_scale, '--r')
ax3.plot(vel, u22 * u22_scale, '-r')
for i in range(ncomps_g2):
one_component = gaussian(params_g2[i],
params_g2[i + ncomps_g2],
params_g2[i + 2 * ncomps_g2])(vel)
ax3.plot(vel, one_component, '-g')
# Plot best-fit model (Panel 4)
# -----------------------------
if perform_final_fit:
ax4.yaxis.tick_right()
ax4.axhline(color='black', linewidth=0.5)
ax4.plot(vel, data, label='data', color='black')
for i in range(ncomps_fit):
one_component = gaussian(params_fit[i],
params_fit[i + ncomps_fit],
params_fit[i + 2 * ncomps_fit])(vel)
ax4.plot(vel, one_component, '--', color='orange')
ax4.plot(vel, best_fit_final, '-', color='orange', linewidth=2)
plt.show()
# Construct output dictionary (odict)
# -----------------------------------
odict = {}
odict['initial_parameters'] = params_gf
odict['N_components'] = ncomps_gf
odict['index'] = idx
if dct['improve_fitting']:
odict['best_fit_rchi2'] = rchi2
odict['best_fit_aicc'] = aicc
odict['pvalue'] = pvalue
odict['N_neg_res_peak'] = N_neg_res_peak
odict['N_blended'] = N_blended
odict['log_gplus'] = log_gplus
odict['quality_control'] = quality_control
if (perform_final_fit is True) and (ncomps_gf > 0):
odict['best_fit_parameters'] = params_fit
odict['best_fit_errors'] = params_errs
return (1, odict)
def AGD(vel,
data,
errors,
alpha1=None,
alpha2=None,
plot=False,
mode='c',
verbose=False,
SNR_thresh=5.0,
BLFrac=0.1,
SNR2_thresh=5.0,
deblend=True,
perform_final_fit=True,
phase='one'):
""" Autonomous Gaussian Decomposition
"""
if type(SNR2_thresh) != type([]): SNR2_thresh = [SNR2_thresh, SNR2_thresh]
if type(SNR_thresh) != type([]): SNR_thresh = [SNR_thresh, SNR_thresh]
say('\n --> AGD() \n', verbose)
if (not alpha2) and (phase == 'two'):
print 'alpha2 value required'
return
dv = np.abs(vel[1] - vel[0])
v_to_i = interp1d(vel, np.arange(len(vel)))
#--------------------------------------#
# Find phase-one guesses #
#--------------------------------------#
agd1 = initialGuess(vel,
data,
errors=None,
alpha=alpha1,
plot=plot,
mode=mode,
verbose=verbose,
SNR_thresh=SNR_thresh[0],
BLFrac=BLFrac,
SNR2_thresh=SNR2_thresh[0],
deblend=deblend)
amps_g1, widths_g1, offsets_g1, u2 = agd1['amps'], agd1['FWHMs'], agd1[
'means'], agd1['u2']
params_g1 = np.append(np.append(amps_g1, widths_g1), offsets_g1)
ncomps_g1 = len(params_g1) / 3
ncomps_g2 = 0 # Default
ncomps_f1 = 0 # Default
# ----------------------------#
# Find phase-two guesses #
# ----------------------------#
if phase == 'two':
say('Beginning phase-two AGD... ', verbose)
ncomps_g2 = 0.
# ----------------------------------------------------------#
# Produce the residual signal #
# -- Either the original data, or intermediate subtraction #
# ----------------------------------------------------------#
if ncomps_g1 == 0:
say(
'Phase 2 with no narrow comps -> No intermediate subtration... ',
verbose)
residuals = data
else:
# "Else" Narrow components were found, and Phase == 2, so perform intermediate subtraction...
# The "fitmask" is a collection of windows around the a list of phase-one components
fitmask, fitmaskw = create_fitmask(len(vel), v_to_i(offsets_g1),
widths_g1 / dv / 2.355 * 0.9)
notfitmask = 1 - fitmask
notfitmaskw = np.logical_not(fitmaskw)
# Error function for intermediate optimization
def objectiveD2_leastsq(paramslm):
params = vals_vec_from_lmfit(paramslm)
model0 = func(vel, *params)
model2 = np.diff(np.diff(model0.ravel())) / dv / dv
resids1 = fitmask[1:-1] * (model2 - u2[1:-1]) / errors[1:-1]
resids2 = notfitmask * (model0 - data) / errors / 10.
return np.append(resids1, resids2)
# Perform the intermediate fit using LMFIT
t0 = time.time()
say('Running LMFIT on initial narrow components...', verbose)
lmfit_params = paramvec_to_lmfit(params_g1)
result = lmfit_minimize(objectiveD2_leastsq,
lmfit_params,
method='leastsq')
params_f1 = vals_vec_from_lmfit(result.params)
ncomps_f1 = len(params_f1) / 3
# Make "FWHMS" positive
params_f1[0:ncomps_f1][
params_f1[0:ncomps_f1] < 0.0] = -1 * params_f1[0:ncomps_f1][
params_f1[0:ncomps_f1] < 0.0]
del lmfit_params
say('LMFIT fit took {0} seconds.'.format(time.time() - t0))
if result.success:
# Compute intermediate residuals
# Median filter on 2x effective scale to remove poor subtractions of strong components
intermediate_model = func(
vel, *params_f1).ravel() # Explicit final (narrow) model
median_window = 2. * 10**((np.log10(alpha1) + 2.187) / 3.859)
residuals = median_filter(data - intermediate_model,
np.int(median_window))
else:
residuals = data
# Finished producing residual signal # ---------------------------
# Search for phase-two guesses
agd2 = initialGuess(
vel,
residuals,
errors=None,
alpha=alpha2,
mode=mode,
verbose=verbose,
SNR_thresh=SNR_thresh[1],
BLFrac=BLFrac,
SNR2_thresh=SNR2_thresh[1], # June 9 2014, change
deblend=deblend,
plot=plot)
ncomps_g2 = agd2['N_components']
if ncomps_g2 > 0:
params_g2 = np.concatenate(
[agd2['amps'], agd2['FWHMs'], agd2['means']])
else:
params_g2 = []
u22 = agd2['u2']
# END PHASE 2 <<<
# Check for phase two components, make final guess list
# ------------------------------------------------------
if phase == 'two' and (ncomps_g2 > 0):
amps_gf = np.append(params_g1[0:ncomps_g1], params_g2[0:ncomps_g2])
widths_gf = np.append(params_g1[ncomps_g1:2 * ncomps_g1],
params_g2[ncomps_g2:2 * ncomps_g2])
offsets_gf = np.append(params_g1[2 * ncomps_g1:3 * ncomps_g1],
params_g2[2 * ncomps_g2:3 * ncomps_g2])
params_gf = np.concatenate([amps_gf, widths_gf, offsets_gf])
ncomps_gf = len(params_gf) / 3
else:
params_gf = params_g1
ncomps_gf = len(params_gf) / 3
# Sort final guess list by amplitude
# ----------------------------------
say('N final parameter guesses: ' + str(ncomps_gf))
amps_temp = params_gf[0:ncomps_gf]
widths_temp = params_gf[ncomps_gf:2 * ncomps_gf]
offsets_temp = params_gf[2 * ncomps_gf:3 * ncomps_gf]
w_sort_amp = np.argsort(amps_temp)[::-1]
params_gf = np.concatenate([
amps_temp[w_sort_amp], widths_temp[w_sort_amp],
offsets_temp[w_sort_amp]
])
if (perform_final_fit == True) and (ncomps_gf > 0):
say('\n\n --> Final Fitting... \n', verbose)
# Objective functions for final fit
def objective_leastsq(paramslm):
params = vals_vec_from_lmfit(paramslm)
resids = (func(vel, *params).ravel() - data.ravel()) / errors
return resids
# Final fit using unconstrained parameters
t0 = time.time()
lmfit_params = paramvec_to_lmfit(params_gf)
result2 = lmfit_minimize(objective_leastsq,
lmfit_params,
method='leastsq')
params_fit = vals_vec_from_lmfit(result2.params)
params_errs = errs_vec_from_lmfit(result2.params)
ncomps_fit = len(params_fit) / 3
del lmfit_params
say('Final fit took {0} seconds.'.format(time.time() - t0), verbose)
# Make "FWHMS" positive
params_fit[0:ncomps_fit][
params_fit[0:ncomps_fit] < 0.0] = -1 * params_fit[0:ncomps_fit][
params_fit[0:ncomps_fit] < 0.0]
best_fit_final = func(vel, *params_fit).ravel()
rchi2 = np.sum((data - best_fit_final)**2 / errors**2) / len(data)
# Check if any amplitudes are identically zero, if so, remove them.
if np.any(params_fit[0:ncomps_gf] == 0.0):
amps_fit = params_fit[0:ncomps_gf]
fwhms_fit = params_fit[ncomps_gf:2 * ncomps_gf]
offsets_fit = params_fit[2 * ncomps_gf:3 * ncomps_gf]
w_keep = amps_fit > 0.
params_fit = np.concatenate(
[amps_fit[w_keep], fwhms_fit[w_keep], offsets_fit[w_keep]])
ncomps_fit = len(params_fit) / 3
if plot:
# P L O T T I N G
datamax = np.max(data)
# Set up figure
fig = plt.figure('AGD results', [12, 12])
ax1 = fig.add_axes([0.1, 0.5, 0.4, 0.4]) # Initial guesses (alpha1)
ax2 = fig.add_axes([0.5, 0.5, 0.4, 0.4]) # D2 fit to peaks(alpha2)
ax3 = fig.add_axes([0.1, 0.1, 0.4, 0.4]) # Initial guesses (alpha2)
ax4 = fig.add_axes([0.5, 0.1, 0.4, 0.4]) # Final fit
# Decorations
plt.figtext(0.52, 0.47, 'Final fit')
if perform_final_fit:
plt.figtext(0.52, 0.45, 'Reduced Chi2: {0:3.1f}'.format(rchi2))
plt.figtext(0.52, 0.43, 'N components: {0}'.format(ncomps_fit))
plt.figtext(0.12, 0.47, 'Phase-two initial guess')
plt.figtext(0.12, 0.45, 'N components: {0}'.format(ncomps_g2))
plt.figtext(0.12, 0.87, 'Phase-one initial guess')
plt.figtext(0.12, 0.85, 'N components: {0}'.format(ncomps_g1))
plt.figtext(0.52, 0.87, 'Intermediate fit')
# Initial Guesses (Panel 1)
# -------------------------
ax1.xaxis.tick_top()
u2_scale = 1. / np.max(np.abs(u2)) * datamax * 0.5
ax1.plot(vel, data, '-k')
ax1.plot(vel, u2 * u2_scale, '-r')
ax1.plot(vel, vel / vel * agd1['thresh'], '-k')
ax1.plot(vel, vel / vel * agd1['thresh2'] * u2_scale, '--r')
for i in range(ncomps_g1):
one_component = gaussian(params_g1[i], params_g1[i + ncomps_g1],
params_g1[i + 2 * ncomps_g1])(vel)
ax1.plot(vel, one_component, '-g')
# Plot intermediate fit components (Panel 2)
# ------------------------------------------
ax2.xaxis.tick_top()
ax2.plot(vel, data, '-k')
ax2.yaxis.tick_right()
for i in range(ncomps_f1):
one_component = gaussian(params_f1[i], params_f1[i + ncomps_f1],
params_f1[i + 2 * ncomps_f1])(vel)
ax2.plot(vel, one_component, '-', color='blue')
# Residual spectrum (Panel 3)
# -----------------------------
if phase == 'two':
u22_scale = 1. / np.abs(u22).max() * np.max(residuals) * 0.5
ax3.plot(vel, residuals, '-k')
ax3.plot(vel, vel / vel * agd2['thresh'], '--k')
ax3.plot(vel, vel / vel * agd2['thresh2'] * u22_scale, '--r')
ax3.plot(vel, u22 * u22_scale, '-r')
for i in range(ncomps_g2):
one_component = gaussian(params_g2[i],
params_g2[i + ncomps_g2],
params_g2[i + 2 * ncomps_g2])(vel)
ax3.plot(vel, one_component, '-g')
# Plot best-fit model (Panel 4)
# -----------------------------
if perform_final_fit:
ax4.yaxis.tick_right()
ax4.plot(vel, best_fit_final, label='final model', color='purple')
ax4.plot(vel, data, label='data', color='black')
for i in range(ncomps_fit):
one_component = gaussian(params_fit[i],
params_fit[i + ncomps_fit],
params_fit[i + 2 * ncomps_fit])(vel)
ax4.plot(vel, one_component, '-', color='purple')
ax4.plot(vel, best_fit_final, '-', color='purple')
plt.show()
# Construct output dictionary (odict)
# -----------------------------------
odict = {}
odict['initial_parameters'] = params_gf
odict['N_components'] = ncomps_gf
if (perform_final_fit == True) and (ncomps_gf > 0):
odict['best_fit_parameters'] = params_fit
odict['best_fit_errors'] = params_errs
odict['rchi2'] = rchi2
return (1, odict)
def decompose(x,
y,
plot_fit=True,
diagnostic_plots=False,
min_peak=0.0,
**wiener_kwargs):
'''
Use a Wiener filter to create a low-pass filter of `y`, find the peaks,
and, starting with the largest peak, fit gaussian components until the
AIC stops decreasing.
'''
yfilt = wiener_filter(x, y, **wiener_kwargs)
# Catch where the PSD outputs have been passed
y_lp = yfilt[0]
y_hp = yfilt[1]
if len(yfilt) > 2:
PSD_out = yfilt[2:]
# Estimate the noise level from the high-pass filtered data
noise_std = astats.mad_std(y_hp)
# Get peaks in the low-pass spectrum using the minima in the 2nd
# derivative
# y_spl = UnivariateSpline(x, y_lp, s=0, k=4)
# y_spl_2der = y_spl.derivative(n=2)
# import matplotlib.pyplot as plt
# plt.plot(x, y_lp)
# plt.plot(x, y_spl(x))
peaks_idx = extrema_find(y_lp, mode='peak')
# peaks_idx = extrema_find(y_spl_2der(x), mode='dip')
peaks = x[peaks_idx]
peak_vals = y_lp[peaks_idx]
# Remove peaks below min_peak
peaks = peaks[peak_vals >= min_peak]
peak_vals = peak_vals[peak_vals >= min_peak]
peaks = peaks[np.argsort(peak_vals)[::-1]]
peak_vals = peak_vals[np.argsort(peak_vals)[::-1]]
# NOTE: This needs to be set using the Wiener filter width! Or some other
# estimator.
# init_width = 0.25
init_width = 10000
# init_width = 100
# u2 = np.diff(np.diff(y_lp)) # / np.diff(x[:2])[0]
# Find points of inflection
# inflection = np.abs(np.diff(np.sign(u2)))
# Find Relative widths, then measure
# peak-to-inflection distance for sharpest peak
# widths = np.sqrt(np.abs(y_lp / y_spl_2der(x))[peaks_idx])
# import matplotlib.pyplot as plt
# plt.plot(x, np.sqrt(np.abs(y_lp / y_spl_2der(x))))
# for peak in peaks:
# plt.axvline(peak)
# print(widths)
# print(argh)
# FWHMs = widths * 2.355
aics = []
bics = []
for i in range(len(peaks)):
if i == 0:
v0 = (peak_vals[i], init_width, peaks[i])
else:
v0 += (peak_vals[i], init_width, peaks[i])
v0_lmfit = gaussian_params_lmfit(v0)
def objective_leastsq(paramslm):
resids = (func(x, paramslm).ravel() - y) / noise_std
return resids
# Final fit using unconstrained parameters
result2 = lmfit_minimize(objective_leastsq, v0_lmfit, method='leastsq')
ncomps_fit = i + 1
params_fit = vals_vec_from_lmfit(result2.params)
if diagnostic_plots:
import matplotlib.pyplot as plt
plt.plot(x, y, alpha=0.7)
plt.plot(x, y_lp)
for peak in peaks:
plt.axvline(peak, color='g', linestyle='-.', alpha=0.5)
plt.plot(x, func(x, result2.params))
for j in range(1, ncomps_fit + 1):
pars = params_fit[j]
num_str = "{}".format(j)
plt.plot(x,
gaussian(pars['amp' + num_str],
pars['fwhm' + num_str],
pars['cent' + num_str])(x),
label=num_str)
plt.legend()
plt.draw()
raw_input("?")
plt.clf()
if i == 0:
aics.append(result2.aic)
bics.append(result2.bic)
result_prev = result2
params_fit_prev = params_fit
ncomps_fit_prev = ncomps_fit
continue
elif result2.aic > aics[i - 1] or result2.bic > bics[i - 1]:
aics.append(result2.aic)
bics.append(result2.bic)
break
else:
aics.append(result2.aic)
bics.append(result2.bic)
result_prev = result2
params_fit_prev = params_fit
ncomps_fit_prev = ncomps_fit
print("AICS: {}".format(aics))
print("BICS: {}".format(bics))
if plot_fit:
import matplotlib.pyplot as plt
plt.plot(x, y, alpha=0.7)
plt.plot(x, y_lp)
plt.axhline(noise_std)
for peak in peaks:
plt.axvline(peak, color='g', linestyle='-.', alpha=0.5)
for i in range(1, ncomps_fit_prev + 1):
pars = params_fit_prev[i]
num_str = "{}".format(i)
plt.plot(x,
gaussian(pars['amp' + num_str], pars['fwhm' + num_str],
pars['cent' + num_str])(x),
label=num_str)
plt.plot(x, func(x, result_prev.params))
plt.legend(frameon=True)
return result_prev, params_fit_prev
def fit_line(self, x, y, zero_lev, err_continuum, Ncomps, fitting_parameters, fitting_parameters_wide, iterations):
#Number of x points in the spectral line
x_Grid_Length = len(x)
#Generate empty containers to store the data
self.Fitting_dict['FluxI_N_vector'] = zeros(iterations)
for key in self.Fitting_dict['parameters_list']:
self.Fitting_dict[key + '_norm'] = zeros(iterations)
self.Fitting_dict[key + '_norm_er'] = zeros(iterations)
#Loop through the iterations (Only 1 if it is not a bootstrap)
for i in range(iterations):
#Fit narrow component
if i == 0:
y_new = y
else:
noise_array = random.normal(0.0, err_continuum, x_Grid_Length).astype(float32)
y_new = y + noise_array
fit_Output = lmfit_minimize(self.lmfit_gaussian_Residual, fitting_parameters, args=(x, y_new, zero_lev, Ncomps, err_continuum))
output_params = fit_Output.params
#Case with a wide component
if self.Fitting_dict['Add_wideComponent']:
sigma_limit = fit_Output.params['sigma1'].value
limit_0 = 6548.05 - self.Fitting_dict['x_scaler'] - sigma_limit * 1.5
limit_1 = 6548.05 - self.Fitting_dict['x_scaler'] + sigma_limit * 1.5
limit_2 = 0 - sigma_limit * 4
limit_3 = 0 + sigma_limit * 4
limit_4 = 6583.46 - self.Fitting_dict['x_scaler'] - sigma_limit * 3
limit_5 = 6583.46 - self.Fitting_dict['x_scaler'] + sigma_limit * 3
indeces = ((x >= limit_0) & (x <= limit_1)) + ((x >= limit_2) & (x <= limit_3)) + ((x >= limit_4) & (x <= limit_5))
mask = invert(indeces)
self.Fitting_dict['wide mask'] = mask
x_wide = x[mask]
y_wide = y_new[mask]
zero_wide = zero_lev[mask]
Ncomps_wide = ['3']
# y_wide[(y_wide < (mean(self.Fitting_dict['zerolev_norm']) - self.Fitting_dict['sig_zerolev_norm']))] = mean(self.Fitting_dict['zerolev_norm'])
fit_Output_wide = lmfit_minimize(self.lmfit_gaussian_Residual_wide, fitting_parameters_wide, args=(x_wide, y_wide, zero_wide, Ncomps_wide, err_continuum))
#Substract the wide components
y_wide = self.gaussian_curve_SingleMixture_wide(fit_Output_wide.params.valuesdict(), x, zero_lev, Ncomps_wide)
y_pure_emission = y_new - y_wide + zero_lev
self.Fitting_dict['emis_limpio'] = y_pure_emission
fit_Output_emission = lmfit_minimize(self.lmfit_gaussian_Residual, fitting_parameters, args=(x, y_pure_emission, zero_lev, Ncomps, err_continuum))
output_params = fit_Output_emission.params + fit_Output_wide.params
print fit_report(output_params)
print 'flux relation', output_params['FluxG2'] / output_params['FluxG0']
print 'A2*sigma2 / (A0*sigma0)', (output_params['A2'] * output_params['sigma2']) / (output_params['A0'] * output_params['sigma0'])
#Store the integrated flux
self.Fitting_dict['FluxI_N_vector'][i] = simps(y_new, x) - simps(zero_lev, x)
#Store the fitting parameters
for key in self.Fitting_dict['parameters_list']:
self.Fitting_dict[key + '_norm'][i] = output_params[key].value
self.Fitting_dict[key + '_norm_er'][i] = output_params[key].stderr
#Store the output fit (Only used for single line output)
self.Fitting_dict['lmfit_output'] = fit_Output
#Finally increase the number of components in case of a wide component
if self.Fitting_dict['Add_wideComponent']:
self.Fitting_dict['blended number'] = self.Fitting_dict['blended number'] + 1
return
v0 = (peak_vals[i], init_width, peaks[i])
else:
v0 += (peak_vals[i], init_width, peaks[i])
v0_lmfit = paramvec_to_lmfit(v0)
def objective_leastsq(paramslm):
if not isinstance(paramslm, tuple):
params = vals_vec_from_lmfit(paramslm)
else:
params = paramslm
resids = (func(x, *params).ravel() - ndata) / noise_std
return resids
# Final fit using unconstrained parameters
result2 = lmfit_minimize(objective_leastsq, v0_lmfit, method='leastsq')
params_fit = vals_vec_from_lmfit(result2.params)
params_errs = errs_vec_from_lmfit(result2.params)
ncomps_fit = len(params_fit) / 3
if i == 0:
aics.append(result2.aic)
result_old = result2
params_fit_old = params_fit
params_errs_old = params_errs
ncomps_fit_old = ncomps_fit
continue
elif result2.aic > aics[i - 1]:
break
else:
aics.append(result2.aic)
