def fit_knot_unified(hdu, j1, j2, u0, lineid='nii'):
NS, NV = hdu.data.shape
w = WCS(hdu.header)
vels, _ = w.all_pix2world(np.arange(NV), [0]*NV, 0)
vels /= 1000.0
# Ensure we don't go out of bounds
j1 = max(j1, 0)
j2 = min(j2, NS)
print('Slit pixels {}:{} out of {}'.format(j1, j2, NS))
knotspec = hdu.data[j1:j2, :].sum(axis=0)
# make sure all pixels are positive, since that helps the fitting/plotting
knotspec -= knotspec.min()
# Levenberg-Marquardt for easy jobs
lmfitter = SherpaFitter(statistic='chi2',
optimizer='levmar',
estmethod='confidence')
# Simulated annealing for trickier jobs
safitter = SherpaFitter(statistic='chi2',
optimizer='neldermead',
estmethod='covariance')
# The idea is that this strategy should work for all knots
# Estimate error from the BG: < -120 or > +100
bgmask = np.abs(vels + 10.0) >= 110.0
bgerr = np.std(knotspec[bgmask]) * np.ones_like(vels)
# Define core as [-10, 50], or 20 +/- 30
coremask = np.abs(vels - 20.0) < 30.0
# Fit to the BG with constant plus Lorentz
try:
vmean = np.average(vels[coremask], weights=knotspec[coremask])
except ZeroDivisionError:
vmean = 15.0
bgmodel = lmfitter(_init_bgmodel(vmean),
vels[bgmask], knotspec[bgmask],
err=bgerr[bgmask])
# Now freeze the BG model and add it to the initial core model
#bgmodel['Lorentz'].fixed['amplitude'] = True
#bgmodel['Constant'].fixed['amplitude'] = True
# Increase the data err in the bright part of the line to mimic Poisson noise
# Even though we don't know what the normalization is really, we will guess ...
spec_err = bgerr + POISSON_SCALE*np.sqrt(knotspec)
## Now for the exciting bit, fit everything at once
##
knotmask = np.abs(vels - u0) <= KNOT_WIDTH
# For low-velocity knots, we need to exclude positive velocities
# from the mask, since they will have large residual errors from
# the core subtraction
knotmask = knotmask & (vels < 0.0)
# Start off with the frozen BG model
fullmodel = bgmodel.copy()
core_components = list(fullmodel.submodel_names)
# Add in a model for the core
DV_INIT = [-15.0, -5.0, 5.0, 10.0, 30.0]
NCORE = len(DV_INIT)
BASE_WIDTH = 10.0 if lineid == 'ha' else 5.0
W_INIT = [BASE_WIDTH]*4 + [1.5*BASE_WIDTH]
for i in range(NCORE):
v0 = vmean + DV_INIT[i]
w0 = W_INIT[i]
component = 'G{}'.format(i)
fullmodel += Gaussian1D(
3.0, v0, w0,
bounds={'amplitude': [0, None],
'mean': [v0 - 10, v0 + 10],
'stddev': [w0, 1.5*w0]},
name=component)
core_components.append(component)
# Now, add in components for the knot to extract
knotmodel_init = Gaussian1D(
0.01, u0, BASE_WIDTH,
# Allow +/- 10 km/s leeway around nominal knot velocity
bounds={'amplitude': [0, None],
'mean': [u0 - 10, u0 + 10],
'stddev': [BASE_WIDTH, 25.0]},
name='Knot')
fullmodel += knotmodel_init
knot_components = ['Knot']
other_components = []
# Depending on the knot velocity, we may need other components to
# take up the slack too
if u0 <= -75.0 or u0 >= -50.0:
# Add in a generic fast knot
fullmodel += Gaussian1D(
0.01, -60.0, BASE_WIDTH,
bounds={'amplitude': [0, None],
'mean': [-70.0, -50.0],
'stddev': [BASE_WIDTH, 25.0]},
name='Fast other')
other_components.append('Fast other')
if u0 <= -50.0:
# Add in a generic slow knot
fullmodel += Gaussian1D(
0.01, -30.0, BASE_WIDTH,
bounds={'amplitude': [0, None],
'mean': [-40.0, -10.0],
'stddev': [BASE_WIDTH, 25.0]},
name='Slow other')
other_components.append('Slow other')
if u0 >= -75.0:
# Add in a very fast component
fullmodel += Gaussian1D(
0.001, -90.0, BASE_WIDTH,
bounds={'amplitude': [0, None],
'mean': [-110.0, -75.0],
'stddev': [BASE_WIDTH, 25.0]},
name='Ultra-fast other')
other_components.append('Ultra-fast other')
if u0 <= 30.0:
# Add in a red-shifted component just in case
fullmodel += Gaussian1D(
0.01, 40.0, BASE_WIDTH,
bounds={'amplitude': [0, None],
'mean': [30.0, 200.0],
'stddev': [BASE_WIDTH, 25.0]},
name='Red other')
other_components.append('Red other')
# Moment of truth: fit models to data
fullmodel = safitter(fullmodel, vels, knotspec, err=spec_err)
full_fit_info = safitter.fit_info
# Isolate the core+other model components
coremodel = fullmodel[core_components[0]]
for component in core_components[1:] + other_components:
coremodel += fullmodel[component]
# Subtract the core model from the data
residspec = knotspec - coremodel(vels)
# Now re-fit the knot model to the residual
# Calculate running std of residual spectrum
NWIN = 11
running_mean = generic_filter(residspec, np.mean, size=(NWIN,))
running_std = generic_filter(residspec, np.std, size=(NWIN,))
# Increase error estimate for data points where this is larger
# than spec_err, but only for velocities that are not in knotmask
residerr = bgerr
# residerr = spec_err
mask = (~knotmask) & (running_std > bgerr)
residerr[mask] = running_std[mask]
# The reason for this is so that poor modelling of the core is
# accounted for in the errors. Otherwise the reduced chi2 of the
# knot model will be too high
# Make an extended mask for fitting the knot, omitting the
# redshifted half of the spectrum since it is irrelevant and we
# don't want it to affect tha chi2 or the confidance intervals
bmask = vels < 50.0
knotmodel = lmfitter(knotmodel_init,
vels[bmask], residspec[bmask],
err=residerr[bmask])
# Calculate the final residuals, which should be flat
final_residual = residspec - knotmodel(vels)
# Look at stddev of the final residuals and use them to rescale
# the residual errors. Then re-fit the knot with this better
# estimate of the errors. But only if rescaling would reduce the
# data error estimate.
residerr_rescale = final_residual[bmask].std() / residerr[bmask].mean()
if residerr_rescale < 1.0:
print('Rescaling data errors by', residerr_rescale)
residerr *= residerr_rescale
knotmodel = lmfitter(knotmodel,
vels[bmask], residspec[bmask],
err=residerr[bmask])
else:
residerr_rescale = 1.0
knot_fit_info = lmfitter.fit_info
lmfitter._fitter.estmethod.config['max_rstat'] = MAX_RSTAT
if knot_fit_info.rstat < MAX_RSTAT:
knot_fit_errors = lmfitter.est_errors(sigma=3)
else:
knot_fit_errors = None
return {
'nominal knot velocity': u0,
'velocities': vels,
'full profile': knotspec,
'error profile': residerr,
'core fit model': coremodel,
'core fit profile': coremodel(vels),
'core fit components': {k: coremodel[k](vels) for k in coremodel.submodel_names},
'core fit info': full_fit_info,
'core-subtracted profile': residspec,
'knot fit model': knotmodel,
'knot fit profile': knotmodel(vels),
'knot fit info': knot_fit_info,
'knot fit errors': knot_fit_errors,
'error rescale factor': residerr_rescale,
'knot j range': (j1, j2),
}
def fit_knot_unified(hdu, j1, j2, u0, lineid='nii'):
NS, NV = hdu.data.shape
w = WCS(hdu.header)
vels, _ = w.all_pix2world(np.arange(NV), [0] * NV, 0)
vels /= 1000.0
# Ensure we don't go out of bounds
j1 = max(j1, 0)
j2 = min(j2, NS)
print('Slit pixels {}:{} out of {}'.format(j1, j2, NS))
knotspec = hdu.data[j1:j2, :].sum(axis=0)
# make sure all pixels are positive, since that helps the fitting/plotting
knotspec -= knotspec.min()
# Levenberg-Marquardt for easy jobs
lmfitter = SherpaFitter(statistic='chi2',
optimizer='levmar',
estmethod='confidence')
# Simulated annealing for trickier jobs
safitter = SherpaFitter(statistic='chi2',
optimizer='neldermead',
estmethod='covariance')
# The idea is that this strategy should work for all knots
# Estimate error from the BG: < -120 or > +100
bgmask = np.abs(vels + 10.0) >= 110.0
bgerr = np.std(knotspec[bgmask]) * np.ones_like(vels)
# Define core as [-10, 50], or 20 +/- 30
coremask = np.abs(vels - 20.0) < 30.0
# Fit to the BG with constant plus Lorentz
try:
vmean = np.average(vels[coremask], weights=knotspec[coremask])
except ZeroDivisionError:
vmean = 15.0
bgmodel = lmfitter(_init_bgmodel(vmean),
vels[bgmask],
knotspec[bgmask],
err=bgerr[bgmask])
# Now freeze the BG model and add it to the initial core model
#bgmodel['Lorentz'].fixed['amplitude'] = True
#bgmodel['Constant'].fixed['amplitude'] = True
# Increase the data err in the bright part of the line to mimic Poisson noise
# Even though we don't know what the normalization is really, we will guess ...
spec_err = bgerr + POISSON_SCALE * np.sqrt(knotspec)
## Now for the exciting bit, fit everything at once
##
knotmask = np.abs(vels - u0) <= KNOT_WIDTH
# For low-velocity knots, we need to exclude positive velocities
# from the mask, since they will have large residual errors from
# the core subtraction
knotmask = knotmask & (vels < 0.0)
# Start off with the frozen BG model
fullmodel = bgmodel.copy()
core_components = list(fullmodel.submodel_names)
# Add in a model for the core
DV_INIT = [-15.0, -5.0, 5.0, 10.0, 30.0]
NCORE = len(DV_INIT)
BASE_WIDTH = 10.0 if lineid == 'ha' else 5.0
W_INIT = [BASE_WIDTH] * 4 + [1.5 * BASE_WIDTH]
for i in range(NCORE):
v0 = vmean + DV_INIT[i]
w0 = W_INIT[i]
component = 'G{}'.format(i)
fullmodel += Gaussian1D(3.0,
v0,
w0,
bounds={
'amplitude': [0, None],
'mean': [v0 - 10, v0 + 10],
'stddev': [w0, 1.5 * w0]
},
name=component)
core_components.append(component)
# Now, add in components for the knot to extract
knotmodel_init = Gaussian1D(
0.01,
u0,
BASE_WIDTH,
# Allow +/- 10 km/s leeway around nominal knot velocity
bounds={
'amplitude': [0, None],
'mean': [u0 - 10, u0 + 10],
'stddev': [BASE_WIDTH, 25.0]
},
name='Knot')
fullmodel += knotmodel_init
knot_components = ['Knot']
other_components = []
# Depending on the knot velocity, we may need other components to
# take up the slack too
if u0 <= -75.0 or u0 >= -50.0:
# Add in a generic fast knot
fullmodel += Gaussian1D(0.01,
-60.0,
BASE_WIDTH,
bounds={
'amplitude': [0, None],
'mean': [-70.0, -50.0],
'stddev': [BASE_WIDTH, 25.0]
},
name='Fast other')
other_components.append('Fast other')
if u0 <= -50.0:
# Add in a generic slow knot
fullmodel += Gaussian1D(0.01,
-30.0,
BASE_WIDTH,
bounds={
'amplitude': [0, None],
'mean': [-40.0, -10.0],
'stddev': [BASE_WIDTH, 25.0]
},
name='Slow other')
other_components.append('Slow other')
if u0 >= -75.0:
# Add in a very fast component
fullmodel += Gaussian1D(0.001,
-90.0,
BASE_WIDTH,
bounds={
'amplitude': [0, None],
'mean': [-110.0, -75.0],
'stddev': [BASE_WIDTH, 25.0]
},
name='Ultra-fast other')
other_components.append('Ultra-fast other')
if u0 <= 30.0:
# Add in a red-shifted component just in case
fullmodel += Gaussian1D(0.01,
40.0,
BASE_WIDTH,
bounds={
'amplitude': [0, None],
'mean': [30.0, 200.0],
'stddev': [BASE_WIDTH, 25.0]
},
name='Red other')
other_components.append('Red other')
# Moment of truth: fit models to data
fullmodel = safitter(fullmodel, vels, knotspec, err=spec_err)
full_fit_info = safitter.fit_info
# Isolate the core+other model components
coremodel = fullmodel[core_components[0]]
for component in core_components[1:] + other_components:
coremodel += fullmodel[component]
# Subtract the core model from the data
residspec = knotspec - coremodel(vels)
# Now re-fit the knot model to the residual
# Calculate running std of residual spectrum
NWIN = 11
running_mean = generic_filter(residspec, np.mean, size=(NWIN, ))
running_std = generic_filter(residspec, np.std, size=(NWIN, ))
# Increase error estimate for data points where this is larger
# than spec_err, but only for velocities that are not in knotmask
residerr = bgerr
# residerr = spec_err
mask = (~knotmask) & (running_std > bgerr)
residerr[mask] = running_std[mask]
# The reason for this is so that poor modelling of the core is
# accounted for in the errors. Otherwise the reduced chi2 of the
# knot model will be too high
# Make an extended mask for fitting the knot, omitting the
# redshifted half of the spectrum since it is irrelevant and we
# don't want it to affect tha chi2 or the confidance intervals
bmask = vels < 50.0
knotmodel = lmfitter(knotmodel_init,
vels[bmask],
residspec[bmask],
err=residerr[bmask])
# Calculate the final residuals, which should be flat
final_residual = residspec - knotmodel(vels)
# Look at stddev of the final residuals and use them to rescale
# the residual errors. Then re-fit the knot with this better
# estimate of the errors. But only if rescaling would reduce the
# data error estimate.
residerr_rescale = final_residual[bmask].std() / residerr[bmask].mean()
if residerr_rescale < 1.0:
print('Rescaling data errors by', residerr_rescale)
residerr *= residerr_rescale
knotmodel = lmfitter(knotmodel,
vels[bmask],
residspec[bmask],
err=residerr[bmask])
else:
residerr_rescale = 1.0
knot_fit_info = lmfitter.fit_info
lmfitter._fitter.estmethod.config['max_rstat'] = MAX_RSTAT
if knot_fit_info.rstat < MAX_RSTAT:
knot_fit_errors = lmfitter.est_errors(sigma=3)
else:
knot_fit_errors = None
return {
'nominal knot velocity': u0,
'velocities': vels,
'full profile': knotspec,
'error profile': residerr,
'core fit model': coremodel,
'core fit profile': coremodel(vels),
'core fit components':
{k: coremodel[k](vels)
for k in coremodel.submodel_names},
'core fit info': full_fit_info,
'core-subtracted profile': residspec,
'knot fit model': knotmodel,
'knot fit profile': knotmodel(vels),
'knot fit info': knot_fit_info,
'knot fit errors': knot_fit_errors,
'error rescale factor': residerr_rescale,
'knot j range': (j1, j2),
}
def fit_knot(hdu, j1, j2, u0):
NS, NV = hdu.data.shape
w = WCS(hdu.header)
vels, _ = w.all_pix2world(np.arange(NV), [0]*NV, 0)
vels /= 1000.0
# Ensure we don't go out of bounds
j1 = max(j1, 0)
j2 = min(j2, NS)
print('Slit pixels {}:{} out of {}'.format(j1, j2, NS))
knotspec = hdu.data[j1:j2, :].sum(axis=0)
# make sure all pixels are positive, since that helps the fitting/plotting
knotspec -= knotspec.min()
# Levenberg-Marquardt for easy jobs
lmfitter = SherpaFitter(statistic='chi2',
optimizer='levmar',
estmethod='confidence')
# Simulated annealing for trickier jobs
safitter = SherpaFitter(statistic='chi2',
optimizer='neldermead',
estmethod='covariance')
# First do the strategy for typical knots (u0 = [-30, -80])
# Estimate error from the BG: < -120 or > +100
bgmask = np.abs(vels + 10.0) >= 110.0
bgerr = np.std(knotspec[bgmask]) * np.ones_like(vels)
# Fit to the BG with constant plus Lorentz
try:
vmean = np.average(vels, weights=knotspec)
except ZeroDivisionError:
vmean = 15.0
bgmodel = lmfitter(_init_bgmodel(vmean),
vels[bgmask], knotspec[bgmask],
err=bgerr[bgmask])
# Now freeze the BG model and add it to the initial core model
bgmodel['Lorentz'].fixed['amplitude'] = True
bgmodel['Constant'].fixed['amplitude'] = True
# Increase the data err in the bright part of the line to mimic Poisson noise
# Even though we don't know what the normalization is really, we will guess ...
spec_err = bgerr + POISSON_SCALE*np.sqrt(knotspec)
# Fit to the line core
knotmask = np.abs(vels - u0) <= KNOT_WIDTH
coremodel = safitter(_init_coremodel() + bgmodel,
vels[~knotmask], knotspec[~knotmask],
err=spec_err[~knotmask])
core_fit_info = safitter.fit_info
# Residual should contain just knot
residspec = knotspec - coremodel(vels)
# Calculate running std of residual spectrum
NWIN = 11
running_mean = generic_filter(residspec, np.mean, size=(NWIN,))
running_std = generic_filter(residspec, np.std, size=(NWIN,))
# Increase error estimate for data points where this is larger
# than spec_err, but only for velocities that are not in knotmask
residerr = bgerr
# residerr = spec_err
mask = (~knotmask) & (running_std > bgerr)
residerr[mask] = running_std[mask]
# The reason for this is so that poor modelling of the core is
# accounted for in the errors. Otherwise the reduced chi2 of the
# knot model will be too high
# Make an extended mask for fitting the knot, omitting the
# redshifted half of the spectrum since it is irrelevant and we
# don't want it to affect tha chi2 or the confidance intervals
bmask = vels < 50.0
# Fit single Gaussian to knot
amplitude_init = residspec[knotmask].max()
if amplitude_init < 0.0:
# ... pure desperation here
amplitude_init = residspec[bmask].max()
knotmodel = lmfitter(_init_knotmodel(amplitude_init, u0),
vels[bmask], residspec[bmask],
err=residerr[bmask])
# Calculate the final residuals, which should be flat
final_residual = residspec - knotmodel(vels)
# Look at stddev of the final residuals and use them to rescale
# the residual errors. Then re-fit the knot with this better
# estimate of the errors. But only if rescaling would reduce the
# data error estimate.
residerr_rescale = final_residual[bmask].std() / residerr[bmask].mean()
if residerr_rescale < 1.0:
print('Rescaling data errors by', residerr_rescale)
residerr *= residerr_rescale
knotmodel = lmfitter(knotmodel,
vels[bmask], residspec[bmask],
err=residerr[bmask])
else:
residerr_rescale = 1.0
knot_fit_info = lmfitter.fit_info
lmfitter._fitter.estmethod.config['max_rstat'] = MAX_RSTAT
if knot_fit_info.rstat < MAX_RSTAT:
knot_fit_errors = lmfitter.est_errors(sigma=3)
else:
knot_fit_errors = None
return {
'nominal knot velocity': u0,
'velocities': vels,
'full profile': knotspec,
'error profile': residerr,
'core fit model': coremodel,
'core fit profile': coremodel(vels),
'core fit components': {k: coremodel[k](vels) for k in coremodel.submodel_names},
'core fit info': core_fit_info,
'core-subtracted profile': residspec,
'knot fit model': knotmodel,
'knot fit profile': knotmodel(vels),
'knot fit info': knot_fit_info,
'knot fit errors': knot_fit_errors,
'error rescale factor': residerr_rescale,
}
def fit_knot(hdu, j1, j2, u0):
NS, NV = hdu.data.shape
w = WCS(hdu.header)
vels, _ = w.all_pix2world(np.arange(NV), [0] * NV, 0)
vels /= 1000.0
# Ensure we don't go out of bounds
j1 = max(j1, 0)
j2 = min(j2, NS)
print('Slit pixels {}:{} out of {}'.format(j1, j2, NS))
knotspec = hdu.data[j1:j2, :].sum(axis=0)
# make sure all pixels are positive, since that helps the fitting/plotting
knotspec -= knotspec.min()
# Levenberg-Marquardt for easy jobs
lmfitter = SherpaFitter(statistic='chi2',
optimizer='levmar',
estmethod='confidence')
# Simulated annealing for trickier jobs
safitter = SherpaFitter(statistic='chi2',
optimizer='neldermead',
estmethod='covariance')
# First do the strategy for typical knots (u0 = [-30, -80])
# Estimate error from the BG: < -120 or > +100
bgmask = np.abs(vels + 10.0) >= 110.0
bgerr = np.std(knotspec[bgmask]) * np.ones_like(vels)
# Fit to the BG with constant plus Lorentz
try:
vmean = np.average(vels, weights=knotspec)
except ZeroDivisionError:
vmean = 15.0
bgmodel = lmfitter(_init_bgmodel(vmean),
vels[bgmask],
knotspec[bgmask],
err=bgerr[bgmask])
# Now freeze the BG model and add it to the initial core model
bgmodel['Lorentz'].fixed['amplitude'] = True
bgmodel['Constant'].fixed['amplitude'] = True
# Increase the data err in the bright part of the line to mimic Poisson noise
# Even though we don't know what the normalization is really, we will guess ...
spec_err = bgerr + POISSON_SCALE * np.sqrt(knotspec)
# Fit to the line core
knotmask = np.abs(vels - u0) <= KNOT_WIDTH
coremodel = safitter(_init_coremodel() + bgmodel,
vels[~knotmask],
knotspec[~knotmask],
err=spec_err[~knotmask])
core_fit_info = safitter.fit_info
# Residual should contain just knot
residspec = knotspec - coremodel(vels)
# Calculate running std of residual spectrum
NWIN = 11
running_mean = generic_filter(residspec, np.mean, size=(NWIN, ))
running_std = generic_filter(residspec, np.std, size=(NWIN, ))
# Increase error estimate for data points where this is larger
# than spec_err, but only for velocities that are not in knotmask
residerr = bgerr
# residerr = spec_err
mask = (~knotmask) & (running_std > bgerr)
residerr[mask] = running_std[mask]
# The reason for this is so that poor modelling of the core is
# accounted for in the errors. Otherwise the reduced chi2 of the
# knot model will be too high
# Make an extended mask for fitting the knot, omitting the
# redshifted half of the spectrum since it is irrelevant and we
# don't want it to affect tha chi2 or the confidance intervals
bmask = vels < 50.0
# Fit single Gaussian to knot
amplitude_init = residspec[knotmask].max()
if amplitude_init < 0.0:
# ... pure desperation here
amplitude_init = residspec[bmask].max()
knotmodel = lmfitter(_init_knotmodel(amplitude_init, u0),
vels[bmask],
residspec[bmask],
err=residerr[bmask])
# Calculate the final residuals, which should be flat
final_residual = residspec - knotmodel(vels)
# Look at stddev of the final residuals and use them to rescale
# the residual errors. Then re-fit the knot with this better
# estimate of the errors. But only if rescaling would reduce the
# data error estimate.
residerr_rescale = final_residual[bmask].std() / residerr[bmask].mean()
if residerr_rescale < 1.0:
print('Rescaling data errors by', residerr_rescale)
residerr *= residerr_rescale
knotmodel = lmfitter(knotmodel,
vels[bmask],
residspec[bmask],
err=residerr[bmask])
else:
residerr_rescale = 1.0
knot_fit_info = lmfitter.fit_info
lmfitter._fitter.estmethod.config['max_rstat'] = MAX_RSTAT
if knot_fit_info.rstat < MAX_RSTAT:
knot_fit_errors = lmfitter.est_errors(sigma=3)
else:
knot_fit_errors = None
return {
'nominal knot velocity': u0,
'velocities': vels,
'full profile': knotspec,
'error profile': residerr,
'core fit model': coremodel,
'core fit profile': coremodel(vels),
'core fit components':
{k: coremodel[k](vels)
for k in coremodel.submodel_names},
'core fit info': core_fit_info,
'core-subtracted profile': residspec,
'knot fit model': knotmodel,
'knot fit profile': knotmodel(vels),
'knot fit info': knot_fit_info,
'knot fit errors': knot_fit_errors,
'error rescale factor': residerr_rescale,
}