def flexure_slit(slf, det):
"""Correct wavelength down slit center for flexure
Parameters:
----------
slf :
det : int
"""
# Load Archive
skyspec_fil, arx_sky = flexure_archive(slf, det)
# Extract
censpec_wv = arextract.boxcar_cen(slf, det, slf._mswave[det-1])
censpec_fx = arextract.boxcar_cen(slf, det, slf._bgframe[det-1])
cen_sky = xspectrum1d.XSpectrum1D.from_tuple((censpec_wv, censpec_fx))
# Find shift
fdict = flex_shift(slf, det, cen_sky, arx_sky)
msgs.work("Flexure shift = {:g} down slit center".format(fdict['shift']))
# Refit
# What if xfit shifts outside of 0-1?
xshift = fdict['shift']/(slf._msarc[det-1].shape[0]-1)
mask, fit = arutils.robust_polyfit(np.array(slf._wvcalib[det-1]['xfit'])+xshift,
np.array(slf._wvcalib[det-1]['yfit']),
len(slf._wvcalib[det-1]['fitc']),
function=slf._wvcalib[det-1]['function'], sigma=slf._wvcalib[det-1]['nrej'], minv=slf._wvcalib[det-1]['fmin'], maxv=slf._wvcalib[det-1]['fmax'])
# Update wvcalib
slf._wvcalib[det-1]['shift'] = fdict['shift'] # pixels
slf._wvcalib[det-1]['fitc'] = fit
msgs.work("Add another QA for wavelengths?")
# Update mswave
wv_calib = slf._wvcalib[det-1]
slf._mswave[det-1] = arutils.func_val(wv_calib['fitc'], slf._tilts[det-1], wv_calib['function'], minv=wv_calib['fmin'], maxv=wv_calib['fmax'])
# Write to Masters? Not for now
# For QA (kludgy..)
censpec_wv = arextract.boxcar_cen(slf, det, slf._mswave[det-1])
fdict['sky_spec'] = xspectrum1d.XSpectrum1D.from_tuple((censpec_wv, censpec_fx))
flex_dict = dict(polyfit=[], shift=[], subpix=[], corr=[],
corr_cen=[], spec_file=skyspec_fil, smooth=[],
arx_spec=[], sky_spec=[])
#debugger.set_trace()
#debugger.xplot(censpec_wv, censpec_fx, xtwo=fdict['arx_spec'].wavelength, ytwo=fdict['arx_spec'].flux*50)
for key in ['polyfit', 'shift', 'subpix', 'corr', 'corr_cen', 'smooth', 'sky_spec', 'arx_spec']:
flex_dict[key].append(fdict[key])
return flex_dict
def flexure_slit(slf, det):
"""Correct wavelength down slit center for flexure
Parameters:
----------
slf :
det : int
"""
# Load Archive
skyspec_fil, arx_sky = flexure_archive(slf, det)
# Extract
censpec_wv = arextract.boxcar_cen(slf, det, slf._mswave[det-1])
censpec_fx = arextract.boxcar_cen(slf, det, slf._bgframe[det-1])
cen_sky = xspectrum1d.XSpectrum1D.from_tuple((censpec_wv, censpec_fx))
# Find shift
fdict = flex_shift(slf, det, cen_sky, arx_sky)
msgs.work("Flexure shift = {:g} down slit center".format(fdict['shift']))
# Refit
# What if xfit shifts outside of 0-1?
xshift = fdict['shift']/(slf._msarc[det-1].shape[0]-1)
mask, fit = arutils.robust_polyfit(np.array(slf._wvcalib[det-1]['xfit'])+xshift,
np.array(slf._wvcalib[det-1]['yfit']),
len(slf._wvcalib[det-1]['fitc']),
function=slf._wvcalib[det-1]['function'], sigma=slf._wvcalib[det-1]['nrej'], minv=slf._wvcalib[det-1]['fmin'], maxv=slf._wvcalib[det-1]['fmax'])
# Update wvcalib
slf._wvcalib[det-1]['shift'] = fdict['shift'] # pixels
slf._wvcalib[det-1]['fitc'] = fit
msgs.work("Add another QA for wavelengths?")
# Update mswave
wv_calib = slf._wvcalib[det-1]
slf._mswave[det-1] = arutils.func_val(wv_calib['fitc'], slf._tilts[det-1], wv_calib['function'], minv=wv_calib['fmin'], maxv=wv_calib['fmax'])
# Write to Masters? Not for now
# For QA (kludgy..)
censpec_wv = arextract.boxcar_cen(slf, det, slf._mswave[det-1])
fdict['sky_spec'] = xspectrum1d.XSpectrum1D.from_tuple((censpec_wv, censpec_fx))
flex_dict = dict(polyfit=[], shift=[], subpix=[], corr=[],
corr_cen=[], spec_file=skyspec_fil, smooth=[],
arx_spec=[], sky_spec=[])
#debugger.set_trace()
#debugger.xplot(censpec_wv, censpec_fx, xtwo=fdict['arx_spec'].wavelength, ytwo=fdict['arx_spec'].flux*50)
for key in ['polyfit', 'shift', 'subpix', 'corr', 'corr_cen', 'smooth', 'sky_spec', 'arx_spec']:
flex_dict[key].append(fdict[key])
return flex_dict
def refine_iter(outpar, orders, mask, irshft, relshift, fitord, function='polynomial'):
fail = False
x0ex = arutils.func_val(outpar['x0res'], orders, function, minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
# Make the refinement
x0ex[irshft] += relshift
# Refit the data to improve the refinement
good = np.where(mask != 0.0)[0]
# x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
null, x0res = arutils.robust_polyfit(orders[good], x0ex[good], fitord, sigma=2.0, function=function,
minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
#x0res = arutils.func_fit(orders[good], x0ex[good], function, fitord, min=outpar['x0in'][0], max=outpar['x0in'][-1])
x0fit = arutils.func_val(x0res, orders, function, minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
chisq = (x0ex[good]-x0fit[good])**2.0
chisqnu = np.sum(chisq)/np.sum(mask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu > 0.5: # The refinement went wrong, ignore the refinement
fail = True
outpar['x0res'] = x0res
extfit = np.dot(outpar['eigv'], outpar['high_matr'].T) + np.outer(x0fit, np.ones(outpar['eigv'].shape[0])).T
return extfit, outpar, fail
def refine_iter(outpar,
orders,
mask,
irshft,
relshift,
fitord,
function='polynomial'):
fail = False
x0ex = arutils.func_val(outpar['x0res'],
orders,
function,
minv=outpar['x0in'][0],
maxv=outpar['x0in'][-1])
# Make the refinement
x0ex[irshft] += relshift
# Refit the data to improve the refinement
good = np.where(mask != 0.0)[0]
# x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
null, x0res = arutils.robust_polyfit(orders[good],
x0ex[good],
fitord,
sigma=2.0,
function=function,
minv=outpar['x0in'][0],
maxv=outpar['x0in'][-1])
#x0res = arutils.func_fit(orders[good], x0ex[good], function, fitord, min=outpar['x0in'][0], max=outpar['x0in'][-1])
x0fit = arutils.func_val(x0res,
orders,
function,
minv=outpar['x0in'][0],
maxv=outpar['x0in'][-1])
chisq = (x0ex[good] - x0fit[good])**2.0
chisqnu = np.sum(chisq) / np.sum(mask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu > 0.5: # The refinement went wrong, ignore the refinement
fail = True
outpar['x0res'] = x0res
extfit = np.dot(outpar['eigv'], outpar['high_matr'].T) + np.outer(
x0fit, np.ones(outpar['eigv'].shape[0])).T
return extfit, outpar, fail
def background_subtraction(slf,
sciframe,
varframe,
slitn,
det,
refine=0.0,
doqa=True):
""" Generate a frame containing the background sky spectrum
Parameters
----------
slf : Class
Science Exposure Class
sciframe : ndarray
science frame
varframe : ndarray
variance frame
slitn : int
Slit number
det : int
Detector index
refine : float or ndarray
refine the object traces. This should be a small value around 0.0.
If a float, a constant offset will be applied.
Otherwise, an array needs to be specified of the same length as
sciframe.shape[0] that contains the refinement of each pixel along
the spectral direction.
Returns
-------
bgframe : ndarray
An image, the same size as sciframe, that contains
the background spectrum within the specified slit.
nl : int
number of pixels from the left slit edge to use as background pixels
nr : int
number of pixels from the right slit edge to use as background pixels
"""
# Obtain all pixels that are within the slit edges, and are not masked
word = np.where((slf._slitpix[det - 1] == slitn + 1)
& (slf._scimask[det - 1] == 0))
if word[0].size == 0:
msgs.warn("There are no pixels in slit {0:d}".format(slitn))
debugger.set_trace()
nl, nr = 0, 0
return np.zeros_like(sciframe), nl, nr
# Calculate the oversampled object profiles
oversampling_factor = 3 # should be an integer according to the description in object_profile()
xedges, modvals = object_profile(slf,
sciframe,
slitn,
det,
refine=refine,
factor=oversampling_factor)
bincent = 0.5 * (xedges[1:] + xedges[:-1])
npix = slf._pixwid[det - 1][slitn]
tilts = slf._tilts[det - 1].copy()
lordloc = slf._lordloc[det - 1][:, slitn]
rordloc = slf._rordloc[det - 1][:, slitn]
# For each pixel, calculate the fraction along the slit's spatial direction
spatval = (word[1] - lordloc[word[0]] + refine) / (rordloc[word[0]] -
lordloc[word[0]])
# Cumulative sum and normalize
csum = np.cumsum(modvals)
csum -= csum[0]
csum /= csum[-1]
# Find a first guess of the edges of the object profile - assume this is the innermost 90 percent of the flux
argl = np.argmin(np.abs(csum - 0.05))
argr = np.argmin(np.abs(csum - 0.95))
# Considering the possible background pixels that are left of the object,
# find the first time where the object profile no longer decreases as you
# move toward the edge of the slit. This is the beginning of the noisy
# object profile, which is where the object can no longer be distinguished
# from the background.
wl = np.where((modvals[1:] < modvals[:-1]) & (bincent[1:] < bincent[argl]))
wr = np.where((modvals[1:] > modvals[:-1]) & (bincent[1:] > bincent[argr]))
nl, nr = 0, 0
if wl[0].size != 0:
# This is the index of the first time where the object profile
# no longer decreases as you move towards the slit edge
nl_index = np.max(wl[0])
# Calculate nl, defined as:
# "number of pixels from the left slit edge to use as background pixels",
# which is just nl_index with the sampling factor taken out
nl_index_origscale = int(nl_index / oversampling_factor + 0.5)
nl = nl_index_origscale
if wr[0].size != 0:
# This is the index of the first time where the object profile
# no longer decreases as you move towards the slit edge
nr_index = np.min(wr[0])
# Calculate nr, defined as:
# "number of pixels from the right slit edge to use as background pixels",
# which is npix minus nr_index with the sampling factor taken out
nr_index_origscale = int(nr_index / oversampling_factor + 0.5)
nr = npix - nr_index_origscale
if nl + nr < 5:
msgs.warn(
"The object profile appears to extrapolate to the edge of the slit"
)
msgs.info(
"A background subtraction will not be performed for slit {0:d}".
format(slitn + 1))
nl, nr = 0, 0
return np.zeros_like(sciframe), nl, nr
# Find background pixels and fit
wbgpix_spatval = np.where(
(spatval <= float(nl) / npix) |
(spatval >= float(npix - nr) /
npix)) # this cannot be used to index the 2D array tilts
wbgpix = (word[0][wbgpix_spatval], word[1][wbgpix_spatval]
) # this may be approproate for indexing the 2D array tilts
if settings.argflag['reduce']['skysub']['method'].lower() == 'bspline':
msgs.info("Using bspline sky subtraction")
srt = np.argsort(tilts[wbgpix])
ivar = arutils.calc_ivar(varframe)
# Perform a weighted b-spline fit to the sky background pixels
mask, bspl = arutils.robust_polyfit(
tilts[wbgpix][srt],
sciframe[wbgpix][srt],
3,
function='bspline',
weights=np.sqrt(ivar)[wbgpix][srt],
sigma=5.,
maxone=False,
**settings.argflag['reduce']['skysub']['bspline'])
bgf_flat = arutils.func_val(bspl, tilts.flatten(), 'bspline')
bgframe = bgf_flat.reshape(tilts.shape)
if doqa:
plt_bspline_sky(tilts, sciframe, bgf_flat, gdp)
debugger.set_trace()
else:
msgs.error('Not ready for this method for skysub {:s}'.format(
settings.argflag['reduce']['skysub']['method'].lower()))
if np.any(np.isnan(bgframe)):
msgs.warn("NAN in bgframe. Replacing with 0")
bad = np.isnan(bgframe)
bgframe[bad] = 0.
return bgframe, nl, nr
def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None, function='polynomial', retmask=False):
nrow = xfit.shape[0]
ntrace = xfit.shape[1]
if x0in is None:
x0in = np.arange(float(ntrace))
# Mask out some orders if they are bad
if mask is None or mask.size == 0:
usetrace = np.arange(ntrace)
outmask = np.ones((nrow, ntrace))
else:
usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0]
outmask = np.ones((nrow, ntrace))
outmask[:,mask] = 0.0
# Do the PCA analysis
eigc, hidden = get_pc(coeff[1:npc+1, usetrace], npc)
modl = arutils.func_vander(xfit[:,0], function, npc)
eigv = np.dot(modl[:,1:], eigc)
med_hidden = np.median(hidden, axis=1)
med_highorder = med_hidden.copy()
med_highorder[0] = 0
high_order_matrix = med_highorder.T[np.newaxis,:].repeat(ntrace, axis=0)
# y = hidden[0,:]
# coeff0 = arutils.robust_regression(x0in[usetrace], y, pnpc[1], 0.1, function=function)
# y = hidden[1,:]
# coeff1 = arutils.robust_regression(x0in[usetrace], y, pnpc[2], 0.1, function=function)
coeffstr = []
for i in xrange(1, npc+1):
# if pnpc[i] == 0:
# coeffstr.append([-9.99E9])
# continue
# coeff0 = arutils.robust_regression(x0in[usetrace], hidden[i-1,:], pnpc[i], 0.1, function=function, min=x0in[0], max=x0in[-1])
if weights is not None:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace], hidden[i-1,:], pnpc[i],
weights=weights[usetrace], sigma=2.0, function=function,
minv=x0in[0], maxv=x0in[-1])
else:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace], hidden[i-1,:], pnpc[i],
sigma=2.0, function=function, minv=x0in[0], maxv=x0in[-1])
coeffstr.append(coeff0)
high_order_matrix[:,i-1] = arutils.func_val(coeff0, x0in, function, minv=x0in[0], maxv=x0in[-1])
# high_order_matrix[:,1] = arutils.func_val(coeff1, x0in, function)
high_fit = high_order_matrix.copy()
high_order_fit = np.dot(eigv, high_order_matrix.T)
sub = (yfit - high_order_fit) * outmask
numer = np.sum(sub, axis=0)
denom = np.sum(outmask, axis=0)
x0 = np.zeros(float(ntrace))
fitmask = np.zeros(float(ntrace))
#fitmask[mask] = 1
x0fit = np.zeros(float(ntrace))
chisqnu = 0.0
chisqold = 0.0
robust = True
#svx0 = numer/(denom+(denom == 0).astype(np.int))
if not skipx0:
fitmask = (np.abs(denom) > 10).astype(np.int)
if robust:
good = np.where(fitmask != 0)[0]
bad = np.where(fitmask == 0)[0]
x0[good] = numer[good]/denom[good]
imask = np.zeros(float(ntrace))
imask[bad] = 1.0
ttmask, x0res = arutils.robust_polyfit(x0in, x0, pnpc[0], weights=weights, sigma=2.0,
function=function, minv=x0in[0], maxv=x0in[-1], initialmask=imask)
x0fit = arutils.func_val(x0res, x0in, function, minv=x0in[0], maxv=x0in[-1])
good = np.where(ttmask == 0)[0]
xstd = 1.0 # This should represent the dispersion in the fit
chisq = ((x0[good]-x0fit[good])/xstd)**2.0
chisqnu = np.sum(chisq)/np.sum(ttmask)
fitmask = 1.0-ttmask
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
else:
for i in xrange(1, 5):
good = np.where(fitmask != 0)[0]
x0[good] = numer[good]/denom[good]
# x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
x0res = arutils.func_fit(x0in[good], x0[good], function, pnpc[0],
weights=weights, minv=x0in[0], maxv=x0in[-1])
x0fit = arutils.func_val(x0res, x0in, function, minv=x0in[0], maxv=x0in[-1])
chisq = (x0[good]-x0fit[good])**2.0
fitmask[good] *= (chisq < np.sum(chisq)/2.0).astype(np.int)
chisqnu = np.sum(chisq)/np.sum(fitmask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu == chisqold:
break
else:
chisqold = chisqnu
if chisqnu > 2.0:
msgs.warn("PCA has very large residuals")
elif chisqnu > 0.5:
msgs.warn("PCA has fairly large residuals")
#bad = np.where(fitmask==0)[0]
#x0[bad] = x0fit[bad]
else:
x0res = 0.0
x3fit = np.dot(eigv,high_order_matrix.T) + np.outer(x0fit,np.ones(nrow)).T
outpar = dict({'high_fit': high_fit, 'x0': x0, 'x0in': x0in, 'x0fit': x0fit, 'x0res': x0res, 'x0mask': fitmask,
'hidden': hidden, 'usetrc': usetrace, 'eigv': eigv, 'npc': npc, 'coeffstr': coeffstr})
if retmask:
return x3fit, outpar, tmask
else:
return x3fit, outpar
def bspline_magfit(wave, flux, var, flux_std, nointerp=False, **kwargs):
"""
Perform a bspline fit to the flux ratio of standard to
observed counts. Used to generate a sensitivity function.
Parameters
----------
wave : ndarray
flux : ndarray
counts/s as observed
var : ndarray
variance
flux_std : Quantity array
standard star true flux (erg/s/cm^2/A)
nointer : bool, optional [False]
Skip interpolation over bad points (not recommended)?
**kwargs : keywords for robust_polyfit
Returns
-------
"""
from pypit import arutils
invvar = (var > 0.)/(var + (var <= 0.))
nx = wave.size
pos_error = 1./np.sqrt(np.maximum(invvar,0.) + (invvar == 0))
pos_mask = (flux > pos_error/10.0) & (invvar > 0) & (flux_std > 0.0)
#pos = pos_mask==1 npos)
fluxlog = 2.5*np.log10(np.maximum(flux,pos_error/10))
logivar = invvar * flux**2 * pos_mask*1.08574
# cap the magfunc so that sensfunc < 1.0e10
magfunc = 2.5*np.log10(np.maximum(flux_std,1.0e-2)) - fluxlog
magfunc = np.minimum(magfunc,25.0)
sensfunc = 10.0**(0.4*magfunc)*pos_mask
# Interpolate over masked pixels
if not nointerp:
bad = logivar <= 0.
if np.sum(bad) > 0:
f = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], magfunc[~bad], k=2)
magfunc[bad] = f(wave[bad])
fi = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], logivar[~bad], k=2)
logivar[bad] = fi(wave[bad])
# First iteration
mask, tck = arutils.robust_polyfit(wave, magfunc, 3, function='bspline', weights=np.sqrt(logivar), **kwargs)
logfit1 = arutils.func_val(tck,wave,'bspline')
modelfit1 = 10.0**(0.4*(logfit1))
residual = sensfunc/(modelfit1 + (modelfit1 == 0)) - 1.
new_mask = pos_mask & (sensfunc > 0)
residual_ivar = (modelfit1*flux/(sensfunc + (sensfunc == 0.0)))**2*invvar
residual_ivar = residual_ivar*new_mask
# Interpolate over masked pixels
if not nointerp:
if np.sum(bad) > 0:
f = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], residual[~bad], k=2)
residual[bad] = f(wave[bad])
fi = scipy.interpolate.InterpolatedUnivariateSpline(wave[~bad], residual_ivar[~bad], k=2)
residual_ivar[bad] = fi(wave[bad])
# Now do one more fit to the ratio of data/model - 1.
# Fuss with the knots first ()
inner_knots = arutils.bspline_inner_knots(tck[0])
#
mask, tck_residual = arutils.robust_polyfit(wave, residual, 3, function='bspline', weights=np.sqrt(residual_ivar), knots=inner_knots, **kwargs)
if tck_residual[1].size != tck[1].size:
msgs.error('Problem with bspline knots in bspline_magfit')
#bset_residual = bspline_iterfit(wave, residual, weights=np.sqrt(residual_ivar), knots = tck[0])
tck_log1 = list(tck)
tck_log1[1] = tck[1] + tck_residual[1]
sensfit = 10.0**(0.4*(arutils.func_val(tck_log1,wave, 'bspline')))
absdev = np.median(np.abs(sensfit/modelfit1-1))
msgs.info('Difference between fits is {:g}'.format(absdev))
# QA
msgs.work("Add QA for sensitivity function")
return tck_log1
def simple_calib(slf, det, get_poly=False):
"""Simple calibration algorithm for longslit wavelengths
Uses slf._arcparam to guide the analysis
Parameters
----------
get_poly : bool, optional
Pause to record the polynomial pix = b0 + b1*lambda + b2*lambda**2
Returns
-------
final_fit : dict
Dict of fit info
"""
# Extract the arc
msgs.work("Detecting lines..")
tampl, tcent, twid, w, satsnd, yprep = detect_lines(
slf, det, slf._msarc[det - 1])
# Cut down to the good ones
tcent = tcent[w]
tampl = tampl[w]
msgs.info('Detected {:d} lines in the arc spectrum.'.format(len(w[0])))
# Parameters (just for convenience)
aparm = slf._arcparam[det - 1]
# Read Arc linelist
llist = aparm['llist']
# IDs were input by hand
if len(settings.argflag['arc']['calibrate']['IDpixels']) > 0:
# Check that there are at least 5 values
pixels = np.array(settings.argflag['arc']['calibrate']['IDpixels'])
if np.sum(pixels > 0.) < 5:
msgs.error("Need to give at least 5 pixel values!")
#
msgs.info("Using input lines to seed the wavelength solution")
# Calculate median offset
mdiff = [
np.min(np.abs(tcent - pix))
for pix in settings.argflag['arc']['calibrate']['IDpixels']
]
med_poff = np.median(np.array(mdiff))
msgs.info("Will apply a median offset of {:g} pixels".format(med_poff))
# Match input lines to observed spectrum
nid = len(settings.argflag['arc']['calibrate']['IDpixels'])
idx_str = np.ones(nid).astype(int)
ids = np.zeros(nid)
idsion = np.array([' '] * nid)
gd_str = np.arange(nid).astype(int)
for jj, pix in enumerate(
settings.argflag['arc']['calibrate']['IDpixels']):
diff = np.abs(tcent - pix - med_poff)
if np.min(diff) > 2.:
debugger.set_trace()
msgs.error("No match with input pixel {:g}!".format(pix))
else:
imn = np.argmin(diff)
# Set
idx_str[jj] = imn
# Take wavelength from linelist instead of input value
wdiff = np.abs(llist['wave'] -
settings.argflag['arc']['calibrate']['IDwaves'][jj])
imnw = np.argmin(wdiff)
if wdiff[imnw] > 0.015: # Arbitrary tolerance
msgs.error(
"Input IDwaves={:g} is not in the linelist. Fix".format(
settings.argflag['arc']['calibrate']['IDwaves'][jj]))
else:
ids[jj] = llist['wave'][imnw]
idsion[jj] = llist['Ion'][imnw]
msgs.info("Identifying arc line: {:s} {:g}".format(
idsion[jj], ids[jj]))
else:
# Generate dpix pairs
msgs.info("Using pair algorithm for wavelength solution")
nlist = len(llist)
dpix_list = np.zeros((nlist, nlist))
for kk, row in enumerate(llist):
#dpix_list[kk,:] = (np.array(row['wave'] - llist['wave']))/disp
dpix_list[kk, :] = slf._msarc[det - 1].shape[0] * (
aparm['b1'] * (np.array(row['wave'] - llist['wave'])) +
aparm['b2'] * np.array(row['wave']**2 - llist['wave']**2))
# Lambda pairs for the strongest N lines
srt = np.argsort(tampl)
idx_str = srt[-aparm['Nstrong']:]
idx_str.sort()
dpix_obs = np.zeros((aparm['Nstrong'], aparm['Nstrong']))
for kk, idx in enumerate(idx_str):
dpix_obs[kk, :] = np.array(tcent[idx] - tcent[idx_str])
# Match up (ugly loops)
ids = np.zeros(aparm['Nstrong'])
idsion = np.array([' '] * aparm['Nstrong'])
for kk in range(aparm['Nstrong']):
med_off = np.zeros(nlist)
for ss in range(nlist):
dpix = dpix_list[ss]
min_off = []
for jj in range(aparm['Nstrong']):
min_off.append(np.min(np.abs(dpix_obs[kk, jj] - dpix)))
med_off[ss] = np.median(min_off)
# Set by minimum
idm = np.argmin(med_off)
ids[kk] = llist['wave'][idm]
idsion[kk] = llist['Ion'][idm]
# Calculate disp of the strong lines
disp_str = np.zeros(aparm['Nstrong'])
for kk in range(aparm['Nstrong']):
disp_val = (ids[kk] - ids) / (tcent[idx_str[kk]] - tcent[idx_str])
isf = np.isfinite(disp_val)
disp_str[kk] = np.median(disp_val[isf])
# Consider calculating the RMS with clipping
gd_str = np.where(
np.abs(disp_str - aparm['disp']) /
aparm['disp'] < aparm['disp_toler'])[0]
msgs.info('Found {:d} lines within the dispersion threshold'.format(
len(gd_str)))
if len(gd_str) < 5:
if msgs._debug['arc']:
msgs.warn('You should probably try your best to ID lines now.')
debugger.set_trace()
debugger.plot1d(yprep)
else:
msgs.error('Insufficient lines to auto-fit.')
# Debug
#debug=True
if msgs._debug['arc']:
#tmp = list(gd_str)
#tmp.pop(1)
#gd_str = np.array(tmp)
#xdb.xpcol(tcent[idx_str[gd_str]],ids[gd_str])
#xdb.xplot(tcent[idx_str[gd_str]],ids[gd_str],scatter=True)
# debugger.xplot(yprep)
debugger.set_trace()
msgs.work('Cross correlate here?')
# Setup for fitting
ifit = idx_str[gd_str]
sv_ifit = list(ifit) # Keep the originals
all_ids = -999. * np.ones(len(tcent))
all_idsion = np.array(['12345'] * len(tcent))
all_ids[ifit] = ids[gd_str]
all_idsion[ifit] = idsion[gd_str]
# Fit
n_order = aparm['n_first']
flg_quit = False
fmin, fmax = -1., 1.
msgs.info('Iterative wavelength fitting..')
while (n_order <= aparm['n_final']) and (flg_quit is False):
#msgs.info('n_order={:d}'.format(n_order))
# Fit with rejection
xfit, yfit = tcent[ifit], all_ids[ifit]
mask, fit = arutils.robust_polyfit(xfit,
yfit,
n_order,
function=aparm['func'],
sigma=aparm['nsig_rej'],
minv=fmin,
maxv=fmax)
# Reject but keep originals (until final fit)
ifit = list(ifit[mask == 0]) + sv_ifit
# Find new points (should we allow removal of the originals?)
twave = arutils.func_val(fit,
tcent,
aparm['func'],
minv=fmin,
maxv=fmax)
for ss, iwave in enumerate(twave):
mn = np.min(np.abs(iwave - llist['wave']))
if mn / aparm['disp'] < aparm['match_toler']:
imn = np.argmin(np.abs(iwave - llist['wave']))
#if msgs._debug['arc']:
# print('Adding {:g} at {:g}'.format(llist['wave'][imn],tcent[ss]))
# Update and append
all_ids[ss] = llist['wave'][imn]
all_idsion[ss] = llist['Ion'][imn]
ifit.append(ss)
# Keep unique ones
ifit = np.unique(np.array(ifit, dtype=int))
#if msgs._debug['arc']:
# debugger.set_trace()
# Increment order
if n_order < aparm['n_final']:
n_order += 1
else:
# This does 2 iterations at the final order
flg_quit = True
# Final fit (originals can now be rejected)
fmin, fmax = 0., 1.
xfit, yfit = tcent[ifit] / (slf._msarc[det - 1].shape[0] -
1), all_ids[ifit]
mask, fit = arutils.robust_polyfit(xfit,
yfit,
n_order,
function=aparm['func'],
sigma=aparm['nsig_rej_final'],
minv=fmin,
maxv=fmax) #, debug=True)
irej = np.where(mask == 1)[0]
if len(irej) > 0:
xrej = xfit[irej]
yrej = yfit[irej]
for imask in irej:
msgs.info('Rejecting arc line {:g}'.format(yfit[imask]))
else:
xrej = []
yrej = []
xfit = xfit[mask == 0]
yfit = yfit[mask == 0]
ions = all_idsion[ifit][mask == 0]
#
if msgs._debug['arc']:
msarc = slf._msarc[det - 1]
wave = arutils.func_val(fit,
np.arange(msarc.shape[0]) /
float(msarc.shape[0]),
'legendre',
minv=fmin,
maxv=fmax)
debugger.set_trace()
#debugger.xplot(xfit, np.ones(len(xfit)), scatter=True,
# xtwo=np.arange(msarc.shape[0]),ytwo=yprep)
#debugger.xplot(xfit,yfit, scatter=True, xtwo=np.arange(msarc.shape[0]),
# ytwo=wave)
#debugger.set_trace()
#wave = arutils.func_val(fit, np.arange(msarc.shape[0])/float(msarc.shape[0]),
# 'legendre', min=fmin, max=fmax)
# 2nd order Poly fit for archival
#get_poly=True
if get_poly:
poly_fit = arutils.func_fit(yfit,
xfit,
'polynomial',
2,
minv=fmin,
maxv=fmax)
print(' Most likely you with to record these values:')
print(poly_fit)
debugger.set_trace()
# Pack up fit
final_fit = dict(fitc=fit,
function=aparm['func'],
xfit=xfit,
yfit=yfit,
ions=ions,
fmin=fmin,
fmax=fmax,
xnorm=float(slf._msarc[det - 1].shape[0]),
xrej=xrej,
yrej=yrej,
mask=mask,
spec=yprep,
nrej=aparm['nsig_rej_final'],
shift=0.,
tcent=tcent)
# QA
arqa.arc_fit_qa(slf, final_fit)
# RMS
rms_ang = arutils.calc_fit_rms(xfit,
yfit,
fit,
aparm['func'],
minv=fmin,
maxv=fmax)
wave = arutils.func_val(fit,
np.arange(slf._msarc[det - 1].shape[0]) /
float(slf._msarc[det - 1].shape[0]),
aparm['func'],
minv=fmin,
maxv=fmax)
rms_pix = rms_ang / np.median(np.abs(wave - np.roll(wave, 1)))
msgs.info("Fit RMS = {} pix".format(rms_pix))
# Return
return final_fit
def clean_cr(spectra,
smask,
n_grow_mask=1,
cr_nsig=7.,
nrej_low=5.,
debug=False,
cr_everyn=6,
cr_bsigma=5.,
cr_two_alg='bspline',
**kwargs):
""" Sigma-clips the flux arrays to remove obvious CR
Parameters
----------
spectra :
smask : ndarray
Data mask
n_grow_mask : int, optional
Number of pixels to grow the initial mask by
on each side. Defaults to 1 pixel
cr_nsig : float, optional
Number of sigma for rejection for CRs
Returns
-------
"""
# Init
fluxes, sigs, wave = unpack_spec(spectra)
npix = wave.size
if spectra.nspec == 2:
msgs.info("Only 2 exposures. Using custom procedure")
if cr_two_alg == 'diff':
diff = fluxes[0, :] - fluxes[1, :]
# Robust mean/median
med, mad = arutils.robust_meanstd(diff)
# Spec0?
cr0 = (diff - med) > cr_nsig * mad
if n_grow_mask > 0:
cr0 = grow_mask(cr0, n_grow=n_grow_mask)
msgs.info("Rejecting {:d} CRs in exposure 0".format(np.sum(cr0)))
smask[0, cr0] = True
if debug:
debugger.plot1d(wave,
fluxes[0, :],
xtwo=wave[cr0],
ytwo=fluxes[0, cr0],
mtwo='s')
# Spec1?
cr1 = (-1 * (diff - med)) > cr_nsig * mad
if n_grow_mask > 0:
cr1 = grow_mask(cr1, n_grow=n_grow_mask)
smask[1, cr1] = True
if debug:
debugger.plot1d(wave,
fluxes[1, :],
xtwo=wave[cr1],
ytwo=fluxes[1, cr1],
mtwo='s')
msgs.info("Rejecting {:d} CRs in exposure 1".format(np.sum(cr1)))
elif cr_two_alg == 'ratio':
diff = fluxes[0, :] - fluxes[1, :]
rtio = fluxes[0, :] / fluxes[1, :]
# Robust mean/median
rmed, rmad = arutils.robust_meanstd(rtio)
dmed, dmad = arutils.robust_meanstd(diff)
# Spec0? med, mad = arutils.robust_meanstd(diff)
cr0 = ((rtio - rmed) > cr_nsig * rmad) & (
(diff - dmed) > cr_nsig * dmad)
if n_grow_mask > 0:
cr0 = grow_mask(cr0, n_grow=n_grow_mask)
msgs.info("Rejecting {:d} CRs in exposure 0".format(np.sum(cr0)))
smask[0, cr0] = True
if debug:
debugger.plot1d(wave,
fluxes[0, :],
xtwo=wave[cr0],
ytwo=fluxes[0, cr0],
mtwo='s')
# Spec1?
cr1 = (-1 * (rtio - rmed) > cr_nsig * rmad) & (-1 * (diff - dmed) >
cr_nsig * dmad)
if n_grow_mask > 0:
cr1 = grow_mask(cr1, n_grow=n_grow_mask)
smask[1, cr1] = True
if debug:
debugger.plot1d(wave,
fluxes[1, :],
xtwo=wave[cr1],
ytwo=fluxes[1, cr1],
mtwo='s')
msgs.info("Rejecting {:d} CRs in exposure 1".format(np.sum(cr1)))
elif cr_two_alg == 'bspline':
# Package Data for convenience
waves = spectra.data['wave'].flatten() # Packed 0,1
flux = fluxes.flatten()
sig = sigs.flatten()
#
gd = np.where(sig > 0.)[0]
srt = np.argsort(waves[gd])
idx = gd[srt]
# The following may eliminate bright, narrow emission lines
mask, spl = arutils.robust_polyfit(waves[idx],
flux[idx],
3,
function='bspline',
weights=1. / sig[gd][srt],
sigma=cr_bsigma,
maxone=False,
everyn=cr_everyn)
# Reject CR (with grow)
spec_fit = arutils.func_val(spl, wave, 'bspline')
for ii in range(2):
diff = fluxes[ii, :] - spec_fit
cr = (diff > cr_nsig * sigs[ii, :]) & (sigs[ii, :] > 0.)
if debug:
debugger.plot1d(spectra.data['wave'][0, :],
spectra.data['flux'][ii, :],
spec_fit,
xtwo=spectra.data['wave'][0, cr],
ytwo=spectra.data['flux'][ii, cr],
mtwo='s')
if n_grow_mask > 0:
cr = grow_mask(cr, n_grow=n_grow_mask)
# Mask
smask[ii, cr] = True
msgs.info("Cleaning {:d} CRs in exposure {:d}".format(
np.sum(cr), ii))
# Reject Low
if nrej_low > 0.:
for ii in range(2):
diff = spec_fit - fluxes[ii, :]
rej_low = (diff > nrej_low * sigs[ii, :]) & (sigs[ii, :] >
0.)
if False:
debugger.plot1d(spectra.data['wave'][0, :],
spectra.data['flux'][ii, :],
spec_fit,
xtwo=spectra.data['wave'][0, rej_low],
ytwo=spectra.data['flux'][ii, rej_low],
mtwo='s')
msgs.info(
"Removing {:d} low values in exposure {:d}".format(
np.sum(rej_low), ii))
smask[ii, rej_low] = True
else:
msgs.error("Bad algorithm for combining two spectra!")
# Check
if debug:
gd0 = ~smask[0, :]
gd1 = ~smask[1, :]
debugger.plot1d(wave[gd0],
fluxes[0, gd0],
xtwo=wave[gd1],
ytwo=fluxes[1, gd1])
debugger.set_trace()
else:
# Median of the masked array -- Best for 3 or more spectra
mflux = np.ma.array(fluxes, mask=smask)
refflux = np.ma.median(mflux, axis=0)
diff = fluxes - refflux.filled(0.)
# Loop on spectra
for ispec in range(spectra.nspec):
# Generate ivar
gds = (~smask[ispec, :]) & (sigs[ispec, :] > 0.)
ivar = np.zeros(npix)
ivar[gds] = 1. / sigs[ispec, gds]**2
#
chi2 = diff[ispec]**2 * ivar
badchi = (ivar > 0.0) & (chi2 > cr_nsig**2)
nbad = np.sum(badchi)
if nbad > 0:
# Grow?
if n_grow_mask > 0:
badchi = grow_mask(badchi, n_grow=n_grow_mask)
# Mask
smask[ispec, badchi] = True
msgs.info("Rejecting {:d} CRs in exposure {:d}".format(
nbad, ispec))
# Return
return
def iterative_fitting(spec,
tcent,
ifit,
IDs,
llist,
disp,
plot_fil=None,
verbose=False,
load_pypit=False,
aparm=None):
if aparm is None:
aparm = dict(
llist='',
disp=disp, # Ang/unbinned pixel
disp_toler=0.1, # 10% tolerance
match_toler=3., # Matcing tolerance (pixels)
func='legendre', # Function for fitting
n_first=3, # Order of polynomial for first fit
n_final=4, # Order of polynomial for final fit
nsig_rej=2., # Number of sigma for rejection
nsig_rej_final=3.0) # Number of sigma for rejection (final fit)
# PYPIT
if load_pypit:
from pypit import pyputils
msgs = pyputils.get_dummy_logger()
from pypit import arutils
from pypit import arqa
if load_pypit:
arutils.dummy_settings()
npix = spec.size
# Setup for fitting
sv_ifit = list(ifit) # Keep the originals
all_ids = -999. * np.ones(len(tcent))
all_idsion = np.array(['UNKNWN'] * len(tcent))
all_ids[ifit] = IDs
# Fit
n_order = aparm['n_first']
flg_quit = False
fmin, fmax = -1., 1.
while (n_order <= aparm['n_final']) and (flg_quit is False):
# Fit with rejection
xfit, yfit = tcent[ifit], all_ids[ifit]
mask, fit = arutils.robust_polyfit(xfit,
yfit,
n_order,
function=aparm['func'],
sigma=aparm['nsig_rej'],
minv=fmin,
maxv=fmax)
rms_ang = arutils.calc_fit_rms(xfit[mask == 0],
yfit[mask == 0],
fit,
aparm['func'],
minv=fmin,
maxv=fmax)
rms_pix = rms_ang / disp
if verbose:
print("RMS = {:g}".format(rms_pix))
# DEBUG
# Reject but keep originals (until final fit)
ifit = list(ifit[mask == 0]) + sv_ifit
# Find new points (should we allow removal of the originals?)
twave = arutils.func_val(fit,
tcent,
aparm['func'],
minv=fmin,
maxv=fmax)
for ss, iwave in enumerate(twave):
mn = np.min(np.abs(iwave - llist['wave']))
if mn / aparm['disp'] < aparm['match_toler']:
imn = np.argmin(np.abs(iwave - llist['wave']))
#if verbose:
# print('Adding {:g} at {:g}'.format(llist['wave'][imn],tcent[ss]))
# Update and append
all_ids[ss] = llist['wave'][imn]
all_idsion[ss] = llist['ion'][imn]
ifit.append(ss)
# Keep unique ones
ifit = np.unique(np.array(ifit, dtype=int))
# Increment order
if n_order < (aparm['n_final'] + 2):
n_order += 1
else:
# This does 2 iterations at the final order
flg_quit = True
# Final fit (originals can now be rejected)
fmin, fmax = 0., 1.
xfit, yfit = tcent[ifit] / (npix - 1), all_ids[ifit]
mask, fit = arutils.robust_polyfit(xfit,
yfit,
n_order,
function=aparm['func'],
sigma=aparm['nsig_rej_final'],
minv=fmin,
maxv=fmax) #, debug=True)
irej = np.where(mask == 1)[0]
if len(irej) > 0:
xrej = xfit[irej]
yrej = yfit[irej]
if verbose:
for kk, imask in enumerate(irej):
wave = arutils.func_val(fit,
xrej[kk],
aparm['func'],
minv=fmin,
maxv=fmax)
print('Rejecting arc line {:g}; {:g}'.format(
yfit[imask], wave))
else:
xrej = []
yrej = []
xfit = xfit[mask == 0]
yfit = yfit[mask == 0]
ions = all_idsion[ifit][mask == 0]
# Final RMS
rms_ang = arutils.calc_fit_rms(xfit,
yfit,
fit,
aparm['func'],
minv=fmin,
maxv=fmax)
rms_pix = rms_ang / disp
#
'''
if msgs._debug['arc']:
msarc = slf._msarc[det-1]
wave = arutils.func_val(fit, np.arange(msarc.shape[0])/float(msarc.shape[0]),
'legendre', minv=fmin, maxv=fmax)
debugger.xplot(xfit,yfit, scatter=True,
xtwo=np.arange(msarc.shape[0])/float(msarc.shape[0]),
ytwo=wave)
debugger.xpcol(xfit*msarc.shape[0], yfit)
debugger.set_trace()
'''
#debugger.xplot(xfit, np.ones(len(xfit)), scatter=True,
# xtwo=np.arange(msarc.shape[0]),ytwo=yprep)
#debugger.xplot(xfit,yfit, scatter=True, xtwo=np.arange(msarc.shape[0]),
# ytwo=wave)
#debugger.set_trace()
#wave = arutils.func_val(fit, np.arange(msarc.shape[0])/float(msarc.shape[0]),
# 'legendre', min=fmin, max=fmax)
# Pack up fit
final_fit = dict(fitc=fit,
function=aparm['func'],
xfit=xfit,
yfit=yfit,
ions=ions,
fmin=fmin,
fmax=fmax,
xnorm=float(npix),
xrej=xrej,
yrej=yrej,
mask=mask,
spec=spec,
nrej=aparm['nsig_rej_final'],
shift=0.,
tcent=tcent,
rms=rms_pix)
# QA
if plot_fil is not None:
arqa.arc_fit_qa(None, final_fit, outfil=plot_fil)
# Return
return final_fit
ireg = HAND_FLAG == False
for iiter in range(niter):
xpos1, xerr1 = trace_fweight(image*mask,xfit1[:,ireg], radius = fwhm_vec[iiter])
# Get the indices of the current trace
tracemask = np.zeros_like(xpos0,dtype=int)
xfit1[:,ireg] = 0.0
# Mask out anything that left the image.
off_image = (xpos1 < -0.2*nspat) | (xpos1 > 1.2*nspat)
tracemask[off_image] = 1
xind = (np.fmax(np.fmin(np.rint(xpos1),nspat-1),0)).astype(int)
for iobj in range(nobj):
if specobjs[iobj].HAND_FLAG==False:
# Mask out anything that has left the slit/order.
tracemask[:,iobj] = (thismask[yind, xind[:,iobj]] == False).astype(int)
# ToDO add maxdev functionality?
polymask, coeff_fit1 = robust_polyfit(spec_vec,xpos1[:,iobj], ncoeff, function = 'legendre',
initialmask= tracemask[:,iobj],forceimask=True)
xfit1[:,iobj] = func_val(coeff_fit1, spec_vec, 'legendre')
plt.plot(spec_vec,xpos1[:,iobj],c='k',marker='+',markersize=1.5,linestyle='None')
if (tracemask[:,iobj] == 1).any():
plt.plot(spec_vec[(tracemask[:,iobj] == 1)],xpos1[(tracemask[:,iobj] == 1),iobj],c='r',marker='+',markersize=1.5,linestyle='None')
plt.plot(spec_vec,xfit1[:,iobj],c='orange')
plt.show()
else:
pass
# pos_set1 = xy2traceset(ypos,xpos1, ncoeff=ncoeff,maxdev = 5.0,invvar = xinvvar)
def flex_shift(slf, det, obj_skyspec, arx_skyspec):
""" Calculate shift between object sky spectrum and archive sky spectrum
Parameters
----------
slf
det
obj_skyspec
arx_skyspec
Returns
-------
flex_dict
"""
#Determine the brightest emission lines
msgs.warn("If we use Paranal, cut down on wavelength early on")
arx_amp, arx_cent, arx_wid, arx_w, arx_satsnd, arx_yprep = ararc.detect_lines(
slf, det, msarc=None, censpec=arx_skyspec.flux.value, MK_SATMASK=False)
obj_amp, obj_cent, obj_wid, obj_w, obj_satsnd, obj_yprep = ararc.detect_lines(
slf, det, msarc=None, censpec=obj_skyspec.flux.value, MK_SATMASK=False)
#Keep only 5 brightest amplitude lines (xxx_keep is array of indices within arx_w of the 5 brightest)
arx_keep = np.argsort(arx_amp[arx_w])[-5:]
obj_keep = np.argsort(obj_amp[obj_w])[-5:]
#Calculate wavelength (Angstrom per pixel)
arx_disp = np.append(
arx_skyspec.wavelength.value[1] - arx_skyspec.wavelength.value[0],
arx_skyspec.wavelength.value[1:] - arx_skyspec.wavelength.value[:-1])
#arx_disp = (np.amax(arx_sky.wavelength.value)-np.amin(arx_sky.wavelength.value))/arx_sky.wavelength.size
obj_disp = np.append(
obj_skyspec.wavelength.value[1] - obj_skyspec.wavelength.value[0],
obj_skyspec.wavelength.value[1:] - obj_skyspec.wavelength.value[:-1])
#obj_disp = (np.amax(obj_sky.wavelength.value)-np.amin(obj_sky.wavelength.value))/obj_sky.wavelength.size
#Calculate resolution (lambda/delta lambda_FWHM)..maybe don't need this? can just use sigmas
arx_idx = (arx_cent + 0.5).astype(
np.int)[arx_w][arx_keep] # The +0.5 is for rounding
arx_res = arx_skyspec.wavelength.value[arx_idx]/\
(arx_disp[arx_idx]*(2*np.sqrt(2*np.log(2)))*arx_wid[arx_w][arx_keep])
obj_idx = (obj_cent + 0.5).astype(
np.int)[obj_w][obj_keep] # The +0.5 is for rounding
obj_res = obj_skyspec.wavelength.value[obj_idx]/ \
(obj_disp[obj_idx]*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
#obj_res = (obj_sky.wavelength.value[0]+(obj_disp*obj_cent[obj_w][obj_keep]))/(
# obj_disp*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
msgs.info("Resolution of Archive={:g} and Observation={:g}".format(
np.median(arx_res), np.median(obj_res)))
#Determine sigma of gaussian for smoothing
arx_sig2 = (arx_disp[arx_idx] * arx_wid[arx_w][arx_keep])**2.
obj_sig2 = (obj_disp[obj_idx] * obj_wid[obj_w][obj_keep])**2.
arx_med_sig2 = np.median(arx_sig2)
obj_med_sig2 = np.median(obj_sig2)
if obj_med_sig2 >= arx_med_sig2:
smooth_sig = np.sqrt(obj_med_sig2 - arx_med_sig2) # Ang
smooth_sig_pix = smooth_sig / np.median(arx_disp[arx_idx])
arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix * 2 *
np.sqrt(2 * np.log(2)))
else:
msgs.warn("Prefer archival sky spectrum to have higher resolution")
smooth_sig_pix = 0.
msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#smooth_sig = np.sqrt(arx_med_sig**2-obj_med_sig**2)
#Determine region of wavelength overlap
min_wave = max(np.amin(arx_skyspec.wavelength.value),
np.amin(obj_skyspec.wavelength.value))
max_wave = min(np.amax(arx_skyspec.wavelength.value),
np.amax(obj_skyspec.wavelength.value))
#Smooth higher resolution spectrum by smooth_sig (flux is conserved!)
# if np.median(obj_res) >= np.median(arx_res):
# msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#obj_sky_newflux = ndimage.gaussian_filter(obj_sky.flux, smooth_sig)
# else:
#tmp = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix*2*np.sqrt(2*np.log(2)))
#arx_sky.flux = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# Define wavelengths of overlapping spectra
keep_idx = np.where((obj_skyspec.wavelength.value >= min_wave)
& (obj_skyspec.wavelength.value <= max_wave))[0]
#keep_wave = [i for i in obj_sky.wavelength.value if i>=min_wave if i<=max_wave]
#Rebin both spectra onto overlapped wavelength range
if len(keep_idx) <= 50:
msgs.error("Not enough overlap between sky spectra")
else: #rebin onto object ALWAYS
keep_wave = obj_skyspec.wavelength[keep_idx]
arx_skyspec = arx_skyspec.rebin(keep_wave)
obj_skyspec = obj_skyspec.rebin(keep_wave)
# Trim edges (rebinning is junk there)
arx_skyspec.data['flux'][0, :2] = 0.
arx_skyspec.data['flux'][0, -2:] = 0.
obj_skyspec.data['flux'][0, :2] = 0.
obj_skyspec.data['flux'][0, -2:] = 0.
# Normalize spectra to unit average sky count
norm = np.sum(obj_skyspec.flux.value) / obj_skyspec.npix
obj_skyspec.flux = obj_skyspec.flux / norm
norm2 = np.sum(arx_skyspec.flux.value) / arx_skyspec.npix
arx_skyspec.flux = arx_skyspec.flux / norm2
if (norm < 0.):
msgs.warn("Bad normalization of object in flexure algorithm")
msgs.warn("Will try the median")
norm = np.median(obj_skyspec.flux.value)
if (norm < 0.):
msgs.error("Improper sky spectrum for flexure. Is it too faint??")
if (norm2 < 0.):
msgs.error(
"Bad normalization of archive in flexure. You are probably using wavelengths well beyond the archive."
)
#deal with bad pixels
msgs.work("Need to mask bad pixels")
#deal with underlying continuum
msgs.work("Consider taking median first [5 pixel]")
everyn = obj_skyspec.npix // 20
mask, ct = arutils.robust_polyfit(obj_skyspec.wavelength.value,
obj_skyspec.flux.value,
3,
function='bspline',
sigma=3.,
everyn=everyn)
obj_sky_cont = arutils.func_val(ct, obj_skyspec.wavelength.value,
'bspline')
obj_sky_flux = obj_skyspec.flux.value - obj_sky_cont
mask, ct_arx = arutils.robust_polyfit(arx_skyspec.wavelength.value,
arx_skyspec.flux.value,
3,
function='bspline',
sigma=3.,
everyn=everyn)
arx_sky_cont = arutils.func_val(ct_arx, arx_skyspec.wavelength.value,
'bspline')
arx_sky_flux = arx_skyspec.flux.value - arx_sky_cont
# Consider shaprness filtering (e.g. LowRedux)
msgs.work("Consider taking median first [5 pixel]")
#Cross correlation of spectra
#corr = np.correlate(arx_skyspec.flux, obj_skyspec.flux, "same")
corr = np.correlate(arx_sky_flux, obj_sky_flux, "same")
#Create array around the max of the correlation function for fitting for subpixel max
# Restrict to pixels within maxshift of zero lag
lag0 = corr.size // 2
mxshft = settings.argflag['reduce']['flexure']['maxshift']
max_corr = np.argmax(corr[lag0 - mxshft:lag0 + mxshft]) + lag0 - mxshft
subpix_grid = np.linspace(max_corr - 3., max_corr + 3., 7.)
#Fit a 2-degree polynomial to peak of correlation function
fit = arutils.func_fit(subpix_grid, corr[subpix_grid.astype(np.int)],
'polynomial', 2)
max_fit = -0.5 * fit[1] / fit[2]
#Calculate and apply shift in wavelength
shift = float(max_fit) - lag0
msgs.info("Flexure correction of {:g} pixels".format(shift))
#model = (fit[2]*(subpix_grid**2.))+(fit[1]*subpix_grid)+fit[0]
if msgs._debug['flexure']:
debugger.plot1d(arx_skyspec.wavelength,
arx_sky_flux,
xtwo=np.roll(obj_skyspec.wavelength, int(-1 * shift)),
ytwo=obj_sky_flux)
#debugger.xplot(arx_sky.wavelength, arx_sky.flux, xtwo=np.roll(obj_sky.wavelength.value,9), ytwo=obj_sky.flux*100)
debugger.set_trace()
flex_dict = dict(polyfit=fit,
shift=shift,
subpix=subpix_grid,
corr=corr[subpix_grid.astype(np.int)],
sky_spec=obj_skyspec,
arx_spec=arx_skyspec,
corr_cen=corr.size / 2,
smooth=smooth_sig_pix)
# Return
return flex_dict
def basis(xfit,
yfit,
coeff,
npc,
pnpc,
weights=None,
skipx0=True,
x0in=None,
mask=None,
function='polynomial',
retmask=False):
nrow = xfit.shape[0]
ntrace = xfit.shape[1]
if x0in is None:
x0in = np.arange(float(ntrace))
# Mask out some orders if they are bad
if mask is None or mask.size == 0:
usetrace = np.arange(ntrace)
outmask = np.ones((nrow, ntrace))
else:
usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0]
outmask = np.ones((nrow, ntrace))
outmask[:, mask] = 0.0
# Do the PCA analysis
eigc, hidden = get_pc(coeff[1:npc + 1, usetrace], npc)
modl = arutils.func_vander(xfit[:, 0], function, npc)
eigv = np.dot(modl[:, 1:], eigc)
med_hidden = np.median(hidden, axis=1)
med_highorder = med_hidden.copy()
med_highorder[0] = 0
high_order_matrix = med_highorder.T[np.newaxis, :].repeat(ntrace, axis=0)
# y = hidden[0,:]
# coeff0 = arutils.robust_regression(x0in[usetrace], y, pnpc[1], 0.1, function=function)
# y = hidden[1,:]
# coeff1 = arutils.robust_regression(x0in[usetrace], y, pnpc[2], 0.1, function=function)
coeffstr = []
for i in xrange(1, npc + 1):
# if pnpc[i] == 0:
# coeffstr.append([-9.99E9])
# continue
# coeff0 = arutils.robust_regression(x0in[usetrace], hidden[i-1,:], pnpc[i], 0.1, function=function, min=x0in[0], max=x0in[-1])
if weights is not None:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace],
hidden[i - 1, :],
pnpc[i],
weights=weights[usetrace],
sigma=2.0,
function=function,
minv=x0in[0],
maxv=x0in[-1])
else:
tmask, coeff0 = arutils.robust_polyfit(x0in[usetrace],
hidden[i - 1, :],
pnpc[i],
sigma=2.0,
function=function,
minv=x0in[0],
maxv=x0in[-1])
coeffstr.append(coeff0)
high_order_matrix[:, i - 1] = arutils.func_val(coeff0,
x0in,
function,
minv=x0in[0],
maxv=x0in[-1])
# high_order_matrix[:,1] = arutils.func_val(coeff1, x0in, function)
high_fit = high_order_matrix.copy()
high_order_fit = np.dot(eigv, high_order_matrix.T)
sub = (yfit - high_order_fit) * outmask
numer = np.sum(sub, axis=0)
denom = np.sum(outmask, axis=0)
x0 = np.zeros(float(ntrace))
fitmask = np.zeros(float(ntrace))
#fitmask[mask] = 1
x0fit = np.zeros(float(ntrace))
chisqnu = 0.0
chisqold = 0.0
robust = True
#svx0 = numer/(denom+(denom == 0).astype(np.int))
if not skipx0:
fitmask = (np.abs(denom) > 10).astype(np.int)
if robust:
good = np.where(fitmask != 0)[0]
bad = np.where(fitmask == 0)[0]
x0[good] = numer[good] / denom[good]
imask = np.zeros(float(ntrace))
imask[bad] = 1.0
ttmask, x0res = arutils.robust_polyfit(x0in,
x0,
pnpc[0],
weights=weights,
sigma=2.0,
function=function,
minv=x0in[0],
maxv=x0in[-1],
initialmask=imask)
x0fit = arutils.func_val(x0res,
x0in,
function,
minv=x0in[0],
maxv=x0in[-1])
good = np.where(ttmask == 0)[0]
xstd = 1.0 # This should represent the dispersion in the fit
chisq = ((x0[good] - x0fit[good]) / xstd)**2.0
chisqnu = np.sum(chisq) / np.sum(ttmask)
fitmask = 1.0 - ttmask
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
else:
for i in xrange(1, 5):
good = np.where(fitmask != 0)[0]
x0[good] = numer[good] / denom[good]
# x0res = arutils.robust_regression(x0in[good],x0[good],pnpc[0],0.2,function=function)
x0res = arutils.func_fit(x0in[good],
x0[good],
function,
pnpc[0],
weights=weights,
minv=x0in[0],
maxv=x0in[-1])
x0fit = arutils.func_val(x0res,
x0in,
function,
minv=x0in[0],
maxv=x0in[-1])
chisq = (x0[good] - x0fit[good])**2.0
fitmask[good] *= (chisq < np.sum(chisq) / 2.0).astype(np.int)
chisqnu = np.sum(chisq) / np.sum(fitmask)
msgs.prindent(" Reduced chi-squared = {0:E}".format(chisqnu))
if chisqnu == chisqold:
break
else:
chisqold = chisqnu
if chisqnu > 2.0:
msgs.warn("PCA has very large residuals")
elif chisqnu > 0.5:
msgs.warn("PCA has fairly large residuals")
#bad = np.where(fitmask==0)[0]
#x0[bad] = x0fit[bad]
else:
x0res = 0.0
x3fit = np.dot(eigv, high_order_matrix.T) + np.outer(x0fit,
np.ones(nrow)).T
outpar = dict({
'high_fit': high_fit,
'x0': x0,
'x0in': x0in,
'x0fit': x0fit,
'x0res': x0res,
'x0mask': fitmask,
'hidden': hidden,
'usetrc': usetrace,
'eigv': eigv,
'npc': npc,
'coeffstr': coeffstr
})
if retmask:
return x3fit, outpar, tmask
else:
return x3fit, outpar
def reduce_echelle(slf,
sciframe,
scidx,
fitsdict,
det,
standard=False,
triml=1,
trimr=1,
mspixelflatnrm=None,
doqa=True):
""" Run standard extraction steps on an echelle frame
Parameters
----------
sciframe : image
Bias subtracted image (using arload.load_frame)
scidx : int
Index of the frame
fitsdict : dict
Contains relevant information from fits header files
det : int
Detector index
standard : bool, optional
Standard star frame?
triml : int (optional)
Number of pixels to trim from the left slit edge
trimr : int (optional)
Number of pixels to trim from the right slit edge
"""
msgs.work("Multiprocess this algorithm")
nspec = sciframe.shape[0]
nord = slf._lordloc[det - 1].shape[1]
# Prepare the frames for tracing and extraction
sciframe, rawvarframe, crmask = reduce_prepare(
slf,
sciframe,
scidx,
fitsdict,
det,
mspixelflatnrm=mspixelflatnrm,
standard=standard,
slitprof=slitprof)
bgframe = np.zeros_like(sciframe)
bgnl, bgnr = np.zeros(nord, dtype=np.int), np.zeros(nord, dtype=np.int)
skysub = True
if settings.argflag['reduce']['skysub']['perform']:
# Identify background pixels, and generate an image of the sky spectrum in each slit
for o in range(nord):
word = np.where((slf._slitpix[det - 1] == o + 1)
& (slf._scimask[det - 1] == 0))
if word[0].size == 0:
msgs.warn("There are no pixels in slit {0:d}".format(o + 1))
continue
tbgframe, nl, nr = background_subtraction(slf, sciframe,
rawvarframe, o, det)
bgnl[o], bgnr[o] = nl, nr
bgframe += tbgframe
if nl == 0 and nr == 0:
pass
# If just one slit cannot do sky subtraction, don't do sky subtraction
# msgs.warn("A sky subtraction will not be performed")
# skysub = False
# bgframe = np.zeros_like(sciframe)
# modelvarframe = rawvarframe.copy()
# break
if skysub:
# Provided the for loop above didn't break early, model the variance frame
dnum = settings.get_dnum(det)
modelvarframe = arprocimg.variance_frame(datasec_img,
det,
sciframe,
scidx,
settings.spect[dnum],
fitsdict=fitsdict,
skyframe=bgframe)
else:
modelvarframe = rawvarframe.copy()
bgframe = np.zeros_like(sciframe)
if not standard: # Need to save
slf._modelvarframe[det - 1] = modelvarframe
slf._bgframe[det - 1] = bgframe
# Obtain a first estimate of the object trace then
# fit the traces and perform a PCA for the refinements
trccoeff = np.zeros(
(settings.argflag['trace']['object']['order'] + 1, nord))
trcxfit = np.arange(nspec)
extrap_slit = np.zeros(nord)
for o in range(nord):
trace, error = artrace.trace_weighted(sciframe - bgframe,
slf._lordloc[det - 1][:, o],
slf._rordloc[det - 1][:, o],
mask=slf._scimask[det - 1],
wght="flux")
if trace is None:
extrap_slit[o] = 1
continue
# Find only the good pixels
w = np.where((error != 0.0) & (~np.isnan(error)))
if w[0].size <= 2 * settings.argflag['trace']['object']['order']:
extrap_slit[o] = 1
continue
# Convert the trace locations to be a fraction of the slit length,
# measured from the left slit edge.
trace -= slf._lordloc[det - 1][:, o]
trace /= (slf._rordloc[det - 1][:, o] - slf._lordloc[det - 1][:, o])
try:
msk, trccoeff[:, o] = arutils.robust_polyfit(
trcxfit[w],
trace[w],
settings.argflag['trace']['object']['order'],
function=settings.argflag['trace']['object']['function'],
weights=1.0 / error[w]**2,
minv=0.0,
maxv=nspec - 1.0)
except:
msgs.info("arproc.reduce_echelle")
debugger.set_trace()
refine = 0.0
if settings.argflag['trace']['object']['method'] == "pca":
# Identify the orders to be extrapolated during reconstruction
orders = 1.0 + np.arange(nord)
msgs.info("Performing a PCA on the object trace")
ofit = settings.argflag['trace']['object']['params']
lnpc = len(ofit) - 1
maskord = np.where(extrap_slit == 1)[0]
xcen = trcxfit[:, np.newaxis].repeat(nord, axis=1)
trccen = arutils.func_val(
trccoeff,
trcxfit,
settings.argflag['trace']['object']['function'],
minv=0.0,
maxv=nspec - 1.0).T
if np.sum(1.0 - extrap_slit) > ofit[0] + 1:
fitted, outpar = arpca.basis(
xcen,
trccen,
trccoeff,
lnpc,
ofit,
skipx0=False,
mask=maskord,
function=settings.argflag['trace']['object']['function'])
if doqa:
# arqa.pca_plot(slf, outpar, ofit, "Object_Trace", pcadesc="PCA of object trace")
arpca.pca_plot(slf.setup,
outpar,
ofit,
"Object_Trace",
pcadesc="PCA of object trace")
# Extrapolate the remaining orders requested
trccen, outpar = arpca.extrapolate(
outpar,
orders,
function=settings.argflag['trace']['object']['function'])
#refine = trccen-trccen[nspec//2, :].reshape((1, nord))
else:
msgs.warn("Could not perform a PCA on the object trace" +
msgs.newline() + "Not enough well-traced orders")
msgs.info("Using direct determination of the object trace instead")
pass
else:
msgs.error("Not ready for object trace method:" + msgs.newline() +
settings.argflag['trace']['object']['method'])
# Construct the left and right traces of the object profile
# The following code ensures that the fraction of the slit
# containing the object remains constant along the spectral
# direction
trcmean = np.mean(trccen, axis=0)
trobjl = (trcmean -
(1 + bgnl) / slf._pixwid[det - 1].astype(np.float)).reshape(
(1, nord)).repeat(nspec, axis=0)
trobjl = trccen - trobjl
trobjr = (-trcmean + (slf._pixwid[det - 1] - bgnr - 1) /
slf._pixwid[det - 1].astype(np.float)).reshape(
(1, nord)).repeat(nspec, axis=0)
trobjr = trccen + trobjr
# Convert trccen to the actual trace locations
trccen *= (slf._rordloc[det - 1] - slf._lordloc[det - 1])
trccen += slf._lordloc[det - 1]
trobjl *= (slf._rordloc[det - 1] - slf._lordloc[det - 1])
trobjl += slf._lordloc[det - 1]
trobjr *= (slf._rordloc[det - 1] - slf._lordloc[det - 1])
trobjr += slf._lordloc[det - 1]
# Generate an image of pixel weights for each object. Each weight can
# take any floating point value from 0 to 1 (inclusive). For the rec_obj_img,
# a weight of 1 means that the pixel is fully contained within the object
# region, and 0 means that the pixel is fully contained within the background
# region. The opposite is true for the rec_bg_img array. A pixel that is on
# the border of object/background is assigned a value between 0 and 1.
msgs.work(
"Eventually allow ARMED to find multiple objects in the one slit")
nobj = 1
rec_obj_img = np.zeros(sciframe.shape + (nobj, ))
rec_bg_img = np.zeros(sciframe.shape + (nobj, ))
for o in range(nord):
# Prepare object/background regions
objl = np.array([bgnl[o]])
objr = np.array([slf._pixwid[det - 1][o] - bgnr[o] - triml - trimr])
bckl = np.zeros((slf._pixwid[det - 1][o] - triml - trimr, 1))
bckr = np.zeros((slf._pixwid[det - 1][o] - triml - trimr, 1))
bckl[:bgnl[o]] = 1
if bgnr[o] != 0:
bckr[-bgnr[o]:] = 1
tobj_img, tbg_img = artrace.trace_objbg_image(slf,
det,
sciframe - bgframe,
o, [objl, objr],
[bckl, bckr],
triml=triml,
trimr=trimr)
rec_obj_img += tobj_img
rec_bg_img += tbg_img
# Create trace dict
scitrace = artrace.trace_object_dict(nobj,
trccen[:,
0].reshape(trccen.shape[0], 1),
object=rec_obj_img,
background=rec_bg_img)
for o in range(1, nord):
scitrace = artrace.trace_object_dict(nobj,
trccen[:, o].reshape(
trccen.shape[0], 1),
tracelist=scitrace)
# Save the quality control
if doqa:
artrace.obj_trace_qa(slf,
sciframe,
trobjl,
trobjr,
None,
det,
root="object_trace",
normalize=False)
# Finalize the Sky Background image
if settings.argflag['reduce']['skysub']['perform'] and (nobj >
0) and skysub:
msgs.info("Finalizing the sky background image")
# Identify background pixels, and generate an image of the sky spectrum in each slit
bgframe = np.zeros_like(sciframe)
for o in range(nord):
tbgframe, nl, nr = background_subtraction(slf,
sciframe,
rawvarframe,
o,
det,
refine=refine)
bgnl[o], bgnr[o] = nl, nr
bgframe += tbgframe
modelvarframe = arprocimg.variance_frame(datasec_img,
det,
sciframe,
scidx,
settings.spect[dnum],
fitsdict=fitsdict,
skyframe=bgframe)
# Perform an optimal extraction
return reduce_frame(slf,
sciframe,
rawvarframe,
modelvarframe,
bgframe,
scidx,
fitsdict,
det,
crmask,
scitrace=scitrace,
standard=standard)
def obj_profiles(slf,
det,
specobjs,
sciframe,
varframe,
skyframe,
crmask,
scitrace,
COUNT_LIM=25.,
doqa=True,
pickle_file=None):
""" Derive spatial profiles for each object
Parameters
----------
slf
det
specobjs
sciframe
varframe
skyframe
crmask
scitrace
Returns
-------
"""
''' FOR DEVELOPING
import pickle
if False:
tilts = slf._tilts[det-1]
args = [det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts]
msgs.warn("Pickling in the profile code")
with open("trc_pickle.p",'wb') as f:
pickle.dump(args,f)
debugger.set_trace()
if pickle_file is not None:
f = open(pickle_file,'r')
args = pickle.load(f)
f.close()
det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts = args
slf = None
else:
tilts = slf._tilts[det-1]
'''
# Init QA
#
sigframe = np.sqrt(varframe)
# Loop on slits
for sl in range(len(specobjs)):
# Loop on objects
nobj = scitrace[sl]['traces'].shape[1]
scitrace[sl]['opt_profile'] = []
msgs.work("Should probably loop on S/N")
for o in range(nobj):
msgs.info(
"Deriving spatial profile of object {0:d}/{1:d} in slit {2:d}/{3:d}"
.format(o + 1, nobj, sl + 1, len(specobjs)))
# Get object pixels
if scitrace[sl]['background'] is None:
# The object for all slits is provided in the first extension
objreg = np.copy(scitrace[0]['object'][:, :, o])
wzro = np.where(slf._slitpix[det - 1] != sl + 1)
objreg[wzro] = 0.0
else:
objreg = scitrace[sl]['object'][:, :, o]
# Calculate slit image
slit_img = artrace.slit_image(slf, det, scitrace[sl],
o) #, tilts=tilts)
# Object pixels
weight = objreg.copy()
# Identify good rows
gdrow = np.where(specobjs[sl][o].boxcar['counts'] > COUNT_LIM)[0]
# Normalized image
norm_img = sciframe / np.outer(specobjs[sl][o].boxcar['counts'],
np.ones(sciframe.shape[1]))
# Eliminate rows with CRs (wipes out boxcar)
crspec = np.sum(crmask * weight, axis=1)
cr_rows = np.where(crspec > 0)[0]
weight[cr_rows, :] = 0.
#
if len(gdrow) > 100: # Good S/N regime
msgs.info("Good S/N for profile")
# Eliminate low count regions
badrow = np.where(
specobjs[sl][o].boxcar['counts'] < COUNT_LIM)[0]
weight[badrow, :] = 0.
# Extract profile
gdprof = (weight > 0) & (sigframe > 0.) & (
~np.isnan(slit_img)
) # slit_img=nan if the slit is partially on the chip
slit_val = slit_img[gdprof]
flux_val = norm_img[gdprof]
#weight_val = sciframe[gdprof]/sigframe[gdprof] # S/N
weight_val = 1. / sigframe[gdprof] # 1/N
msgs.work(
"Weight by S/N in boxcar extraction? [avoid CRs; smooth?]")
# Fit
fdict = dict(
func=settings.argflag['science']['extraction']['profile'],
deg=3,
extrap=False)
if fdict['func'] == 'gaussian':
fdict['deg'] = 2
elif fdict['func'] == 'moffat':
fdict['deg'] = 3
else:
msgs.error("Not ready for this type of object profile")
msgs.work(
"Might give our own guess here instead of using default")
guess = None
# Check if there are enough pixels in the slit to perform fit
if slit_val.size <= fdict['deg'] + 1:
msgs.warn(
"Not enough pixels to determine profile of object={0:s} in slit {1:d}"
.format(specobjs[sl][o].idx, sl + 1) + msgs.newline() +
"Skipping Optimal")
fdict['extrap'] = True
scitrace[sl]['opt_profile'].append(copy.deepcopy(fdict))
continue
# Fit the profile
try:
mask, gfit = arutils.robust_polyfit(slit_val,
flux_val,
fdict['deg'],
function=fdict['func'],
weights=weight_val,
maxone=False,
guesses=guess)
except RuntimeError:
msgs.warn("Bad Profile fit for object={:s}." +
msgs.newline() +
"Skipping Optimal".format(specobjs[sl][o].idx))
fdict['extrap'] = True
scitrace[sl]['opt_profile'].append(copy.deepcopy(fdict))
continue
except ValueError:
debugger.set_trace() # NaNs in the values? Check
msgs.work("Consider flagging/removing CRs here")
# Record
fdict['param'] = gfit.copy()
fdict['mask'] = mask
fdict['slit_val'] = slit_val
fdict['flux_val'] = flux_val
scitrace[sl]['opt_profile'].append(copy.deepcopy(fdict))
specobjs[sl][o].optimal['fwhm'] = fdict['param'][1] # Pixels
if msgs._debug['obj_profile']:
gdp = mask == 0
mn = np.min(slit_val[gdp])
mx = np.max(slit_val[gdp])
xval = np.linspace(mn, mx, 1000)
model = arutils.func_val(gfit, xval, fdict['func'])
import matplotlib.pyplot as plt
plt.clf()
ax = plt.gca()
ax.scatter(slit_val[gdp],
flux_val[gdp],
marker='.',
s=0.7,
edgecolor='none',
facecolor='black')
ax.plot(xval, model, 'b')
# Gaussian too?
if False:
fdictg = dict(func='gaussian', deg=2)
maskg, gfitg = arutils.robust_polyfit(
slit_val,
flux_val,
fdict['deg'],
function=fdictg['func'],
weights=weight_val,
maxone=False)
modelg = arutils.func_val(gfitg, xval, fdictg['func'])
ax.plot(xval, modelg, 'r')
plt.show()
debugger.set_trace()
elif len(gdrow) > 10: #
msgs.warn(
"Low extracted flux for obj={:s} in slit {:d}. Not ready for Optimal"
.format(specobjs[sl][o].idx, sl + 1))
scitrace[sl]['opt_profile'].append({})
continue
elif len(gdrow) >= 0: # limit is ">= 0" to avoid crash for gdrow=0
msgs.warn(
"Low extracted flux for obj={:s} in slit {:d}. Not ready for Optimal"
.format(specobjs[sl][o].idx, sl + 1))
scitrace[sl]['opt_profile'].append({})
continue
# QA
if doqa: #not msgs._debug['no_qa'] and doqa:
msgs.info("Preparing QA for spatial object profiles")
arqa.obj_profile_qa(slf, specobjs, scitrace, det)
return
tracemask = np.zeros_like(xpos0, dtype=int)
xfit1[:, ireg] = 0.0
# Mask out anything that left the image.
off_image = (xpos1 < -0.2 * nspat) | (xpos1 > 1.2 * nspat)
tracemask[off_image] = 1
xind = (np.fmax(np.fmin(np.rint(xpos1), nspat - 1), 0)).astype(int)
for iobj in range(nobj):
if specobjs[iobj].HAND_FLAG == False:
# Mask out anything that has left the slit/order.
tracemask[:, iobj] = (thismask[yind,
xind[:, iobj]] == False).astype(int)
# ToDO add maxdev functionality?
polymask, coeff_fit1 = robust_polyfit(spec_vec,
xpos1[:, iobj],
ncoeff,
function='legendre',
initialmask=tracemask[:,
iobj],
forceimask=True)
xfit1[:, iobj] = func_val(coeff_fit1, spec_vec, 'legendre')
plt.plot(spec_vec,
xpos1[:, iobj],
c='k',
marker='+',
markersize=1.5,
linestyle='None')
if (tracemask[:, iobj] == 1).any():
plt.plot(spec_vec[(tracemask[:, iobj] == 1)],
xpos1[(tracemask[:, iobj] == 1), iobj],
c='r',
marker='+',
def obj_profiles(slf, det, specobjs, sciframe, varframe, skyframe, crmask,
scitrace, COUNT_LIM=25., pickle_file=None):
""" Derive spatial profiles for each object
Parameters
----------
slf
det
specobjs
sciframe
varframe
skyframe
crmask
scitrace
Returns
-------
"""
''' FOR DEVELOPING
import pickle
if False:
tilts = slf._tilts[det-1]
args = [det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts]
msgs.warn("Pickling in the profile code")
with open("trc_pickle.p",'wb') as f:
pickle.dump(args,f)
debugger.set_trace()
if pickle_file is not None:
f = open(pickle_file,'r')
args = pickle.load(f)
f.close()
det, specobjs, sciframe, varframe, skyframe, crmask, scitrace, tilts = args
slf = None
else:
tilts = slf._tilts[det-1]
'''
# Init QA
#
sigframe = np.sqrt(varframe)
# Loop
nobj = scitrace['traces'].shape[1]
scitrace['opt_profile'] = []
msgs.work("Should probably loop on S/N")
for o in range(nobj):
# Calculate slit image
slit_img = artrace.slit_image(slf, det, scitrace, o)#, tilts=tilts)
# Object pixels
weight = scitrace['object'][:,:,o].copy()
# Identify good rows
gdrow = np.where(specobjs[o].boxcar['counts'] > COUNT_LIM)[0]
# Normalized image
norm_img = sciframe / np.outer(specobjs[o].boxcar['counts'], np.ones(sciframe.shape[1]))
# Eliminate rows with CRs (wipes out boxcar)
crspec = np.sum(crmask*weight,axis=1)
cr_rows = np.where(crspec > 0)[0]
weight[cr_rows,:] = 0.
#
if len(gdrow) > 100: # Good S/N regime
msgs.info("Good S/N for profile")
# Eliminate low count regions
badrow = np.where(specobjs[o].boxcar['counts'] < COUNT_LIM)[0]
weight[badrow,:] = 0.
# Extract profile
gdprof = (weight > 0) & (sigframe > 0.)
slit_val = slit_img[gdprof]
flux_val = norm_img[gdprof]
#weight_val = sciframe[gdprof]/sigframe[gdprof] # S/N
weight_val = 1./sigframe[gdprof] # 1/N
msgs.work("Weight by S/N in boxcar extraction? [avoid CRs; smooth?]")
# Fit
fdict = dict(func=slf._argflag['science']['extraction']['profile'], deg=3)
if fdict['func'] == 'gaussian':
fdict['deg'] = 2
elif fdict['func'] == 'moffat':
fdict['deg'] = 3
else:
msgs.error("Not ready for this type of object profile")
msgs.work("Might give our own guess here instead of using default")
guess = None
try:
mask, gfit = arutils.robust_polyfit(slit_val, flux_val, fdict['deg'], function=fdict['func'], weights=weight_val, maxone=False, guesses=guess)
except RuntimeError:
msgs.warn("Bad Profile fit for object={:s}. Skipping Optimal".format(specobjs[o].idx))
scitrace['opt_profile'].append(fdict)
continue
except ValueError:
debugger.set_trace() # NaNs in the values? Check
msgs.work("Consider flagging/removing CRs here")
# Record
fdict['param'] = gfit
fdict['mask'] = mask
fdict['slit_val'] = slit_val
fdict['flux_val'] = flux_val
scitrace['opt_profile'].append(fdict)
if msgs._debug['obj_profile']: #
gdp = mask == 0
mn = np.min(slit_val[gdp])
mx = np.max(slit_val[gdp])
xval = np.linspace(mn,mx,1000)
model = arutils.func_val(gfit, xval, fdict['func'])
import matplotlib.pyplot as plt
plt.clf()
ax = plt.gca()
ax.scatter(slit_val[gdp], flux_val[gdp], marker='.', s=0.7, edgecolor='none', facecolor='black')
ax.plot(xval, model, 'b')
# Gaussian too?
if False:
fdictg = dict(func='gaussian', deg=2)
maskg, gfitg = arutils.robust_polyfit(slit_val, flux_val, fdict['deg'], function=fdictg['func'], weights=weight_val, maxone=False)
modelg = arutils.func_val(gfitg, xval, fdictg['func'])
ax.plot(xval, modelg, 'r')
plt.show()
debugger.set_trace()
elif len(gdrow) > 10: #
msgs.warn("Low extracted flux for obj={:s}. Not ready for Optimal".format(specobjs[o].idx))
scitrace['opt_profile'].append({})
continue
# QA
arqa.obj_profile_qa(slf, specobjs, scitrace)
return scitrace['opt_profile']
