def test_calc_ivar():
""" Run the parameter setup script
"""
x = np.array([-1.0, -0.1, 0.0, 0.1, 1.0])
res = arut.calc_ivar(x)
assert np.array_equal(res, np.array([0.0, 0.0, 0.0, 10.0, 1.0]))
assert np.array_equal(arut.calc_ivar(res),
np.array([0.0, 0.0, 0.0, 0.1, 1.0]))
def save_obj_info(slf, fitsdict, clobber=True):
# Lists for a Table
slits, names, boxsize, opt_fwhm, s2n = [], [], [], [], []
# Loop on detectors
for kk in range(settings.spect['mosaic']['ndet']):
det = kk + 1
if slf._specobjs[det - 1] is None:
continue
dnum = settings.get_dnum(det)
# Loop on slits
for sl in range(len(slf._specobjs[det - 1])):
# Loop on spectra
for specobj in slf._specobjs[det - 1][sl]:
# Append
names.append(specobj.idx)
slits.append(sl + 1)
# Boxcar width
if 'size' in specobj.boxcar.keys():
slit_pix = specobj.boxcar['size']
# Convert to arcsec
binspatial, binspectral = settings.parse_binning(
fitsdict['binning'][specobj.scidx])
boxsize.append(slit_pix * binspatial *
settings.spect[dnum]['platescale'])
else:
boxsize.append(0.)
# Optimal profile (FWHM)
if 'fwhm' in specobj.optimal.keys():
binspatial, binspectral = settings.parse_binning(
fitsdict['binning'][specobj.scidx])
opt_fwhm.append(specobj.optimal['fwhm'] * binspatial *
settings.spect[dnum]['platescale'])
else:
opt_fwhm.append(0.)
# S2N -- default to boxcar
sext = (specobj.boxcar if
(len(specobj.boxcar) > 0) else specobj.optimal)
ivar = arutils.calc_ivar(sext['var'])
is2n = np.median(sext['counts'] * np.sqrt(ivar))
s2n.append(is2n)
# Generate the table, if we have at least one source
if len(names) > 0:
obj_tbl = Table()
obj_tbl['slit'] = slits
obj_tbl['slit'].format = 'd'
obj_tbl['name'] = names
obj_tbl['box_width'] = boxsize
obj_tbl['box_width'].format = '.2f'
obj_tbl['box_width'].unit = u.arcsec
obj_tbl['opt_fwhm'] = opt_fwhm
obj_tbl['opt_fwhm'].format = '.3f'
obj_tbl['opt_fwhm'].unit = u.arcsec
obj_tbl['s2n'] = s2n
obj_tbl['s2n'].format = '.2f'
# Write
obj_tbl.write(settings.argflag['run']['directory']['science'] +
'/objinfo_{:s}.txt'.format(slf._basename),
format='ascii.fixed_width',
overwrite=True)
def background_subtraction(slf,
sciframe,
varframe,
slitn,
det,
refine=0.0,
doqa=True):
""" Generate a frame containing the background sky spectrum
Parameters
----------
slf : Class
Science Exposure Class
sciframe : ndarray
science frame
varframe : ndarray
variance frame
slitn : int
Slit number
det : int
Detector index
refine : float or ndarray
refine the object traces. This should be a small value around 0.0.
If a float, a constant offset will be applied.
Otherwise, an array needs to be specified of the same length as
sciframe.shape[0] that contains the refinement of each pixel along
the spectral direction.
Returns
-------
bgframe : ndarray
An image, the same size as sciframe, that contains
the background spectrum within the specified slit.
nl : int
number of pixels from the left slit edge to use as background pixels
nr : int
number of pixels from the right slit edge to use as background pixels
"""
# Obtain all pixels that are within the slit edges, and are not masked
word = np.where((slf._slitpix[det - 1] == slitn + 1)
& (slf._scimask[det - 1] == 0))
if word[0].size == 0:
msgs.warn("There are no pixels in slit {0:d}".format(slitn))
debugger.set_trace()
nl, nr = 0, 0
return np.zeros_like(sciframe), nl, nr
# Calculate the oversampled object profiles
oversampling_factor = 3 # should be an integer according to the description in object_profile()
xedges, modvals = object_profile(slf,
sciframe,
slitn,
det,
refine=refine,
factor=oversampling_factor)
bincent = 0.5 * (xedges[1:] + xedges[:-1])
npix = slf._pixwid[det - 1][slitn]
tilts = slf._tilts[det - 1].copy()
lordloc = slf._lordloc[det - 1][:, slitn]
rordloc = slf._rordloc[det - 1][:, slitn]
# For each pixel, calculate the fraction along the slit's spatial direction
spatval = (word[1] - lordloc[word[0]] + refine) / (rordloc[word[0]] -
lordloc[word[0]])
# Cumulative sum and normalize
csum = np.cumsum(modvals)
csum -= csum[0]
csum /= csum[-1]
# Find a first guess of the edges of the object profile - assume this is the innermost 90 percent of the flux
argl = np.argmin(np.abs(csum - 0.05))
argr = np.argmin(np.abs(csum - 0.95))
# Considering the possible background pixels that are left of the object,
# find the first time where the object profile no longer decreases as you
# move toward the edge of the slit. This is the beginning of the noisy
# object profile, which is where the object can no longer be distinguished
# from the background.
wl = np.where((modvals[1:] < modvals[:-1]) & (bincent[1:] < bincent[argl]))
wr = np.where((modvals[1:] > modvals[:-1]) & (bincent[1:] > bincent[argr]))
nl, nr = 0, 0
if wl[0].size != 0:
# This is the index of the first time where the object profile
# no longer decreases as you move towards the slit edge
nl_index = np.max(wl[0])
# Calculate nl, defined as:
# "number of pixels from the left slit edge to use as background pixels",
# which is just nl_index with the sampling factor taken out
nl_index_origscale = int(nl_index / oversampling_factor + 0.5)
nl = nl_index_origscale
if wr[0].size != 0:
# This is the index of the first time where the object profile
# no longer decreases as you move towards the slit edge
nr_index = np.min(wr[0])
# Calculate nr, defined as:
# "number of pixels from the right slit edge to use as background pixels",
# which is npix minus nr_index with the sampling factor taken out
nr_index_origscale = int(nr_index / oversampling_factor + 0.5)
nr = npix - nr_index_origscale
if nl + nr < 5:
msgs.warn(
"The object profile appears to extrapolate to the edge of the slit"
)
msgs.info(
"A background subtraction will not be performed for slit {0:d}".
format(slitn + 1))
nl, nr = 0, 0
return np.zeros_like(sciframe), nl, nr
# Find background pixels and fit
wbgpix_spatval = np.where(
(spatval <= float(nl) / npix) |
(spatval >= float(npix - nr) /
npix)) # this cannot be used to index the 2D array tilts
wbgpix = (word[0][wbgpix_spatval], word[1][wbgpix_spatval]
) # this may be approproate for indexing the 2D array tilts
if settings.argflag['reduce']['skysub']['method'].lower() == 'bspline':
msgs.info("Using bspline sky subtraction")
srt = np.argsort(tilts[wbgpix])
ivar = arutils.calc_ivar(varframe)
# Perform a weighted b-spline fit to the sky background pixels
mask, bspl = arutils.robust_polyfit(
tilts[wbgpix][srt],
sciframe[wbgpix][srt],
3,
function='bspline',
weights=np.sqrt(ivar)[wbgpix][srt],
sigma=5.,
maxone=False,
**settings.argflag['reduce']['skysub']['bspline'])
bgf_flat = arutils.func_val(bspl, tilts.flatten(), 'bspline')
bgframe = bgf_flat.reshape(tilts.shape)
if doqa:
plt_bspline_sky(tilts, sciframe, bgf_flat, gdp)
debugger.set_trace()
else:
msgs.error('Not ready for this method for skysub {:s}'.format(
settings.argflag['reduce']['skysub']['method'].lower()))
if np.any(np.isnan(bgframe)):
msgs.warn("NAN in bgframe. Replacing with 0")
bad = np.isnan(bgframe)
bgframe[bad] = 0.
return bgframe, nl, nr
def coadd_spectra(spectra,
wave_grid_method='concatenate',
niter=5,
scale_method='auto',
do_offset=False,
sigrej_final=3.,
do_var_corr=True,
qafile=None,
outfile=None,
do_cr=True,
**kwargs):
"""
Parameters
----------
spectra : XSpectrum1D
wave_grid_method :
Returns
-------
spec1d : XSpectrum1D
"""
# Init
if niter <= 0:
msgs.error('Not prepared for zero iterations')
# Single spectrum?
if spectra.nspec == 1:
msgs.info('Only one spectrum. Writing, as desired, and ending..')
if outfile is not None:
write_to_disk(spectra, outfile)
return spectra
# Final wavelength array
new_wave = new_wave_grid(spectra.data['wave'],
method=wave_grid_method,
**kwargs)
# Rebin
rspec = spectra.rebin(new_wave * u.AA,
all=True,
do_sig=True,
grow_bad_sig=True,
masking='none')
# Define mask -- THIS IS THE ONLY ONE TO USE
rmask = rspec.data['sig'].filled(0.) <= 0.
# S/N**2, weights
sn2, weights = sn_weight(rspec, rmask)
# Scale (modifies rspec in place)
scales, omethod = scale_spectra(rspec,
rmask,
sn2,
scale_method=scale_method,
**kwargs)
# Clean bad CR :: Should be run *after* scaling
if do_cr:
clean_cr(rspec, rmask, **kwargs)
# Initial coadd
spec1d = one_d_coadd(rspec, rmask, weights)
# Init standard deviation
std_dev, _ = get_std_dev(rspec, rmask, spec1d, **kwargs)
msgs.info("Initial std_dev = {:g}".format(std_dev))
iters = 0
std_dev = 0.
var_corr = 1.
# Scale the standard deviation
while np.absolute(std_dev - 1.) >= 0.1 and iters < niter:
iters += 1
msgs.info("Iterating on coadding... iter={:d}".format(iters))
# Setup (strip out masks, if any)
tspec = spec1d.copy()
tspec.unmask()
newvar = tspec.data['sig'][0, :].filled(
0.)**2 # JFH Interpolates over bad values?
newflux = tspec.data['flux'][0, :].filled(0.)
newflux_now = newflux # JFH interpolates
# Convenient for coadding
uspec = rspec.copy()
uspec.unmask()
# Loop on images to update noise model for rejection
for qq in range(rspec.nspec):
# Grab full spectrum (unmasked)
flux = uspec.data['flux'][qq, :].filled(0.)
sig = uspec.data['sig'][qq, :].filled(0.)
ivar = np.zeros_like(sig)
gd = sig > 0.
ivar[gd] = 1. / sig[gd]**2
# var_tot
var_tot = newvar + arutils.calc_ivar(ivar)
ivar_real = arutils.calc_ivar(var_tot)
# smooth out possible outliers in noise
var_med = medfilt(var_tot, 5)
var_smooth = medfilt(var_tot, 99) #, boundary = 'reflect')
# conservatively always take the largest variance
var_final = np.maximum(var_med, var_smooth)
ivar_final = arutils.calc_ivar(var_final)
# Cap S/N ratio at SN_MAX to prevent overly aggressive rejection
SN_MAX = 20.0
ivar_cap = np.minimum(ivar_final, (SN_MAX / newflux_now +
(newflux_now <= 0.0))**2)
#; adjust rejection to reflect the statistics of the distribtuion
#; of errors. This fixes cases where for not totally understood
#; reasons the noise model is not quite right and
#; many pixels are rejected.
#; Is the model offset relative to the data? If so take it out
if do_offset:
diff1 = flux - newflux_now
#idum = np.where(arrmask[*, j] EQ 0, nnotmask)
debugger.set_trace() # GET THE MASK RIGHT!
nnotmask = np.sum(~mask)
nmed_diff = np.maximum(nnotmask // 20, 10)
#; take out the smoothly varying piece
#; JXP -- This isnt going to work well if the data has a bunch of
#; null values in it
w = np.ones(5, 'd')
diff_sm = np.convolve(w / w.sum(),
medfilt(diff1 * (~mask), nmed_diff),
mode='same')
chi2 = (diff1 - diff_sm)**2 * ivar_real
# debugger.set_trace()
goodchi = (~mask) & (ivar_real > 0.0) & (chi2 <= 36.0
) # AND masklam, ngd)
if np.sum(goodchi) == 0:
goodchi = np.array([True] * flux.size)
# debugger.set_trace() # Port next line to Python to use this
#djs_iterstat, (arrflux[goodchi, j]-newflux_now[goodchi]) $
# , invvar = ivar_real[goodchi], mean = offset_mean $
# , median = offset $
else:
offset = 0.
chi2 = (flux - newflux_now - offset)**2 * ivar_real
goodchi = (~rmask[qq, :]) & (ivar_real > 0.0) & (
chi2 <= 36.0) # AND masklam, ngd)
ngd = np.sum(goodchi)
if ngd == 0:
goodchi = np.array([True] * flux.size)
#; evalute statistics of chi2 for good pixels and excluding
#; extreme 6-sigma outliers
chi2_good = chi2[goodchi]
chi2_srt = chi2_good.copy()
chi2_srt.sort()
#; evaluate at 1-sigma and then scale
gauss_prob = 1.0 - 2.0 * (1. - scipy.stats.norm.cdf(1.)
) #gaussint(-double(1.0d))
sigind = int(np.round(gauss_prob * ngd))
chi2_sigrej = chi2_srt[sigind]
one_sigma = np.minimum(np.maximum(np.sqrt(chi2_sigrej), 1.0), 5.0)
sigrej_eff = sigrej_final * one_sigma
chi2_cap = (flux - newflux_now - offset)**2 * ivar_cap
# Grow??
chi_mask = (chi2_cap > sigrej_eff**2) & (~rmask[qq, :])
nrej = np.sum(chi_mask)
# Apply
if nrej > 0:
msgs.info("Rejecting {:d} pixels in exposure {:d}".format(
nrej, qq))
print(rspec.data['wave'][qq, chi_mask])
rmask[qq, chi_mask] = True
#rspec.select = qq
#rspec.add_to_mask(chi_mask)
#outmask[*, j] = (arrmask[*, j] EQ 1) OR (chi2_cap GT sigrej_eff^2)
# Incorporate saving of each dev/sig panel onto one page? Currently only saves last fit
#qa_plots(wavelengths, masked_fluxes, masked_vars, new_wave, new_flux, new_var)
# Coadd anew
spec1d = one_d_coadd(rspec, rmask, weights, **kwargs)
# Calculate std_dev
std_dev, _ = get_std_dev(rspec, rmask, spec1d, **kwargs)
#var_corr = var_corr * std_dev
msgs.info("Desired variance correction: {:g}".format(var_corr))
msgs.info("New standard deviation: {:g}".format(std_dev))
if do_var_corr:
msgs.info("Correcting variance")
for ispec in range(rspec.nspec):
rspec.data['sig'][ispec] *= np.sqrt(std_dev)
spec1d = one_d_coadd(rspec, rmask, weights)
if iters == 0:
msgs.warn("No iterations on coadding done")
#qa_plots(wavelengths, masked_fluxes, masked_vars, new_wave, new_flux, new_var)
else: #if iters > 0:
msgs.info(
"Final correction to initial variances: {:g}".format(var_corr))
# QA
if qafile is not None:
msgs.info("Writing QA file: {:s}".format(qafile))
arqa.coaddspec_qa(spectra, rspec, rmask, spec1d, qafile=qafile)
# Write to disk?
if outfile is not None:
write_to_disk(spec1d, outfile)
return spec1d
def optimal_extract(slf,
det,
specobjs,
sciframe,
varframe,
skyframe,
crmask,
scitrace,
pickle_file=None,
profiles=None):
""" Preform optimal extraction
Standard Horne approach
Parameters
----------
slf
det
specobjs
sciframe
varframe
crmask
scitrace
COUNT_LIM
pickle_file
Returns
-------
newvar : ndarray
Updated variance array that includes object model
"""
from pypit import arproc
# Setup
#rnimg = arproc.rn_frame(slf,det)
#model_var = np.abs(skyframe + sciframe - np.sqrt(2)*rnimg + rnimg**2) # sqrt 2 term deals with negative flux/sky
#model_ivar = 1./model_var
# Inverse variance
model_ivar = np.zeros_like(varframe)
gdvar = varframe > 0.
model_ivar[gdvar] = arutils.calc_ivar(varframe[gdvar])
cr_mask = 1.0 - crmask
# Object model image
obj_model = np.zeros_like(varframe)
# Loop on slits
for sl in range(len(specobjs)):
# Loop on objects
nobj = scitrace[sl]['traces'].shape[1]
for o in range(nobj):
msgs.info(
"Performing optimal extraction of object {0:d}/{1:d} in slit {2:d}/{3:d}"
.format(o + 1, nobj, sl + 1, len(specobjs)))
# Get object pixels
if scitrace[sl]['background'] is None:
# The object for all slits is provided in the first extension
objreg = np.copy(scitrace[0]['object'][:, :, o])
wzro = np.where(slf._slitpix[det - 1] != sl + 1)
objreg[wzro] = 0.0
else:
objreg = scitrace[sl]['object'][:, :, o]
# Fit dict
fit_dict = scitrace[sl]['opt_profile'][o]
if 'param' not in fit_dict.keys():
continue
# Slit image
slit_img = artrace.slit_image(slf, det, scitrace[sl],
o) #, tilts=tilts)
#msgs.warn("Turn off tilts")
# Object pixels
weight = objreg.copy()
gdo = (weight > 0) & (model_ivar > 0)
# Profile image
prof_img = np.zeros_like(weight)
prof_img[gdo] = arutils.func_val(fit_dict['param'], slit_img[gdo],
fit_dict['func'])
# Normalize
norm_prof = np.sum(prof_img, axis=1)
prof_img /= np.outer(norm_prof + (norm_prof == 0.),
np.ones(prof_img.shape[1]))
# Mask (1=good)
mask = np.zeros_like(prof_img)
mask[gdo] = 1.
mask *= cr_mask
# Optimal flux
opt_num = np.sum(mask * sciframe * model_ivar * prof_img, axis=1)
opt_den = np.sum(mask * model_ivar * prof_img**2, axis=1)
opt_flux = opt_num / (opt_den + (opt_den == 0.))
# Optimal wave
opt_num = np.sum(slf._mswave[det - 1] * model_ivar * prof_img**2,
axis=1)
opt_den = np.sum(model_ivar * prof_img**2, axis=1)
opt_wave = opt_num / (opt_den + (opt_den == 0.))
if (np.sum(opt_wave < 1.) > 0) and settings.argflag["reduce"][
"calibrate"]["wavelength"] != "pixel":
debugger.set_trace()
msgs.error("Zero value in wavelength array. Uh-oh")
# Optimal ivar
opt_num = np.sum(mask * model_ivar * prof_img**2, axis=1)
ivar_den = np.sum(mask * prof_img, axis=1)
opt_ivar = opt_num * arutils.calc_ivar(ivar_den)
# Save
specobjs[sl][o].optimal['wave'] = opt_wave.copy(
) * u.AA # Yes, units enter here
specobjs[sl][o].optimal['counts'] = opt_flux.copy()
gdiv = (opt_ivar > 0.) & (ivar_den > 0.)
opt_var = np.zeros_like(opt_ivar)
opt_var[gdiv] = arutils.calc_ivar(opt_ivar[gdiv])
specobjs[sl][o].optimal['var'] = opt_var.copy()
#specobjs[o].boxcar['sky'] = skysum # per pixel
# Update object model
counts_image = np.outer(opt_flux, np.ones(prof_img.shape[1]))
obj_model += prof_img * counts_image
'''
if 'OPTIMAL' in msgs._debug:
debugger.set_trace()
debugger.xplot(opt_wave, opt_flux, np.sqrt(opt_var))
'''
# Generate new variance image
newvar = arproc.variance_frame(slf,
det,
sciframe,
-1,
skyframe=skyframe,
objframe=obj_model)
# Return
return newvar