def proc_diff(self,
file_list,
bg_file_list,
reject_cr=True,
sigma_clip=False,
sigrej=None,
maxiters=5):
"""
Process a list of science images and their background frames
Primarily for near-IR reductions
Wrapper to proc_sci for
Needed in part to set self.sciframe, although I could kludge it another way..
Returns
-------
self.sciframe
self.rawvarframe
self.crmask
"""
sciimg_sci, sciivar_sci, rn2img_sci, mask_sci, crmask_sci = self.proc_sci(
file_list,
reject_cr=reject_cr,
sigma_clip=sigma_clip,
sigrej=sigrej,
maxiters=maxiters)
sciimg_bg, sciivar_bg, rn2img_bg, mask_bg, crmask_bg = self.proc_sci(
bg_file_list,
reject_cr=reject_cr,
sigma_clip=sigma_clip,
sigrej=sigrej,
maxiters=maxiters)
# Combine the images
outmask_comb = (mask_sci == 0) & (mask_bg == 0)
sciimg = sciimg_sci - sciimg_bg
varcomb = utils.calc_ivar(sciivar_sci) + utils.calc_ivar(sciivar_bg)
sciivar = utils.calc_ivar(varcomb) * outmask_comb
rn2img = rn2img_sci + rn2img_bg
# Now reject CRs again on the differenced image
crmask_diff = self.build_crmask(sciimg,
self.par['process'],
self.det,
self.spectrograph,
ivar=sciivar,
binning=self.binning)
# crmask_eff assumes evertything masked in the outmask_comb is a CR in the individual images
crmask = crmask_diff | np.invert(outmask_comb)
# Create a mask for this image now
mask = self.build_mask(sciimg,
sciivar,
crmask,
self.bpm,
saturation=self.saturation)
return sciimg, sciivar, rn2img, mask, crmask
def test_calc_ivar():
""" Run the parameter setup script
"""
x = np.array([-1.0, -0.1, 0.0, 0.1, 1.0])
res = utils.calc_ivar(x)
assert np.array_equal(res, np.array([0.0, 0.0, 0.0, 10.0, 1.0]))
assert np.array_equal(utils.calc_ivar(res),
np.array([0.0, 0.0, 0.0, 0.1, 1.0]))
def proc_sci(self, file_list, reject_cr=True, sigma_clip=False, sigrej=None, maxiters=5):
"""
Process a list of science images
This includes stacking the images if there is more than 1
Args:
file_list (list): List of filenames for science frames
reject_cr (bool, optional):
sigrej (int or float, optional): Rejection threshold for sigma clipping.
Code defaults to determining this automatically based on the numberr of images provided.
maxiters (int, optional):
show (bool, optional):
Returns:
ndarray, ndarray, ndarray, ndarray, ndarray:
sciimg
sciivar
rn2img
mask
crmask
"""
nsci = len(file_list)
weights = np.ones(nsci)/float(nsci)
sciimg_stack, sciivar_stack, rn2img_stack, crmask_stack, mask_stack = \
self.read_stack(file_list, self.bias, self.pixel_flat, self.bpm, self.det, self.par['process'], self.spectrograph,
illum_flat=self.illum_flat, reject_cr=reject_cr, binning=self.binning)
# ToDO The bitmask is not being properly propagated here!
if nsci > 1:
sci_list = [sciimg_stack]
var_stack = utils.calc_ivar(sciivar_stack)
var_list = [var_stack, rn2img_stack]
sci_list_out, var_list_out, outmask, nused = coadd2d.weighted_combine(
weights, sci_list, var_list, (mask_stack == 0),
sigma_clip=sigma_clip, sigma_clip_stack = sciimg_stack, sigrej=sigrej, maxiters=maxiters)
sciimg = sci_list_out[0]
sciivar = utils.calc_ivar(var_list_out[0])
rn2img = var_list_out[1]
# assumes everything masked in the outmask is a CR in the individual images
crmask = np.invert(outmask)
# Create a mask for this image now
mask = self.build_mask(sciimg, sciivar, crmask, self.bpm, saturation=self.saturation, mincounts=self.mincounts)
else:
mask = mask_stack[0, :, :]
crmask = crmask_stack[0, :, :]
sciimg = sciimg_stack[0, :, :]
sciivar = sciivar_stack[0, :, :]
rn2img = rn2img_stack[0, :, :]
return sciimg, sciivar, rn2img, mask, crmask
def build_crmask(stack, proc_par, det, spectrograph, ivar=None, binning=None):
"""
Generate the CR mask frame
Wrapper to procimg.lacosmic
Parameters
----------
varframe : ndarray, optional
Returns
-------
self.crmask : ndarray
1. = Masked CR
"""
# Run LA Cosmic to get the cosmic ray mask
varframe = utils.calc_ivar(ivar)
saturation = spectrograph.detector[det-1]['saturation']
nonlinear = spectrograph.detector[det-1]['nonlinear']
#sigclip, objlim = spectrograph.get_lacosmics_par(proc_par,binning=binning)
crmask = procimg.lacosmic(det, stack, saturation, nonlinear,
varframe=varframe, maxiter=proc_par['lamaxiter'],
grow=proc_par['grow'],
remove_compact_obj=proc_par['rmcompact'],
sigclip=proc_par['sigclip'],
sigfrac=proc_par['sigfrac'],
objlim=proc_par['objlim'])
# Return
return crmask
def read_lris_stack():
stackfile = '/Users/joe/REDUX/lris_redux/Nov_2004/Final/SDSSJ073522.43+295710.1_N.fits'
hdu = fits.open(stackfile)
flux_ref = hdu[0].data
sig_ref = hdu[1].data
ivar_ref = utils.calc_ivar(sig_ref)
mask_ref = (ivar_ref > 0.0)
wave_ref = hdu[2].data
infiles = [
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0063.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0064.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0065.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0066.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0219.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0220.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0221.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0222.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0063.fits.gz'
]
nfiles = len(infiles)
objid = 0
for idx, file in enumerate(infiles):
obj = Table.read(file, hdu=5)
flux = obj[objid]['FLUX_OPT']
ivar = obj[objid]['IVAR_OPT']
wave = obj[objid]['WAVE_OPT']
if idx == 0:
nspec = flux.size
flux_arr = np.zeros((nfiles, nspec))
wave_arr = np.zeros((nfiles, nspec))
ivar_arr = np.zeros((nfiles, nspec))
flux_arr[idx, :] = flux
ivar_arr[idx, :] = ivar
wave_arr[idx, :] = wave
mask_arr = (ivar_arr > 0.0)
return wave_arr, flux_arr, ivar_arr, mask_arr
#plt.plot(wave_ref, flux_ref, drawstyle='steps-mid')
#for ii in range(nfiles):
# plt.plot(wave_arr[:,ii], flux_arr[:,ii],drawstyle='steps-mid')
#flux_inter, ivar_inter, mask_inter = interp_spec(wave_ref, wave_arr, flux_arr, ivar_arr, mask_arr)
#idx = 5
#wave = wave_ref
#flux = flux_inter[idx, :]
#ivar = ivar_inter[idx, :]
#mask = ivar > 0
#norder = 3
#ymult,flux_rescale, ivar_rescale, outmask = solve_poly_ratio(wave, flux, ivar, flux_ref, ivar_ref, norder,
# mask=mask, mask_ref=mask_ref, debug=True)
def apply_sens_tell_spec(wave, counts, ivar, sensfunc, airmass, exptime, mask=None, extinct_correct=True, telluric=None,
longitude=None, latitude=None, debug=False):
if mask is None:
mask = ivar > 0.0
# Did the user request a telluric correction from the same file?
if telluric is not None:
# This assumes there is a separate telluric key in this dict.
msgs.info('Applying telluric correction')
sensfunc = sensfunc*(telluric > 1e-10)/(telluric + (telluric < 1e-10))
if extinct_correct:
if longitude is None or latitude is None:
msgs.error('You must specify longitude and latitude if we are extinction correcting')
# Apply Extinction if optical bands
msgs.info("Applying extinction correction")
msgs.warn("Extinction correction applyed only if the spectra covers <10000Ang.")
extinct = load_extinction_data(longitude,latitude)
ext_corr = extinction_correction(wave* units.AA, airmass, extinct)
senstot = sensfunc * ext_corr
else:
senstot = sensfunc.copy()
flam = counts * senstot/ exptime
flam_ivar = ivar / (senstot / exptime) **2
# Mask bad pixels
msgs.info(" Masking bad pixels")
outmask = mask & (senstot>0.)
# debug
if debug:
wave_mask = wave > 1.0
fig = plt.figure(figsize=(12, 8))
ymin, ymax = coadd1d.get_ylim(flam, flam_ivar, outmask)
plt.plot(wave[wave_mask], flam[wave_mask], color='black', drawstyle='steps-mid', zorder=1, alpha=0.8)
plt.plot(wave[wave_mask], np.sqrt(utils.calc_ivar(flam_ivar[wave_mask])), zorder=2, color='red', alpha=0.7,
drawstyle='steps-mid', linestyle=':')
plt.ylim([ymin,ymax])
plt.xlim([wave[wave_mask].min(),wave[wave_mask].max()])
plt.xlabel('Wavelength (Angstrom)')
plt.ylabel('Flux')
plt.show()
return flam, flam_ivar, outmask
def read_stack(cls, files, bias, pixel_flat, bpm, det, proc_par, spectrograph, illum_flat=None, reject_cr=False,
binning=None):
""" Utility function for reading in image stacks using ProcessImages
Parameters
file_list:
bias:
pixel_flat:
bpm:
illum_flat:
Returns:
"""
nfiles = len(files)
for ifile in range(nfiles):
this_proc = ProcessImages(spectrograph, proc_par, [files[ifile]], det=det)
# TODO I think trim should be hard wired, and am not letting it be a free parameter
sciimg = this_proc.process(bias_subtract=bias,pixel_flat=pixel_flat, illum_flat=illum_flat, bpm=bpm,
apply_gain=True, trim=True)
# Allocate the images
if ifile == 0:
# numpy is row major so stacking will be fastest with nfiles as the first dimensions
shape = (nfiles, sciimg.shape[0],sciimg.shape[1])
sciimg_stack = np.zeros(shape)
sciivar_stack = np.zeros(shape)
rn2img_stack = np.zeros(shape)
crmask_stack = np.zeros(shape,dtype=bool)
mask_stack = np.zeros(shape,this_proc.bitmask.minimum_dtype(asuint=True))
# Construct raw variance image
rawvarframe = this_proc.build_rawvarframe(trim=True)
# Mask cosmic rays
sciivar_stack[ifile,:,:] = utils.calc_ivar(rawvarframe)
if reject_cr:
crmask_stack[ifile,:,:] = this_proc.build_crmask(sciimg, proc_par, det, spectrograph,
ivar=sciivar_stack[ifile,:,:], binning=binning)
sciimg_stack[ifile,:,:] = sciimg
# Build read noise squared image
rn2img_stack[ifile,:,:] = this_proc.build_rn2img()
# Final mask for this image
mask_stack[ifile,:,:] = this_proc.build_mask(sciimg, sciivar_stack[ifile,:,:], crmask_stack[ifile,:,:],
bpm, saturation = spectrograph.detector[det - 1]['saturation'],
mincounts = spectrograph.detector[det - 1]['mincounts'])
return sciimg_stack, sciivar_stack, rn2img_stack, crmask_stack, mask_stack
def read_lris_stack():
stackfile = '/Users/joe/REDUX/lris_redux/Nov_2004/Final/SDSSJ073522.43+295710.1_N.fits'
hdu = fits.open(stackfile)
flux_ref = hdu[0].data
sig_ref = hdu[1].data
ivar_ref = utils.calc_ivar(sig_ref)
mask_ref = (ivar_ref > 0.0)
wave_ref = hdu[2].data
infiles = [
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0063.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0064.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0065.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0066.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0219.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0220.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0221.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0222.fits.gz',
'/Users/joe/REDUX/lris_redux/Nov_2004/0735+2957/Blue/Science/sci-lblue0063.fits.gz'
]
nfiles = len(infiles)
objid = 0
for idx, file in enumerate(infiles):
obj = Table.read(file, hdu=5)
flux = obj[objid]['FLUX_OPT']
ivar = obj[objid]['IVAR_OPT']
wave = obj[objid]['WAVE_OPT']
if idx == 0:
nspec = flux.size
flux_arr = np.zeros((nspec, nfiles))
wave_arr = np.zeros((nspec, nfiles))
ivar_arr = np.zeros((nspec, nfiles))
flux_arr[:, idx] = np.flip(flux)
ivar_arr[:, idx] = np.flip(ivar)
wave_arr[:, idx] = np.flip(wave)
mask_arr = (ivar_arr > 0.0)
return wave_arr, flux_arr, ivar_arr, mask_arr
def coadd2d(trace_stack, sciimg_stack, sciivar_stack, skymodel_stack, inmask_stack, tilts_stack,
waveimg_stack, thismask_stack, weights=None, loglam_grid=None, wave_grid=None):
"""
Construct a 2d co-add of a stack of PypeIt spec2d reduction outputs.
Slits are 'rectified' onto a spatial and spectral grid, which
encompasses the spectral and spatial coverage of the image stacks.
The rectification uses nearest grid point interpolation to avoid
covariant errors. Dithering is supported as all images are centered
relative to a set of reference traces in trace_stack.
Args:
trace_stack (`numpy.ndarray`_):
Stack of reference traces about which the images are
rectified and coadded. If the images were not dithered then
this reference trace can simply be the center of the slit::
slitcen = (slit_left + slit_righ)/2
If the images were dithered, then this object can either be
the slitcen appropriately shifted with the dither pattern,
or it could the trace of the object of interest in each
exposure determined by running PypeIt on the individual
images. Shape is (nimgs, nspec).
sciimg_stack (`numpy.ndarray`_):
Stack of science images. Shape is (nimgs, nspec, nspat).
sciivar_stack (`numpy.ndarray`_):
Stack of inverse variance images. Shape is (nimgs, nspec,
nspat).
skymodel_stack (`numpy.ndarray`_):
Stack of the model sky. Shape is (nimgs, nspec, nspat).
inmask_stack (`numpy.ndarray`_):
Boolean array with the input masks for each image; `True`
values are *good*, `False` values are *bad*. Shape is
(nimgs, nspec, nspat).
tilts_stack (`numpy.ndarray`_):
Stack of the wavelength tilts traces. Shape is (nimgs,
nspec, nspat).
waveimg_stack (`numpy.ndarray`_):
Stack of the wavelength images. Shape is (nimgs, nspec,
nspat).
thismask_stack (`numpy.ndarray`_):
Boolean array with the masks indicating which pixels are on
the slit in question. `True` values are on the slit;
`False` values are off the slit. Shape is (nimgs, nspec,
nspat).
weights (`numpy.ndarray`_, optional):
The weights used when combining the rectified images (see
:func:`weighted_combine`). If no weights are provided,
uniform weighting is used. Weights are broadast to the
correct size of the image stacks (see
:func:`broadcast_weights`), as necessary. Shape must be
(nimgs,), (nimgs, nspec), or (nimgs, nspec, nspat).
loglam_grid (`numpy.ndarray`_, optional):
Wavelength grid in log10(wave) onto which the image stacks
will be rectified. The code will automatically choose the
subset of this grid encompassing the wavelength coverage of
the image stacks provided (see :func:`waveimg_stack`).
Either `loglam_grid` or `wave_grid` must be provided.
wave_grid (`numpy.ndarray`_, optional):
Same as `loglam_grid` but in angstroms instead of
log(angstroms). (TODO: Check units...)
Returns:
TODO: This needs to be updated.
(sciimg, sciivar, imgminsky, outmask, nused, tilts, waveimg, dspat, thismask, tslits_dict)
sciimg: float ndarray shape = (nspec_coadd, nspat_coadd)
Rectified and coadded science image
sciivar: float ndarray shape = (nspec_coadd, nspat_coadd)
Rectified and coadded inverse variance image with correct error propagation
imgminsky: float ndarray shape = (nspec_coadd, nspat_coadd)
Rectified and coadded sky subtracted image
outmask: bool ndarray shape = (nspec_coadd, nspat_coadd)
Output mask for rectified and coadded images. True = Good, False=Bad.
nused: int ndarray shape = (nspec_coadd, nspat_coadd)
Image of integers indicating the number of images from the image stack that contributed to each pixel
tilts: float ndarray shape = (nspec_coadd, nspat_coadd)
The averaged tilts image corresponding to the rectified and coadded data.
waveimg: float ndarray shape = (nspec_coadd, nspat_coadd)
The averaged wavelength image corresponding to the rectified and coadded data.
dspat: float ndarray shape = (nspec_coadd, nspat_coadd)
The average spatial offsets in pixels from the reference trace trace_stack corresponding to the rectified
and coadded data.
thismask: bool ndarray shape = (nspec_coadd, nspat_coadd)
Output mask for rectified and coadded images. True = Good, False=Bad. This image is trivial, and
is simply an image of True values the same shape as the rectified and coadded data.
tslits_dict: dict
tslits_dict dictionary containing the information about the slits boundaries. The slit boundaries
are trivial and are simply vertical traces at 0 and nspat_coadd-1.
"""
nimgs, nspec, nspat = sciimg_stack.shape
# Determine the wavelength grid that we will use for the current slit/order
# TODO This cut on waveimg_stack should not be necessary
wavemask = thismask_stack & (waveimg_stack > 1.0)
wave_lower = waveimg_stack[wavemask].min()
wave_upper = waveimg_stack[wavemask].max()
if loglam_grid is not None:
ind_lower, ind_upper = get_wave_ind(loglam_grid, np.log10(wave_lower), np.log10(wave_upper))
loglam_bins = loglam_grid[ind_lower:ind_upper + 1]
wave_bins = np.power(10.0, loglam_bins)
elif wave_grid is not None:
ind_lower, ind_upper = get_wave_ind(wave_grid, wave_lower, wave_upper)
wave_bins = wave_grid[ind_lower:ind_upper + 1]
loglam_bins = np.log10(wave_bins)
else:
msgs.error('You must input either a uniformly space loglam grid or wave grid')
if weights is None:
msgs.info('No weights were provided. Using uniform weights.')
weights = np.ones(nimgs)/float(nimgs)
weights_stack = broadcast_weights(weights, sciimg_stack.shape)
# Create the slit_cen_stack and determine the minimum and maximum
# spatial offsets that we need to cover to determine the spatial
# bins
spat_img = np.outer(np.ones(nspec), np.arange(nspat))
dspat_stack = np.zeros_like(sciimg_stack)
spat_min = np.inf
spat_max = -np.inf
for img in range(nimgs):
# center of the slit replicated spatially
slit_cen_img = np.outer(trace_stack[img, :], np.ones(nspat))
dspat_iexp = (spat_img - slit_cen_img)
dspat_stack[img, :, :] = dspat_iexp
thismask_now = thismask_stack[img, :, :]
spat_min = np.fmin(spat_min, dspat_iexp[thismask_now].min())
spat_max = np.fmax(spat_max, dspat_iexp[thismask_now].max())
spat_min_int = int(np.floor(spat_min))
spat_max_int = int(np.ceil(spat_max))
dspat_bins = np.arange(spat_min_int, spat_max_int + 1, 1)
sci_list = [weights_stack, sciimg_stack, sciimg_stack - skymodel_stack, tilts_stack,
waveimg_stack, dspat_stack]
var_list = [utils.calc_ivar(sciivar_stack)]
sci_list_rebin, var_list_rebin, norm_rebin_stack, nsmp_rebin_stack \
= rebin2d(wave_bins, dspat_bins, waveimg_stack, dspat_stack, thismask_stack,
inmask_stack, sci_list, var_list)
# Now compute the final stack with sigma clipping
sigrej = 3.0
maxiters = 10
# sci_list_rebin[0] = rebinned weights image stack
# sci_list_rebin[1:] = stacks of images that we want to weighted combine
# sci_list_rebin[2] = rebinned sciimg-sky_model images that we used for the sigma clipping
sci_list_out, var_list_out, outmask, nused \
= weighted_combine(sci_list_rebin[0], sci_list_rebin[1:], var_list_rebin,
norm_rebin_stack != 0, sigma_clip=True,
sigma_clip_stack=sci_list_rebin[2], sigrej=sigrej,
maxiters=maxiters)
sciimg, imgminsky, tilts, waveimg, dspat = sci_list_out
sciivar = utils.calc_ivar(var_list_out[0])
# Compute the midpoints vectors, and lower/upper bins of the rectified image
wave_mid = ((wave_bins + np.roll(wave_bins,1))/2.0)[1:]
wave_min = wave_bins[:-1]
wave_max = wave_bins[1:]
dspat_mid = ((dspat_bins + np.roll(dspat_bins,1))/2.0)[1:]
# Interpolate the dspat images wherever the coadds are masked
# because a given pixel was not sampled. This is done because the
# dspat image is not allowed to have holes if it is going to work
# with local_skysub_extract
nspec_coadd, nspat_coadd = imgminsky.shape
spat_img_coadd, spec_img_coadd = np.meshgrid(np.arange(nspat_coadd), np.arange(nspec_coadd))
if np.any(np.invert(outmask)):
points_good = np.stack((spec_img_coadd[outmask], spat_img_coadd[outmask]), axis=1)
points_bad = np.stack((spec_img_coadd[np.invert(outmask)],
spat_img_coadd[np.invert(outmask)]), axis=1)
values_dspat = dspat[outmask]
dspat_bad = scipy.interpolate.griddata(points_good, values_dspat, points_bad,
method='cubic')
dspat[np.invert(outmask)] = dspat_bad
# Points outside the convex hull of the data are set to nan. We
# identify those and simply assume them values from the
# dspat_img_fake, which is what dspat would be on a regular
# perfectly rectified image grid.
nanpix = np.isnan(dspat)
if np.any(nanpix):
dspat_img_fake = spat_img_coadd + dspat_mid[0]
dspat[nanpix] = dspat_img_fake[nanpix]
return dict(wave_bins=wave_bins, dspat_bins=dspat_bins, wave_mid=wave_mid, wave_min=wave_min,
wave_max=wave_max, dspat_mid=dspat_mid, sciimg=sciimg, sciivar=sciivar,
imgminsky=imgminsky, outmask=outmask, nused=nused, tilts=tilts, waveimg=waveimg,
dspat=dspat, nspec=imgminsky.shape[0], nspat=imgminsky.shape[1])
objminsky = hdu[0].data - hdu[2].data
ivar = hdu[4].data
#ivar = utils.calc_ivar(var)
mask = (ivar > 0.0)
slit_left = readsav(user + 'Dropbox/hires_fndobj/MAGE/left_edge.sav',python_dict=False)['left_edge'].T
slit_righ = readsav(user + 'Dropbox/hires_fndobj/MAGE/right_edge.sav',python_dict=False)['right_edge'].T
plate_scale = 0.3
hdu_std = fits.open(user + 'Dropbox/hires_fndobj/MAGE/ObjStr1046.fits')
std_trace = hdu_std[1].data['trace'].T
# Standard is the ObjStr1046.fits exten = 1
elif spectro == 'ESI':
# ESI
hdu = fits.open(user + '/Dropbox/Cowie_2002-02-17/Final/fringe_ES.20020217.35453.fits.gz')
sciimg = hdu[0].data
var = hdu[1].data
ivar = utils.calc_ivar(var)
mask = (var > 0.0)
skyimg = hdu[2].data
objminsky = sciimg - skyimg
hdu_sedg = fits.open(user + '/Dropbox/Cowie_2002-02-17/Flats/SEdgECH75_1x1.fits')
data = hdu_sedg[0].data
slit_left = data[0,:,:].T
slit_righ = data[1,:,:].T
plate_scale = 0.149
hdu_std =fits.open(user + '/Dropbox/Cowie_2002-02-17/Extract/Obj_ES.20020217.20037.fits.gz')
std_trace = hdu_std[1].data['trace'][:,0:slit_left.shape[0]].T
std_trace[:,-1] = std_trace[:,-2]+ (np.median(std_trace[:200,-1])-np.median(std_trace[:200,-2]))
#The structure is exten = 1, and the xpos,ypos are the standard trace
elif spectro == 'NIRES':
from linetools import utils as ltu
jdict = ltu.loadjson(user + '/Dropbox/hires_fndobj/tilt_nires.json')
def ech_objfind(image,
ivar,
ordermask,
slit_left,
slit_righ,
inmask=None,
plate_scale=0.2,
npca=2,
ncoeff=5,
min_snr=0.0,
nabove_min_snr=0,
pca_percentile=20.0,
snr_pca=3.0,
box_radius=2.0,
show_peaks=False,
show_fits=False,
show_trace=False):
if inmask is None:
inmask = (ordermask > 0)
frameshape = image.shape
nspec = frameshape[0]
norders = slit_left.shape[1]
if isinstance(plate_scale, (float, int)):
plate_scale_ord = np.full(
norders, plate_scale) # 0.12 binned by 3 spatially for HIRES
elif isinstance(plate_scale, (np.ndarray, list, tuple)):
if len(plate_scale) == norders:
plate_scale_ord = plate_scale
elif len(plate_scale) == 1:
plate_scale_ord = np.full(norders, plate_scale[0])
else:
msgs.error(
'Invalid size for plate_scale. It must either have one element or norders elements'
)
else:
msgs.error('Invalid type for plate scale')
specmid = nspec // 2
slit_width = slit_righ - slit_left
spec_vec = np.arange(nspec)
slit_spec_pos = nspec / 2.0
slit_spat_pos = np.zeros((norders, 2))
for iord in range(norders):
slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec,
slit_left[:, iord]),
np.interp(slit_spec_pos, spec_vec,
slit_righ[:, iord]))
# Loop over orders and find objects
sobjs = specobjs.SpecObjs()
show_peaks = True
show_fits = True
# ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx!
for iord in range(norders):
msgs.info('Finding objects on slit # {:d}'.format(iord + 1))
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
specobj_dict = {
'setup': 'HIRES',
'slitid': iord + 1,
'scidx': 0,
'det': 1,
'objtype': 'science'
}
sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \
extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks,
show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0)
# ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord
for spec in sobjs_slit:
spec.ech_orderindx = iord
sobjs.add_sobj(sobjs_slit)
nfound = len(sobjs)
# Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0"
FOFSEP = 1.0 # separation of FOF algorithm in arcseconds
FOF_frac = FOFSEP / (np.median(slit_width) * np.median(plate_scale_ord))
# Feige: made the code also works for only one object found in one order
# Run the FOF. We use fake coordinaes
fracpos = sobjs.spat_fracpos
ra_fake = fracpos / 1000.0 # Divide all angles by 1000 to make geometry euclidian
dec_fake = 0.0 * fracpos
if nfound > 1:
(ingroup, multgroup, firstgroup,
nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac / 1000.0)
group = ingroup.copy()
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(uni_group)
msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects')
elif nfound == 1:
group = np.zeros(1, dtype='int')
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(group)
msgs.warn('Only find one object no FOF matching is needed')
gfrac = np.zeros(nfound)
for jj in range(nobj):
this_group = group == uni_group[jj]
gfrac[this_group] = np.median(fracpos[this_group])
uni_frac = gfrac[uni_ind]
sobjs_align = sobjs.copy()
# Now fill in the missing objects and their traces
for iobj in range(nobj):
for iord in range(norders):
# Is there an object on this order that grouped into the current group in question?
on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx
== iord)
if not np.any(on_slit):
# Add this to the sobjs_align, and assign required tags
thisobj = specobjs.SpecObj(frameshape,
slit_spat_pos[iord, :],
slit_spec_pos,
det=sobjs_align[0].det,
setup=sobjs_align[0].setup,
slitid=(iord + 1),
scidx=sobjs_align[0].scidx,
objtype=sobjs_align[0].objtype)
thisobj.ech_orderindx = iord
thisobj.spat_fracpos = uni_frac[iobj]
thisobj.trace_spat = slit_left[:,
iord] + slit_width[:, iord] * uni_frac[
iobj] # new trace
thisobj.trace_spec = spec_vec
thisobj.spat_pixpos = thisobj.trace_spat[specmid]
thisobj.set_idx()
# Use the real detections of this objects for the FWHM
this_group = group == uni_group[iobj]
# Assign to the fwhm of the nearest detected order
imin = np.argmin(
np.abs(sobjs_align[this_group].ech_orderindx - iord))
thisobj.fwhm = sobjs_align[imin].fwhm
thisobj.maskwidth = sobjs_align[imin].maskwidth
thisobj.ech_fracpos = uni_frac[iobj]
thisobj.ech_group = uni_group[iobj]
thisobj.ech_usepca = True
sobjs_align.add_sobj(thisobj)
group = np.append(group, uni_group[iobj])
gfrac = np.append(gfrac, uni_frac[iobj])
else:
# ToDo fix specobjs to get rid of these crappy loops!
for spec in sobjs_align[on_slit]:
spec.ech_fracpos = uni_frac[iobj]
spec.ech_group = uni_group[iobj]
spec.ech_usepca = False
# Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order
# respectively
sobjs_sort = specobjs.SpecObjs()
for iobj in range(nobj):
this_group = group == uni_group[iobj]
this_sobj = sobjs_align[this_group]
sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)])
# Loop over the objects and perform a quick and dirty extraction to assess S/N.
varimg = utils.calc_ivar(ivar)
flux_box = np.zeros((nspec, norders, nobj))
ivar_box = np.zeros((nspec, norders, nobj))
mask_box = np.zeros((nspec, norders, nobj))
SNR_arr = np.zeros((norders, nobj))
for iobj in range(nobj):
for iord in range(norders):
indx = (sobjs_sort.ech_group
== uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord)
spec = sobjs_sort[indx]
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
box_rad_pix = box_radius / plate_scale_ord[iord]
flux_tmp = extract.extract_boxcar(image * inmask_iord,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
var_tmp = extract.extract_boxcar(varimg * inmask_iord,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
ivar_tmp = utils.calc_ivar(var_tmp)
pixtot = extract.extract_boxcar(ivar * 0 + 1.0,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
mask_tmp = (extract.extract_boxcar(ivar * inmask_iord == 0.0,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec) != pixtot)
flux_box[:, iord, iobj] = flux_tmp * mask_tmp
ivar_box[:, iord, iobj] = np.fmax(ivar_tmp * mask_tmp, 0.0)
mask_box[:, iord, iobj] = mask_tmp
(mean, med_sn, stddev) = sigma_clipped_stats(
flux_box[mask_tmp, iord, iobj] *
np.sqrt(ivar_box[mask_tmp, iord, iobj]),
sigma_lower=5.0,
sigma_upper=5.0)
SNR_arr[iord, iobj] = med_sn
# Purge objects with low SNR and that don't show up in enough orders
keep_obj = np.zeros(nobj, dtype=bool)
sobjs_trim = specobjs.SpecObjs()
uni_group_trim = np.array([], dtype=int)
uni_frac_trim = np.array([], dtype=float)
for iobj in range(nobj):
if (np.sum(SNR_arr[:, iobj] > min_snr) >= nabove_min_snr):
keep_obj[iobj] = True
ikeep = sobjs_sort.ech_group == uni_group[iobj]
sobjs_trim.add_sobj(sobjs_sort[ikeep])
uni_group_trim = np.append(uni_group_trim, uni_group[iobj])
uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj])
else:
msgs.info(
'Purging object #{:d}'.format(iobj) +
' which does not satisfy min_snr > {:5.2f}'.format(min_snr) +
' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) +
' orders')
nobj_trim = np.sum(keep_obj)
if nobj_trim == 0:
return specobjs.SpecObjs()
SNR_arr_trim = SNR_arr[:, keep_obj]
# Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA
for iobj in range(nobj_trim):
SNR_now = SNR_arr_trim[:, iobj]
indx = (sobjs_trim.ech_group == uni_group_trim[iobj])
# PCA interp/extrap if:
# (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca)
# OR
# (if this order was not originally traced by the object finding, see above)
usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) &
(SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca
# ToDo fix specobjs to get rid of these crappy loops!
for iord, spec in enumerate(sobjs_trim[indx]):
spec.ech_usepca = usepca[iord]
if usepca[iord]:
msgs.info('Using PCA to predict trace for object #{:d}'.format(
iobj) + ' on order #{:d}'.format(iord))
sobjs_final = sobjs_trim.copy()
# Loop over the objects one by one and adjust/predict the traces
npoly_cen = 3
pca_fits = np.zeros((nspec, norders, nobj_trim))
for iobj in range(nobj_trim):
igroup = sobjs_final.ech_group == uni_group_trim[iobj]
# PCA predict the masked orders which were not traced
pca_fits[:, :, iobj] = pca_trace((sobjs_final[igroup].trace_spat).T,
usepca=None,
npca=npca,
npoly_cen=npoly_cen)
# usepca = sobjs_final[igroup].ech_usepca,
# Perform iterative flux weighted centroiding using new PCA predictions
xinit_fweight = pca_fits[:, :, iobj].copy()
inmask_now = inmask & (ordermask > 0)
xfit_fweight = extract.iter_tracefit(image,
xinit_fweight,
ncoeff,
inmask=inmask_now,
show_fits=show_fits)
# Perform iterative Gaussian weighted centroiding
xinit_gweight = xfit_fweight.copy()
xfit_gweight = extract.iter_tracefit(image,
xinit_gweight,
ncoeff,
inmask=inmask_now,
gweight=True,
show_fits=show_fits)
# Assign the new traces
for iord, spec in enumerate(sobjs_final[igroup]):
spec.trace_spat = xfit_gweight[:, iord]
spec.spat_pixpos = spec.trace_spat[specmid]
# Set the IDs
sobjs_final.set_idx()
if show_trace:
viewer, ch = ginga.show_image(objminsky * (ordermask > 0))
for iobj in range(nobj_trim):
for iord in range(norders):
ginga.show_trace(viewer,
ch,
pca_fits[:, iord, iobj],
str(uni_frac[iobj]),
color='yellow')
for spec in sobjs_trim:
color = 'green' if spec.ech_usepca else 'magenta'
ginga.show_trace(viewer,
ch,
spec.trace_spat,
spec.idx,
color=color)
#for spec in sobjs_final:
# color = 'red' if spec.ech_usepca else 'green'
# ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)
return sobjs_final
hdu = fits.open('/Users/feige/Dropbox/hires_fndobj/f_hires0184G.fits.gz')
objminsky = hdu[2].data
ivar = hdu[1].data
mask = (ivar > 0.0)
order_str = Table.read('/Users/feige/Dropbox/hires_fndobj/OStr_G_04.fits')
slit_left = (order_str['LHEDG']).T
slit_righ = (order_str['RHEDG']).T
plate_scale = 0.36
elif spectro == 'ESI':
# ESI
hdu = fits.open(
'/Users/feige/Dropbox/Cowie_2002-02-17/Final/fringe_ES.20020217.35453.fits.gz'
)
sciimg = hdu[0].data
var = hdu[1].data
ivar = utils.calc_ivar(var)
mask = (var > 0.0)
skyimg = hdu[2].data
objminsky = sciimg - skyimg
hdu_sedg = fits.open(
'/Users/feige/Dropbox/Cowie_2002-02-17/Flats/SEdgECH75_1x1.fits')
data = hdu_sedg[0].data
slit_left = data[0, :, :].T
slit_righ = data[1, :, :].T
plate_scale = 0.149
elif spectro == 'NIRES':
from linetools import utils as ltu
jdict = ltu.loadjson('/Users/feige/Dropbox/hires_fndobj/tilt_nires.json')
slit_left = np.array(jdict['lcen'])
slit_righ = np.array(jdict['rcen'])
hdu = fits.open(
def ech_objfind(image, ivar, ordermask, slit_left, slit_righ,inmask=None,plate_scale=0.2,npca=2,ncoeff = 5,min_snr=0.0,nabove_min_snr=0,
pca_percentile=20.0,snr_pca=3.0,box_radius=2.0,show_peaks=False,show_fits=False,show_trace=False):
if inmask is None:
inmask = (ordermask > 0)
frameshape = image.shape
nspec = frameshape[0]
norders = slit_left.shape[1]
if isinstance(plate_scale,(float, int)):
plate_scale_ord = np.full(norders, plate_scale) # 0.12 binned by 3 spatially for HIRES
elif isinstance(plate_scale,(np.ndarray, list, tuple)):
if len(plate_scale) == norders:
plate_scale_ord = plate_scale
elif len(plate_scale) == 1:
plate_scale_ord = np.full(norders, plate_scale[0])
else:
msgs.error('Invalid size for plate_scale. It must either have one element or norders elements')
else:
msgs.error('Invalid type for plate scale')
specmid = nspec // 2
slit_width = slit_righ - slit_left
spec_vec = np.arange(nspec)
slit_spec_pos = nspec/2.0
slit_spat_pos = np.zeros((norders, 2))
for iord in range(norders):
slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec, slit_left[:,iord]), np.interp(slit_spec_pos, spec_vec, slit_righ[:,iord]))
# Loop over orders and find objects
sobjs = specobjs.SpecObjs()
show_peaks=True
show_fits=True
# ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx!
for iord in range(norders):
msgs.info('Finding objects on slit # {:d}'.format(iord + 1))
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
specobj_dict = {'setup': 'HIRES', 'slitid': iord + 1, 'scidx': 0,'det': 1, 'objtype': 'science'}
sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \
extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks,
show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0)
# ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord
for spec in sobjs_slit:
spec.ech_orderindx = iord
sobjs.add_sobj(sobjs_slit)
nfound = len(sobjs)
# Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0"
FOFSEP = 1.0 # separation of FOF algorithm in arcseconds
FOF_frac = FOFSEP/(np.median(slit_width)*np.median(plate_scale_ord))
# Feige: made the code also works for only one object found in one order
# Run the FOF. We use fake coordinaes
fracpos = sobjs.spat_fracpos
ra_fake = fracpos/1000.0 # Divide all angles by 1000 to make geometry euclidian
dec_fake = 0.0*fracpos
if nfound>1:
(ingroup, multgroup, firstgroup, nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac/1000.0)
group = ingroup.copy()
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(uni_group)
msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects')
elif nfound==1:
group = np.zeros(1,dtype='int')
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(group)
msgs.warn('Only find one object no FOF matching is needed')
gfrac = np.zeros(nfound)
for jj in range(nobj):
this_group = group == uni_group[jj]
gfrac[this_group] = np.median(fracpos[this_group])
uni_frac = gfrac[uni_ind]
sobjs_align = sobjs.copy()
# Now fill in the missing objects and their traces
for iobj in range(nobj):
for iord in range(norders):
# Is there an object on this order that grouped into the current group in question?
on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx == iord)
if not np.any(on_slit):
# Add this to the sobjs_align, and assign required tags
thisobj = specobjs.SpecObj(frameshape, slit_spat_pos[iord,:], slit_spec_pos, det = sobjs_align[0].det,
setup = sobjs_align[0].setup, slitid = (iord + 1),
scidx = sobjs_align[0].scidx, objtype=sobjs_align[0].objtype)
thisobj.ech_orderindx = iord
thisobj.spat_fracpos = uni_frac[iobj]
thisobj.trace_spat = slit_left[:,iord] + slit_width[:,iord]*uni_frac[iobj] # new trace
thisobj.trace_spec = spec_vec
thisobj.spat_pixpos = thisobj.trace_spat[specmid]
thisobj.set_idx()
# Use the real detections of this objects for the FWHM
this_group = group == uni_group[iobj]
# Assign to the fwhm of the nearest detected order
imin = np.argmin(np.abs(sobjs_align[this_group].ech_orderindx - iord))
thisobj.fwhm = sobjs_align[imin].fwhm
thisobj.maskwidth = sobjs_align[imin].maskwidth
thisobj.ech_fracpos = uni_frac[iobj]
thisobj.ech_group = uni_group[iobj]
thisobj.ech_usepca = True
sobjs_align.add_sobj(thisobj)
group = np.append(group, uni_group[iobj])
gfrac = np.append(gfrac, uni_frac[iobj])
else:
# ToDo fix specobjs to get rid of these crappy loops!
for spec in sobjs_align[on_slit]:
spec.ech_fracpos = uni_frac[iobj]
spec.ech_group = uni_group[iobj]
spec.ech_usepca = False
# Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order
# respectively
sobjs_sort = specobjs.SpecObjs()
for iobj in range(nobj):
this_group = group == uni_group[iobj]
this_sobj = sobjs_align[this_group]
sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)])
# Loop over the objects and perform a quick and dirty extraction to assess S/N.
varimg = utils.calc_ivar(ivar)
flux_box = np.zeros((nspec, norders, nobj))
ivar_box = np.zeros((nspec, norders, nobj))
mask_box = np.zeros((nspec, norders, nobj))
SNR_arr = np.zeros((norders, nobj))
for iobj in range(nobj):
for iord in range(norders):
indx = (sobjs_sort.ech_group == uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord)
spec = sobjs_sort[indx]
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
box_rad_pix = box_radius/plate_scale_ord[iord]
flux_tmp = extract.extract_boxcar(image*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
var_tmp = extract.extract_boxcar(varimg*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
ivar_tmp = utils.calc_ivar(var_tmp)
pixtot = extract.extract_boxcar(ivar*0 + 1.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
mask_tmp = (extract.extract_boxcar(ivar*inmask_iord == 0.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) != pixtot)
flux_box[:,iord,iobj] = flux_tmp*mask_tmp
ivar_box[:,iord,iobj] = np.fmax(ivar_tmp*mask_tmp,0.0)
mask_box[:,iord,iobj] = mask_tmp
(mean, med_sn, stddev) = sigma_clipped_stats(flux_box[mask_tmp,iord,iobj]*np.sqrt(ivar_box[mask_tmp,iord,iobj]),
sigma_lower=5.0,sigma_upper=5.0)
SNR_arr[iord,iobj] = med_sn
# Purge objects with low SNR and that don't show up in enough orders
keep_obj = np.zeros(nobj,dtype=bool)
sobjs_trim = specobjs.SpecObjs()
uni_group_trim = np.array([],dtype=int)
uni_frac_trim = np.array([],dtype=float)
for iobj in range(nobj):
if (np.sum(SNR_arr[:,iobj] > min_snr) >= nabove_min_snr):
keep_obj[iobj] = True
ikeep = sobjs_sort.ech_group == uni_group[iobj]
sobjs_trim.add_sobj(sobjs_sort[ikeep])
uni_group_trim = np.append(uni_group_trim, uni_group[iobj])
uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj])
else:
msgs.info('Purging object #{:d}'.format(iobj) + ' which does not satisfy min_snr > {:5.2f}'.format(min_snr) +
' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) + ' orders')
nobj_trim = np.sum(keep_obj)
if nobj_trim == 0:
return specobjs.SpecObjs()
SNR_arr_trim = SNR_arr[:,keep_obj]
# Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA
for iobj in range(nobj_trim):
SNR_now = SNR_arr_trim[:,iobj]
indx = (sobjs_trim.ech_group == uni_group_trim[iobj])
# PCA interp/extrap if:
# (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca)
# OR
# (if this order was not originally traced by the object finding, see above)
usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) & (SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca
# ToDo fix specobjs to get rid of these crappy loops!
for iord, spec in enumerate(sobjs_trim[indx]):
spec.ech_usepca = usepca[iord]
if usepca[iord]:
msgs.info('Using PCA to predict trace for object #{:d}'.format(iobj) + ' on order #{:d}'.format(iord))
sobjs_final = sobjs_trim.copy()
# Loop over the objects one by one and adjust/predict the traces
npoly_cen = 3
pca_fits = np.zeros((nspec, norders, nobj_trim))
for iobj in range(nobj_trim):
igroup = sobjs_final.ech_group == uni_group_trim[iobj]
# PCA predict the masked orders which were not traced
pca_fits[:,:,iobj] = pca_trace((sobjs_final[igroup].trace_spat).T, usepca = None, npca = npca, npoly_cen = npoly_cen)
# usepca = sobjs_final[igroup].ech_usepca,
# Perform iterative flux weighted centroiding using new PCA predictions
xinit_fweight = pca_fits[:,:,iobj].copy()
inmask_now = inmask & (ordermask > 0)
xfit_fweight = extract.iter_tracefit(image, xinit_fweight, ncoeff, inmask = inmask_now, show_fits=show_fits)
# Perform iterative Gaussian weighted centroiding
xinit_gweight = xfit_fweight.copy()
xfit_gweight = extract.iter_tracefit(image, xinit_gweight, ncoeff, inmask = inmask_now, gweight=True,show_fits=show_fits)
# Assign the new traces
for iord, spec in enumerate(sobjs_final[igroup]):
spec.trace_spat = xfit_gweight[:,iord]
spec.spat_pixpos = spec.trace_spat[specmid]
# Set the IDs
sobjs_final.set_idx()
if show_trace:
viewer, ch = ginga.show_image(objminsky*(ordermask > 0))
for iobj in range(nobj_trim):
for iord in range(norders):
ginga.show_trace(viewer, ch, pca_fits[:,iord, iobj], str(uni_frac[iobj]), color='yellow')
for spec in sobjs_trim:
color = 'green' if spec.ech_usepca else 'magenta'
ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)
#for spec in sobjs_final:
# color = 'red' if spec.ech_usepca else 'green'
# ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)
return sobjs_final
#tslits_dict=tslits_dict,
#tilts=mstilts,
det=1,
#datasec_img=datasec_img,
#bpm=bpm,
#pixlocn=pixlocn,
#fitstbl=fitstbl)
)
sciframe_B, sciivar_B, rawvarframe_B, crmask_B = sciIMG_B.process(bias_subtract=None,
pixel_flat=None,
bpm=bpm,
apply_gain=True,
trim=False)
# This addition of a tiny bit of effor imposes a maximum S/N ratio which stabilizes the fits
adderr = 0.01 # Additional error to add to the formal errors, as a fraction of the flux; default to 0.01 (1 per cent).
ivar_A = utils.calc_ivar(rawvarframe_A) * (crmask_A == False)
gmask_A = (ivar_A > 0.0).astype(int) # = 1 for good points, 0 for bad points
ivar_A = gmask_A / (1.0 / (ivar_A + (1.0 - gmask_A)) + (adderr ** 2) * (np.abs(sciframe_A)) ** 2)
ivar_B = utils.calc_ivar(rawvarframe_B) * (crmask_B == False)
gmask_B = (ivar_B > 0.0).astype(int) # = 1 for good points, 0 for bad points
ivar_B = gmask_B / (1.0 / (ivar_B + (1.0 - gmask_B)) + (adderr ** 2) * (np.abs(sciframe_B)) ** 2)
# Diff the images
diff_AB = sciframe_A - sciframe_B
var_A = utils.calc_ivar(ivar_A)
var_B = utils.calc_ivar(ivar_B)
var_AB = var_A + var_B
# I think processimages already masks saturation but am not sure
mask_AB = (sciframe_A > 0.0) & (sciframe_B > 0.0) & \
(sciframe_A < spectrograph.detector[0]['saturation']) & \
(sciframe_B < spectrograph.detector[0]['saturation']) & \
(crmask_A == False) & (crmask_B == False)
