def extract_one(self, frames, det, bg_frames=[], std_outfile=None):
"""
Extract a single exposure/detector pair
sci_ID and det need to have been set internally prior to calling this method
Args:
frames (list): List of frames to extract; stacked if more than one is provided
det (int):
bg_frames (list, optional): List of frames to use as the background
std_outfile (str, optional):
Returns:
eight objects are returned::
- ndarray: Science image
- ndarray: Science inverse variance image
- ndarray: Model of the sky
- ndarray: Model of the object
- ndarray: Model of inverse variance
- ndarray: Mask
- :obj:`pypeit.specobjs.SpecObjs`: spectra
- astropy.units.Quantity: velocity correction
"""
# Grab some meta-data needed for the reduction from the fitstbl
self.objtype, self.setup, self.obstime, self.basename, self.binning = self.get_sci_metadata(frames[0], det)
# Is this a standard star?
self.std_redux = 'standard' in self.objtype
# Get the standard trace if need be
std_trace = self.get_std_trace(self.std_redux, det, std_outfile)
# Instantiate ScienceImage for the files we will reduce
sci_files = self.fitstbl.frame_paths(frames)
self.sciI = scienceimage.ScienceImage(self.spectrograph, sci_files,
bg_file_list=self.fitstbl.frame_paths(bg_frames),
ir_redux = self.ir_redux,
par=self.par['scienceframe'],
det=det,
binning=self.binning)
# For QA on crash.
msgs.sciexp = self.sciI
# Process images (includes inverse variance image, rn2 image, and CR mask)
self.sciimg, self.sciivar, self.rn2img, self.mask, self.crmask = \
self.sciI.proc(self.caliBrate.msbias, self.caliBrate.mspixflatnrm.copy(),
self.caliBrate.msbpm, illum_flat=self.caliBrate.msillumflat,
show=self.show)
# Object finding, first pass on frame without sky subtraction
self.maskslits = self.caliBrate.maskslits.copy()
self.redux = reduce.instantiate_me(self.spectrograph, self.caliBrate.tslits_dict,
self.mask, self.par,
ir_redux = self.ir_redux,
objtype=self.objtype, setup=self.setup,
det=det, binning=self.binning)
# Prep for manual extraction (if requested)
manual_extract_dict = self.fitstbl.get_manual_extract(frames, det)
# Do one iteration of object finding, and sky subtract to get initial sky model
self.sobjs_obj, self.nobj, skymask_init = \
self.redux.find_objects(self.sciimg, self.sciivar, std=self.std_redux, ir_redux=self.ir_redux,
std_trace=std_trace,maskslits=self.maskslits,
show=self.show & (not self.std_redux),
manual_extract_dict=manual_extract_dict)
# Global sky subtraction, first pass. Uses skymask from object finding step above
self.initial_sky = \
self.redux.global_skysub(self.sciimg, self.sciivar, self.caliBrate.tilts_dict['tilts'], skymask=skymask_init,
std=self.std_redux, maskslits=self.maskslits, show=self.show)
if not self.std_redux:
# Object finding, second pass on frame *with* sky subtraction. Show here if requested
self.sobjs_obj, self.nobj, self.skymask = \
self.redux.find_objects(self.sciimg - self.initial_sky, self.sciivar, std=self.std_redux, ir_redux=self.ir_redux,
std_trace=std_trace,maskslits=self.maskslits,show=self.show,
manual_extract_dict=manual_extract_dict)
# If there are objects, do 2nd round of global_skysub, local_skysub_extract, flexure, geo_motion
if self.nobj > 0:
# Global sky subtraction second pass. Uses skymask from object finding
self.global_sky = self.initial_sky if self.std_redux else \
self.redux.global_skysub(self.sciimg, self.sciivar, self.caliBrate.tilts_dict['tilts'],
skymask=self.skymask, maskslits=self.maskslits, show=self.show)
self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs = \
self.redux.local_skysub_extract(self.sciimg, self.sciivar, self.caliBrate.tilts_dict['tilts'], self.caliBrate.mswave,
self.global_sky, self.rn2img, self.sobjs_obj,
model_noise=(not self.ir_redux),std = self.std_redux,
maskslits=self.maskslits, show_profile=self.show,show=self.show)
# Purge out the negative objects if this was a near-IR reduction.
# TODO should we move this purge call to local_skysub_extract??
if self.ir_redux:
self.sobjs.purge_neg()
# Flexure correction if this is not a standard star
if not self.std_redux:
self.redux.flexure_correct(self.sobjs, self.basename)
# Grab coord
radec = ltu.radec_to_coord((self.fitstbl["ra"][frames[0]], self.fitstbl["dec"][frames[0]]))
self.vel_corr = self.redux.helio_correct(self.sobjs, radec, self.obstime)
else:
# Print status message
msgs_string = 'No objects to extract for target {:s}'.format(self.fitstbl['target'][frames[0]]) + msgs.newline()
msgs_string += 'On frames:' + msgs.newline()
for iframe in frames:
msgs_string += '{0:s}'.format(self.fitstbl['filename'][iframe]) + msgs.newline()
msgs.warn(msgs_string)
# set to first pass global sky
self.skymodel = self.initial_sky
self.objmodel = np.zeros_like(self.sciimg)
# Set to sciivar. Could create a model but what is the point?
self.ivarmodel = np.copy(self.sciivar)
# Set to the initial mask in case no objects were found
self.outmask = self.redux.mask
# empty specobjs object from object finding
if self.ir_redux:
self.sobjs_obj.purge_neg()
self.sobjs = self.sobjs_obj
self.vel_corr = None
return self.sciimg, self.sciivar, self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs, self.vel_corr
def extract_coadd2d(stack_dict, master_dir, samp_fact = 1.0,ir_redux=False, par=None, std=False, show=False, show_peaks=False):
"""
Main routine to run the extraction for 2d coadds.
Algorithm steps are as follows:
- Fill this in.
This performs 2d coadd specific tasks, and then also performs some
of the tasks analogous to the pypeit.extract_one method. Docs coming
soon....
Args:
stack_dict:
master_dir:
samp_fact: float
sampling factor to make the wavelength grid finer or coarser. samp_fact > 1.0 oversamples (finer),
samp_fact < 1.0 undersamples (coarser)
ir_redux:
par:
show:
show_peaks:
Returns:
"""
# Find the objid of the brighest object, and the average snr across all orders
nslits = stack_dict['tslits_dict']['slit_left'].shape[1]
objid, snr_bar = get_brightest_obj(stack_dict['specobjs_list'], echelle=True)
# TODO Print out a report here on the image stack, i.e. S/N of each image
spectrograph = util.load_spectrograph(stack_dict['spectrograph'])
par = spectrograph.default_pypeit_par() if par is None else par
binning = np.array([stack_dict['tslits_dict']['binspectral'],stack_dict['tslits_dict']['binspatial']])
# Grab the wavelength grid that we will rectify onto
wave_grid = spectrograph.wavegrid(binning=binning,samp_fact=samp_fact)
wave_grid_mid = spectrograph.wavegrid(midpoint=True,binning=binning,samp_fact=samp_fact)
coadd_list = []
nspec_vec = np.zeros(nslits,dtype=int)
nspat_vec = np.zeros(nslits,dtype=int)
# TODO: Generalize this to be a loop over detectors, such tha the
# coadd_list is an ordered dict (perhaps) with all the slits on all
# detectors
for islit in range(nslits):
msgs.info('Performing 2d coadd for slit: {:d}/{:d}'.format(islit,nslits-1))
# Determine the wavelength dependent optimal weights and grab the reference trace
rms_sn, weights, trace_stack, wave_stack = optimal_weights(stack_dict['specobjs_list'],
islit, objid)
thismask_stack = stack_dict['slitmask_stack'] == islit
# Perform the 2d coadd
coadd_dict = coadd2d(trace_stack, stack_dict['sciimg_stack'], stack_dict['sciivar_stack'],
stack_dict['skymodel_stack'], stack_dict['mask_stack'] == 0,
stack_dict['tilts_stack'], stack_dict['waveimg_stack'],
thismask_stack, weights=weights, wave_grid=wave_grid)
coadd_list.append(coadd_dict)
nspec_vec[islit]=coadd_dict['nspec']
nspat_vec[islit]=coadd_dict['nspat']
# Determine the size of the psuedo image
nspat_pad = 10
nspec_psuedo = nspec_vec.max()
nspat_psuedo = np.sum(nspat_vec) + (nslits + 1)*nspat_pad
spec_vec_psuedo = np.arange(nspec_psuedo)
shape_psuedo = (nspec_psuedo, nspat_psuedo)
imgminsky_psuedo = np.zeros(shape_psuedo)
sciivar_psuedo = np.zeros(shape_psuedo)
waveimg_psuedo = np.zeros(shape_psuedo)
tilts_psuedo = np.zeros(shape_psuedo)
spat_psuedo = np.zeros(shape_psuedo)
nused_psuedo = np.zeros(shape_psuedo, dtype=int)
inmask_psuedo = np.zeros(shape_psuedo, dtype=bool)
wave_mid = np.zeros((nspec_psuedo, nslits))
wave_mask = np.zeros((nspec_psuedo, nslits),dtype=bool)
wave_min = np.zeros((nspec_psuedo, nslits))
wave_max = np.zeros((nspec_psuedo, nslits))
dspat_mid = np.zeros((nspat_psuedo, nslits))
spat_left = nspat_pad
slit_left = np.zeros((nspec_psuedo, nslits))
slit_righ = np.zeros((nspec_psuedo, nslits))
spec_min1 = np.zeros(nslits)
spec_max1 = np.zeros(nslits)
for islit, coadd_dict in enumerate(coadd_list):
spat_righ = spat_left + nspat_vec[islit]
ispec = slice(0,nspec_vec[islit])
ispat = slice(spat_left,spat_righ)
imgminsky_psuedo[ispec, ispat] = coadd_dict['imgminsky']
sciivar_psuedo[ispec, ispat] = coadd_dict['sciivar']
waveimg_psuedo[ispec, ispat] = coadd_dict['waveimg']
tilts_psuedo[ispec, ispat] = coadd_dict['tilts']
# spat_psuedo is the sub-pixel image position on the rebinned psuedo image
inmask_psuedo[ispec, ispat] = coadd_dict['outmask']
image_temp = (coadd_dict['dspat'] - coadd_dict['dspat_mid'][0] + spat_left)*coadd_dict['outmask']
spat_psuedo[ispec, ispat] = image_temp
nused_psuedo[ispec, ispat] = coadd_dict['nused']
wave_min[ispec, islit] = coadd_dict['wave_min']
wave_max[ispec, islit] = coadd_dict['wave_max']
wave_mid[ispec, islit] = coadd_dict['wave_mid']
wave_mask[ispec, islit] = True
# Fill in the rest of the wave_mid with the corresponding points in the wave_grid
wave_this = wave_mid[wave_mask[:,islit], islit]
ind_upper = np.argmin(np.abs(wave_grid_mid - np.max(wave_this.max()))) + 1
if nspec_vec[islit] != nspec_psuedo:
wave_mid[nspec_vec[islit]:, islit] = wave_grid_mid[ind_upper:ind_upper + (nspec_psuedo-nspec_vec[islit])]
dspat_mid[ispat, islit] = coadd_dict['dspat_mid']
slit_left[:,islit] = np.full(nspec_psuedo, spat_left)
slit_righ[:,islit] = np.full(nspec_psuedo, spat_righ)
spec_max1[islit] = nspec_vec[islit]-1
spat_left = spat_righ + nspat_pad
slitcen = (slit_left + slit_righ)/2.0
tslits_dict_psuedo = dict(slit_left=slit_left, slit_righ=slit_righ, slitcen=slitcen,
nspec=nspec_psuedo, nspat=nspat_psuedo, pad=0,
nslits = nslits, binspectral=1, binspatial=1, spectrograph=spectrograph.spectrograph,
spec_min=spec_min1, spec_max=spec_max1)
slitmask_psuedo = pixels.tslits2mask(tslits_dict_psuedo)
# This is a kludge to deal with cases where bad wavelengths result in large regions where the slit is poorly sampled,
# which wreaks havoc on the local sky-subtraction
min_slit_frac = 0.70
spec_min = np.zeros(nslits)
spec_max = np.zeros(nslits)
for islit in range(nslits):
slit_width = np.sum(inmask_psuedo*(slitmask_psuedo == islit),axis=1)
slit_width_img = np.outer(slit_width, np.ones(nspat_psuedo))
med_slit_width = np.median(slit_width_img[slitmask_psuedo==islit])
nspec_eff = np.sum(slit_width > min_slit_frac*med_slit_width)
nsmooth = int(np.fmax(np.ceil(nspec_eff*0.02),10))
slit_width_sm = scipy.ndimage.filters.median_filter(slit_width, size=nsmooth, mode='reflect')
igood = (slit_width_sm > min_slit_frac*med_slit_width)
spec_min[islit] = spec_vec_psuedo[igood].min()
spec_max[islit] = spec_vec_psuedo[igood].max()
bad_pix = (slit_width_img < min_slit_frac*med_slit_width) & (slitmask_psuedo == islit)
inmask_psuedo[bad_pix] = False
# Update with tslits_dict_psuedo
tslits_dict_psuedo['spec_min'] = spec_min
tslits_dict_psuedo['spec_max'] = spec_max
slitmask_psuedo = pixels.tslits2mask(tslits_dict_psuedo)
# Make a fake bitmask from the outmask. We are kludging the crmask to be the outmask_psuedo here, and setting the bpm to
# be good everywhere
mask = processimages.ProcessImages.build_mask(imgminsky_psuedo, sciivar_psuedo, np.invert(inmask_psuedo),
np.zeros_like(inmask_psuedo), slitmask=slitmask_psuedo)
redux = reduce.instantiate_me(spectrograph, tslits_dict_psuedo, mask, ir_redux=ir_redux, par=par,
objtype = 'science', binning=binning)
if show:
redux.show('image', image=imgminsky_psuedo*(mask == 0), chname = 'imgminsky', slits=True, clear=True)
# Object finding
sobjs_obj, nobj, skymask_init = redux.find_objects(imgminsky_psuedo, sciivar_psuedo, ir_redux=ir_redux, std=std,
show_peaks=show_peaks, show=show)
# Local sky-subtraction
global_sky_psuedo = np.zeros_like(imgminsky_psuedo) # No global sky for co-adds since we go straight to local
rn2img_psuedo = global_sky_psuedo # No rn2img for co-adds since we go do not model noise
skymodel_psuedo, objmodel_psuedo, ivarmodel_psuedo, outmask_psuedo, sobjs = \
redux.local_skysub_extract(imgminsky_psuedo, sciivar_psuedo, tilts_psuedo, waveimg_psuedo, global_sky_psuedo,
rn2img_psuedo, sobjs_obj, spat_pix=spat_psuedo, std=std,
model_noise=False, show_profile=show, show=show)
if ir_redux:
sobjs.purge_neg()
# Add the information about the fixed wavelength grid to the sobjs
for spec in sobjs:
spec.boxcar['WAVE_GRID_MASK'] = wave_mask[:,spec.slitid]
spec.boxcar['WAVE_GRID'] = wave_mid[:,spec.slitid]
spec.boxcar['WAVE_GRID_MIN'] = wave_min[:,spec.slitid]
spec.boxcar['WAVE_GRID_MAX'] = wave_max[:,spec.slitid]
spec.optimal['WAVE_GRID_MASK'] = wave_mask[:,spec.slitid]
spec.optimal['WAVE_GRID'] = wave_mid[:,spec.slitid]
spec.optimal['WAVE_GRID_MIN'] = wave_min[:,spec.slitid]
spec.optimal['WAVE_GRID_MAX'] = wave_max[:,spec.slitid]
# TODO Implement flexure and heliocentric corrections on the single exposure 1d reductions and apply them to the
# waveimage. Change the data model to accomodate a wavelength model for each image.
# Using the same implementation as in core/pypeit
# Write out the psuedo master files to disk
master_key_dict = stack_dict['master_key_dict']
# TODO: These saving operations are a temporary kludge
waveImage = WaveImage(None, None, None, None, None, master_key=master_key_dict['arc'],
master_dir=master_dir)
waveImage.save(mswave=waveimg_psuedo)
traceSlits = TraceSlits(None, None, master_key=master_key_dict['trace'], master_dir=master_dir)
traceSlits.save(tslits_dict=tslits_dict_psuedo)
return imgminsky_psuedo, sciivar_psuedo, skymodel_psuedo, objmodel_psuedo, ivarmodel_psuedo, outmask_psuedo, sobjs
def extract_one(self, frames, det, bg_frames, std_outfile=None):
"""
Extract a single exposure/detector pair
sci_ID and det need to have been set internally prior to calling this method
Args:
frames (list):
List of frames to extract; stacked if more than one is provided
det (int):
bg_frames (list):
List of frames to use as the background
Can be empty
std_outfile (str, optional):
Returns:
seven objects are returned::
- ndarray: Science image
- ndarray: Science inverse variance image
- ndarray: Model of the sky
- ndarray: Model of the object
- ndarray: Model of inverse variance
- ndarray: Mask
- :obj:`pypeit.specobjs.SpecObjs`: spectra
"""
# Grab some meta-data needed for the reduction from the fitstbl
self.objtype, self.setup, self.obstime, self.basename, self.binning = self.get_sci_metadata(
frames[0], det)
# Is this a standard star?
self.std_redux = 'standard' in self.objtype
# Get the standard trace if need be
std_trace = self.get_std_trace(self.std_redux, det, std_outfile)
# Build Science image
sci_files = self.fitstbl.frame_paths(frames)
self.sciImg = scienceimage.build_from_file_list(
self.spectrograph,
det,
self.par['scienceframe']['process'],
self.caliBrate.msbpm,
sci_files,
self.caliBrate.msbias,
self.caliBrate.mspixelflat,
illum_flat=self.caliBrate.msillumflat)
# Background Image?
if len(bg_frames) > 0:
bg_file_list = self.fitstbl.frame_paths(bg_frames)
self.sciImg = self.sciImg - scienceimage.build_from_file_list(
self.spectrograph,
det,
self.par['scienceframe']['process'],
self.caliBrate.msbpm,
bg_file_list,
self.caliBrate.msbias,
self.caliBrate.mspixelflat,
illum_flat=self.caliBrate.msillumflat)
# Update mask for slitmask
slitmask = pixels.tslits2mask(self.caliBrate.tslits_dict)
self.sciImg.update_mask_slitmask(slitmask)
# For QA on crash
msgs.sciexp = self.sciImg
# Instantiate Reduce object
self.maskslits = self.caliBrate.tslits_dict['maskslits'].copy()
# Required for pypeline specific object
# TODO -- caliBrate should be replaced by the ~3 primary Objects needed
# once we have the data models in place.
self.redux = reduce.instantiate_me(self.sciImg,
self.spectrograph,
self.par,
self.caliBrate,
maskslits=self.maskslits,
ir_redux=self.ir_redux,
std_redux=self.std_redux,
objtype=self.objtype,
setup=self.setup,
show=self.show,
det=det,
binning=self.binning)
# Show?
if self.show:
self.redux.show('image',
image=self.sciImg.image,
chname='processed',
slits=True,
clear=True)
# Prep for manual extraction (if requested)
manual_extract_dict = self.fitstbl.get_manual_extract(frames, det)
self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs = self.redux.run(
std_trace=std_trace,
manual_extract_dict=manual_extract_dict,
show_peaks=self.show,
basename=self.basename,
ra=self.fitstbl["ra"][frames[0]],
dec=self.fitstbl["dec"][frames[0]],
obstime=self.obstime)
# Return
return self.sciImg.image, self.sciImg.ivar, self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs