def test_init():
""" Run the parameter setup script
"""
# Null
sobjs1 = specobjs.SpecObjs()
# With a few objs
sobjs2 = specobjs.SpecObjs([sobj1,sobj2])
def test_add_rm(sobj1, sobj2, sobj3):
sobjs = specobjs.SpecObjs([sobj1, sobj2])
sobjs.add_sobj(sobj3)
assert sobjs.nobj == 3
# Remove
sobjs.remove_sobj(2)
assert len(sobjs.specobjs) == 2
# Numpy 18
sobjs1 = specobjs.SpecObjs()
sobjs2 = specobjs.SpecObjs()
sobjs2.add_sobj(sobjs1)
def test_io(sobj1, sobj2, sobj3, sobj4):
sobjs = specobjs.SpecObjs([sobj1, sobj2, sobj3, sobj4])
sobjs[0]['BOX_WAVE'] = np.arange(1000).astype(float)
sobjs[1]['BOX_WAVE'] = np.arange(1000).astype(float)
sobjs[2]['BOX_WAVE'] = np.arange(1000).astype(float)
#sobjs[0]['BOX_COUNTS'] = np.ones_like(sobjs[0].BOX_WAVE) # This tests single array
sobjs[1]['BOX_COUNTS'] = np.ones_like(sobjs[0].BOX_WAVE)
sobjs[2]['BOX_COUNTS'] = np.ones_like(sobjs[0].BOX_WAVE)
# Detector
sobjs[0]['DETECTOR'] = tstutils.get_kastb_detector()
tmp = tstutils.get_kastb_detector()
tmp['det'] = 2
sobjs[1]['DETECTOR'] = tmp
# Write
header = fits.PrimaryHDU().header
header['TST'] = 'TEST'
ofile = data_path('tst_specobjs.fits')
if os.path.isfile(ofile):
os.remove(ofile)
sobjs.write_to_fits(header, ofile, overwrite=False)
# Read
hdul = fits.open(ofile)
assert len(hdul) == 7 # Primary + 4 Obj + 2 Detectors
assert hdul[0].header['NSPEC'] == 4
hdul.close()
#
_sobjs = specobjs.SpecObjs.from_fitsfile(ofile)
assert _sobjs.nobj == 4
assert np.array_equal(sobjs[0].BOX_WAVE, _sobjs[0].BOX_WAVE)
assert np.array_equal(sobjs[1].BOX_WAVE, _sobjs[1].BOX_WAVE)
_sobjs.write_to_fits(header, ofile, overwrite=True)
# Detector
assert _sobjs[0].DETECTOR is not None, '1st object started with Detector'
assert _sobjs[
1].DETECTOR is not None, '2nd object has DET=1 so should get decorated'
assert _sobjs[2].DETECTOR is None
# Now try updates!
sobjs1 = specobjs.SpecObjs([sobj1])
sobjs[0]['BOX_WAVE'] = np.arange(2000).astype(float)
sobjs1[0]['DETECTOR'] = tstutils.get_kastb_detector()
header1 = fits.PrimaryHDU().header
sobjs1.write_to_fits(header1, ofile, overwrite=True, update_det=1)
# Test
_sobjs1 = specobjs.SpecObjs.from_fitsfile(ofile)
assert _sobjs1.nobj == 3
assert _sobjs1[2].BOX_WAVE.size == 2000
os.remove(ofile)
def test_add_rm():
sobjs = specobjs.SpecObjs([sobj1, sobj2])
sobjs.add_sobj(sobj3)
assert sobjs.nobj == 3
# Remove
sobjs.remove_sobj(2)
assert len(sobjs.specobjs) == 2
def test_geocorrect(fitstbl):
"""
"""
# Specobj (wrap in a list to mimic a slit)
npix = 1000
sobj = specobj.SpecObj('MultiSlit', 1, SLITID=0)
sobj.BOX_WAVE = np.linspace(4000., 6000., npix)
sobj.BOX_COUNTS = 50. * (sobj.BOX_WAVE / 5000.)**-1.
sobj.BOX_COUNTS_IVAR = 1. / sobj.BOX_COUNTS.copy()
# SpecObjs
specObjs = specobjs.SpecObjs()
specObjs.add_sobj(sobj)
scidx = 5
obstime = Time(fitstbl['mjd'][scidx],
format='mjd') #'%Y-%m-%dT%H:%M:%S.%f')
maskslits = np.array([False] * specObjs.nobj)
radec = ltu.radec_to_coord((fitstbl["ra"][scidx], fitstbl["dec"][scidx]))
helio, hel_corr = wave.geomotion_correct(specObjs, radec, obstime,
maskslits, lon, lat, alt,
'heliocentric')
assert np.isclose(helio, -9.17461338,
rtol=1e-5) # Checked against x_keckhelio
#assert np.isclose(helio, -9.3344957, rtol=1e-5) # Original
assert np.isclose(specObjs[0].BOX_WAVE[0], 3999.877589008, rtol=1e-8)
def test_save1d_fits():
""" save1d to FITS and HDF5
"""
# Init
fitstbl = dummy_fitstbl(spectro_name='shane_kast_blue', directory=data_path(''))
sobj = mk_specobj()
specObjs = specobjs.SpecObjs([sobj])
spectrograph = util.load_spectrograph('shane_kast_blue')
# Write to FITS
basename = 'test'
outfile = data_path('') + 'spec1d_{:s}.fits'.format(basename)
save.save_1d_spectra_fits(specObjs, fitstbl[5], spectrograph, outfile)
def test_set(sobj1, sobj2, sobj3):
sobjs = specobjs.SpecObjs([sobj1, sobj2, sobj3])
# All
sobjs.DET = 'DET03'
assert np.all(sobjs[:].DET == np.array(['DET03', 'DET03', 'DET03']))
sobjs[:].DET = 'DET04'
assert np.all(sobjs[:].DET == np.array(['DET04', 'DET04', 'DET04']))
# Slice
sobjs[1:2].DET = 'DET02'
assert sobjs.DET[1] == 'DET02'
# With logic
det2 = sobjs.DET == 'DET02'
sobjs[det2].PYPELINE = 'BLAH'
assert sobjs.PYPELINE[1] == 'BLAH'
assert sobjs.PYPELINE[0] == 'MultiSlit'
def test_set(sobj1, sobj2, sobj3):
sobjs = specobjs.SpecObjs([sobj1, sobj2, sobj3])
# All
sobjs.DET = 3
assert np.all(sobjs[:].DET == np.array([3, 3, 3]))
sobjs[:].DET = 4
assert np.all(sobjs[:].DET == np.array([4, 4, 4]))
# Slice
sobjs[1:2].DET = 2
assert sobjs.DET[1] == 2
# With logic
det2 = sobjs.DET == 2
sobjs[det2].PYPELINE = 'BLAH'
assert sobjs.PYPELINE[1] == 'BLAH'
assert sobjs.PYPELINE[0] == 'MultiSlit'
def test_set():
sobjs = specobjs.SpecObjs([sobj1,sobj2,sobj3])
# All
sobjs['det'] = 3
assert np.all(sobjs[:].det == np.array([3,3,3]))
# Slice
sobjs[1:2]['det'] = 2
assert sobjs.det[1] == 2
# Under the hood
sobjs.set(0, 'det', 3)
sobjs.set(slice(1,2), 'det', 2)
# Test
assert np.all(sobjs[:].det == np.array([3,2,3]))
# Hennawi test
idx = sobjs.det == 3
sobjs[idx]['det'] = 1
def test_geocorrect(fitstbl):
"""
"""
# Spectrograph
# (KBW) Had to change this to keck to match the telecope parameters,
# then just changed to use definitions above directly.
# spectrograph = load_spectrograph('keck_lris_blue')
# Specobjs (wrap in a list to mimic a slit)
sobj_list = specobjs.dummy_specobj((2048,2048), extraction=True)
specObjs = specobjs.SpecObjs(sobj_list)
scidx = 5
obstime = Time(fitstbl['mjd'][scidx], format='mjd')#'%Y-%m-%dT%H:%M:%S.%f')
maskslits = np.array([False]*specObjs.nobj)
radec = ltu.radec_to_coord((fitstbl["ra"][scidx], fitstbl["dec"][scidx]))
helio, hel_corr = wave.geomotion_correct(specObjs, radec, obstime, maskslits,
lon, lat, alt, 'heliocentric')
assert np.isclose(helio, -9.17461338, rtol=1e-5) # Checked against x_keckhelio
#assert np.isclose(helio, -9.3344957, rtol=1e-5) # Original
assert np.isclose(specObjs[0].boxcar['WAVE'][0].value, 3999.877589008, rtol=1e-8)
def write_science(sci_specobjs, sci_header, outfile):
"""
Write the flux-calibrated science spectra
Parameters
----------
outfile : str
Returns
-------
"""
if len(sci_specobjs) == 0:
msgs.warn("No science spectra to write to disk!")
#
if 'VEL-TYPE' in sci_header.keys():
helio_dict = dict(refframe=sci_header['VEL-TYPE'],
vel_correction=sci_header['VEL'])
else:
helio_dict = None
telescope = None
if 'LON-OBS' in sci_header.keys():
telescope = TelescopePar(longitude=sci_header['LON-OBS'],
latitude=sci_header['LAT-OBS'],
elevation=sci_header['ALT-OBS'])
# KLUDGE ME
if isinstance(sci_specobjs, list):
specObjs = specobjs.SpecObjs(sci_specobjs)
elif isinstance(sci_specobjs, specobjs.SpecObjs):
specObjs = sci_specobjs
else:
msgs.error("BAD INPUT")
save.save_1d_spectra_fits(specObjs,
sci_header,
'ECHELLE',
outfile,
helio_dict=helio_dict,
telescope=telescope,
overwrite=True)
def write_science(self, outfile):
"""
Write the flux-calibrated science spectra
Parameters
----------
outfile : str
Returns
-------
"""
if len(self.sci_specobjs) == 0:
msgs.warn("No science spectra to write to disk!")
#
if 'VEL-TYPE' in self.sci_header.keys():
helio_dict = dict(refframe=self.sci_header['VEL-TYPE'],
vel_correction=self.sci_header['VEL'])
else:
helio_dict = None
# KLUDGE ME
if isinstance(self.sci_specobjs, list):
specObjs = specobjs.SpecObjs(self.sci_specobjs)
elif isinstance(self.sci_specobjs, specobjs.SpecObjs):
specObjs = self.sci_specobjs
else:
msgs.error("BAD INPUT")
#save_1d_spectra_fits(specObjs, header, spectrograph, outfile, helio_dict=None, overwrite=True,update_det=None)
save.save_1d_spectra_fits(specObjs,
self.sci_header,
self.spectrograph,
outfile,
helio_dict=helio_dict,
overwrite=True)
# Step
self.steps.append(inspect.stack()[0][3])
def reduce_exposure(self, frames, bg_frames=None, std_outfile=None):
"""
Reduce a single exposure
Args:
frame (:obj:`int`):
0-indexed row in :attr:`fitstbl` with the frame to
reduce.
bg_frames (:obj:`list`, optional):
List of frame indices for the background.
std_outfile (:obj:`str`, optional):
File with a previously reduced standard spectrum from
PypeIt.
Returns:
dict: The dictionary containing the primary outputs of
extraction.
"""
# TODO:
# - change doc string to reflect that more than one frame can be
# provided
# if show is set, clear the ginga channels at the start of each new sci_ID
if self.show:
# TODO: Put this in a try/except block?
display.clear_all()
has_bg = True if bg_frames is not None and len(
bg_frames) > 0 else False
# Is this an IR reduction?
# TODO: Why specific to IR?
self.ir_redux = True if has_bg else False
# Container for all the Spec2DObj
all_spec2d = spec2dobj.AllSpec2DObj()
all_spec2d['meta']['ir_redux'] = self.ir_redux
# TODO -- Should we reset/regenerate self.slits.mask for a new exposure
all_specobjs = specobjs.SpecObjs()
# Print status message
msgs_string = 'Reducing target {:s}'.format(
self.fitstbl['target'][frames[0]]) + msgs.newline()
# TODO: Print these when the frames are actually combined,
# backgrounds are used, etc?
msgs_string += 'Combining frames:' + msgs.newline()
for iframe in frames:
msgs_string += '{0:s}'.format(
self.fitstbl['filename'][iframe]) + msgs.newline()
msgs.info(msgs_string)
if has_bg:
bg_msgs_string = ''
for iframe in bg_frames:
bg_msgs_string += '{0:s}'.format(
self.fitstbl['filename'][iframe]) + msgs.newline()
bg_msgs_string = msgs.newline(
) + 'Using background from frames:' + msgs.newline(
) + bg_msgs_string
msgs.info(bg_msgs_string)
# Find the detectors to reduce
detectors = PypeIt.select_detectors(
detnum=self.par['rdx']['detnum'],
slitspatnum=self.par['rdx']['slitspatnum'],
ndet=self.spectrograph.ndet)
if len(detectors) != self.spectrograph.ndet:
msgs.warn('Not reducing detectors: {0}'.format(' '.join([
str(d) for d in set(np.arange(self.spectrograph.ndet)) -
set(detectors)
])))
# Loop on Detectors
# TODO: Attempt to put in a multiprocessing call here?
for self.det in detectors:
msgs.info("Working on detector {0}".format(self.det))
# Instantiate Calibrations class
self.caliBrate = calibrations.Calibrations.get_instance(
self.fitstbl,
self.par['calibrations'],
self.spectrograph,
self.calibrations_path,
qadir=self.qa_path,
reuse_masters=self.reuse_masters,
show=self.show,
slitspat_num=self.par['rdx']['slitspatnum'])
# These need to be separate to accomodate COADD2D
self.caliBrate.set_config(frames[0], self.det,
self.par['calibrations'])
self.caliBrate.run_the_steps()
# Extract
# TODO: pass back the background frame, pass in background
# files as an argument. extract one takes a file list as an
# argument and instantiates science within
all_spec2d[self.det], tmp_sobjs \
= self.reduce_one(frames, self.det, bg_frames, std_outfile=std_outfile)
# Hold em
if tmp_sobjs.nobj > 0:
all_specobjs.add_sobj(tmp_sobjs)
# JFH TODO write out the background frame?
# TODO -- Save here? Seems like we should. Would probably need to use update_det=True
# Return
return all_spec2d, all_specobjs
def test_access(sobj1, sobj2):
sobjs = specobjs.SpecObjs([sobj1, sobj2])
#
assert sobjs[0]['PYPELINE'] == 'MultiSlit'
assert len(sobjs['PYPELINE']) == 2
def test_access():
sobjs = specobjs.SpecObjs([sobj1,sobj2])
#
assert sobjs[0]['shape'] == shape
assert len(sobjs['shape']) == 2
def reduce_exposure(self, frames, bg_frames=None, std_outfile=None):
"""
Reduce a single exposure
Args:
frames (:obj:`list`):
List of 0-indexed rows in :attr:`fitstbl` with the frames to
reduce.
bg_frames (:obj:`list`, optional):
List of frame indices for the background.
std_outfile (:obj:`str`, optional):
File with a previously reduced standard spectrum from
PypeIt.
Returns:
dict: The dictionary containing the primary outputs of
extraction.
"""
# if show is set, clear the ginga channels at the start of each new sci_ID
if self.show:
# TODO: Put this in a try/except block?
display.clear_all(allow_new=True)
has_bg = True if bg_frames is not None and len(
bg_frames) > 0 else False
# Is this an b/g subtraction reduction?
if has_bg:
self.bkg_redux = True
# The default is to find_negative objects if the bg_frames are classified as "science", and to not find_negative
# objects if the bg_frames are classified as "sky". This can be explicitly overridden if
# par['reduce']['findobj']['find_negative'] is set to something other than the default of None.
self.find_negative = (('science' in self.fitstbl['frametype'][bg_frames[0]]) |
('standard' in self.fitstbl['frametype'][bg_frames[0]]))\
if self.par['reduce']['findobj']['find_negative'] is None else self.par['reduce']['findobj']['find_negative']
else:
self.bkg_redux = False
self.find_negative = False
# Container for all the Spec2DObj
all_spec2d = spec2dobj.AllSpec2DObj()
all_spec2d['meta']['bkg_redux'] = self.bkg_redux
all_spec2d['meta']['find_negative'] = self.find_negative
# TODO -- Should we reset/regenerate self.slits.mask for a new exposure
# container for specobjs during first loop (objfind)
all_specobjs_objfind = specobjs.SpecObjs()
# container for specobjs during second loop (extraction)
all_specobjs_extract = specobjs.SpecObjs()
# list of global_sky obtained during objfind and used in extraction
initial_sky_list = []
# list of sciImg
sciImg_list = []
# List of detectors with successful calibration
calibrated_det = []
# list of successful MasterSlits calibrations to be used in the extraction loop
calib_slits = []
# List of objFind objects
objFind_list = []
# Print status message
msgs_string = 'Reducing target {:s}'.format(
self.fitstbl['target'][frames[0]]) + msgs.newline()
# TODO: Print these when the frames are actually combined,
# backgrounds are used, etc?
msgs_string += 'Combining frames:' + msgs.newline()
for iframe in frames:
msgs_string += '{0:s}'.format(
self.fitstbl['filename'][iframe]) + msgs.newline()
msgs.info(msgs_string)
if has_bg:
bg_msgs_string = ''
for iframe in bg_frames:
bg_msgs_string += '{0:s}'.format(
self.fitstbl['filename'][iframe]) + msgs.newline()
bg_msgs_string = msgs.newline(
) + 'Using background from frames:' + msgs.newline(
) + bg_msgs_string
msgs.info(bg_msgs_string)
# Find the detectors to reduce
subset = self.par['rdx']['slitspatnum'] if self.par['rdx']['slitspatnum'] is not None \
else self.par['rdx']['detnum']
detectors = self.spectrograph.select_detectors(subset=subset)
msgs.info(f'Detectors to work on: {detectors}')
# Loop on Detectors
# TODO: Attempt to put in a multiprocessing call here?
# objfind
for self.det in detectors:
msgs.info("Working on detector {0}".format(self.det))
# run calibration
self.caliBrate = self.calib_one(frames, self.det)
if not self.caliBrate.success:
msgs.warn(
f'Calibrations for detector {self.det} were unsuccessful! The step '
f'that failed was {self.caliBrate.failed_step}. Continuing by '
f'skipping this detector.')
continue
# we save only the detectors that had a successful calibration,
# and we use only those in the extract loop below
calibrated_det.append(self.det)
# we also save the successful MasterSlits calibrations because they are used and modified
# in the slitmask stuff in between the two loops
calib_slits.append(self.caliBrate.slits)
# global_sky, skymask and sciImg are needed in the extract loop
initial_sky, sobjs_obj, sciImg, objFind = self.objfind_one(
frames, self.det, bg_frames, std_outfile=std_outfile)
if len(sobjs_obj) > 0:
all_specobjs_objfind.add_sobj(sobjs_obj)
initial_sky_list.append(initial_sky)
sciImg_list.append(sciImg)
objFind_list.append(objFind)
# slitmask stuff
if self.par['reduce']['slitmask']['assign_obj']:
# get object positions from slitmask design and slitmask offsets for all the detectors
spat_flexure = np.array([ss.spat_flexure for ss in sciImg_list])
# Grab platescale with binning
bin_spec, bin_spat = parse.parse_binning(self.binning)
platescale = np.array(
[ss.detector.platescale * bin_spat for ss in sciImg_list])
# get the dither offset if available
if self.par['reduce']['slitmask']['use_dither_offset']:
dither = self.spectrograph.parse_dither_pattern(
[self.fitstbl.frame_paths(frames[0])])
dither_off = dither[2][0] if dither is not None else None
else:
dither_off = None
calib_slits = slittrace.get_maskdef_objpos_offset_alldets(
all_specobjs_objfind,
calib_slits,
spat_flexure,
platescale,
self.par['calibrations']['slitedges']['det_buffer'],
self.par['reduce']['slitmask'],
dither_off=dither_off)
# determine if slitmask offsets exist and compute an average offsets over all the detectors
calib_slits = slittrace.average_maskdef_offset(
calib_slits, platescale[0],
self.spectrograph.list_detectors(
mosaic='MSC' in calib_slits[0].detname))
# slitmask design matching and add undetected objects
all_specobjs_objfind = slittrace.assign_addobjs_alldets(
all_specobjs_objfind, calib_slits, spat_flexure, platescale,
self.par['reduce']['slitmask'],
self.par['reduce']['findobj']['find_fwhm'])
# Extract
for i, self.det in enumerate(calibrated_det):
# re-run (i.e., load) calibrations
self.caliBrate = self.calib_one(frames, self.det)
self.caliBrate.slits = calib_slits[i]
detname = sciImg_list[i].detector.name
# TODO: pass back the background frame, pass in background
# files as an argument. extract one takes a file list as an
# argument and instantiates science within
if all_specobjs_objfind.nobj > 0:
all_specobjs_on_det = all_specobjs_objfind[
all_specobjs_objfind.DET == detname]
else:
all_specobjs_on_det = all_specobjs_objfind
# Extract
all_spec2d[detname], tmp_sobjs \
= self.extract_one(frames, self.det, sciImg_list[i], objFind_list[i],
initial_sky_list[i], all_specobjs_on_det)
# Hold em
if tmp_sobjs.nobj > 0:
all_specobjs_extract.add_sobj(tmp_sobjs)
# JFH TODO write out the background frame?
# TODO -- Save here? Seems like we should. Would probably need to use update_det=True
# Return
return all_spec2d, all_specobjs_extract
def find_objects_pypeline(self,
image,
std_trace=None,
manual_extract_dict=None,
show_peaks=False,
show_fits=False,
show_trace=False,
show=False,
debug=False):
"""
Pipeline specific find objects routine
Args:
image (np.ndarray):
std_trace (np.ndarray, optional):
manual_extract_dict (dict, optional):
show_peaks (bool, optional):
Generate QA showing peaks identified by object finding
show_fits (bool, optional):
Generate QA showing fits to traces
show_trace (bool, optional):
Generate QA showing traces identified. Requires an open ginga RC modules window
show (bool, optional):
debug (bool, optional):
Returns:
tuple:
specobjs : Specobjs object
Container holding Specobj objects
nobj (int):
Number of objects identified
skymask : ndarray
Boolean image indicating which pixels are useful for global sky subtraction
"""
gdslits = np.where(np.invert(self.reduce_bpm))[0]
# create the ouptut image for skymask
skymask = np.zeros_like(image, dtype=bool)
# Instantiate the specobjs container
sobjs = specobjs.SpecObjs()
# Loop on slits
for slit_idx in gdslits:
slit_spat = self.slits.spat_id[slit_idx]
qa_title = "Finding objects on slit # {:d}".format(slit_spat)
msgs.info(qa_title)
thismask = self.slitmask == slit_spat
inmask = (self.sciImg.fullmask == 0) & thismask
# Find objects
specobj_dict = {
'SLITID': slit_spat,
'DET': self.det,
'OBJTYPE': self.objtype,
'PYPELINE': self.pypeline
}
# TODO we need to add QA paths and QA hooks. QA should be
# done through objfind where all the relevant information
# is. This will be a png file(s) per slit.
sobjs_slit, skymask[thismask] = \
extract.objfind(image, thismask,
self.slits_left[:,slit_idx],
self.slits_right[:,slit_idx],
inmask=inmask, ir_redux=self.ir_redux,
ncoeff=self.par['reduce']['findobj']['trace_npoly'],
std_trace=std_trace,
sig_thresh=self.par['reduce']['findobj']['sig_thresh'],
hand_extract_dict=manual_extract_dict,
specobj_dict=specobj_dict, show_peaks=show_peaks,
show_fits=show_fits, show_trace=show_trace,
trim_edg=self.par['reduce']['findobj']['find_trim_edge'],
cont_fit=self.par['reduce']['findobj']['find_cont_fit'],
npoly_cont=self.par['reduce']['findobj']['find_npoly_cont'],
fwhm=self.par['reduce']['findobj']['find_fwhm'],
maxdev=self.par['reduce']['findobj']['find_maxdev'],
qa_title=qa_title, nperslit=self.par['reduce']['findobj']['maxnumber'],
debug_all=debug)
sobjs.add_sobj(sobjs_slit)
# Steps
self.steps.append(inspect.stack()[0][3])
if show:
self.show('image',
image=image * (self.sciImg.fullmask == 0),
chname='objfind',
sobjs=sobjs,
slits=True)
# Return
return sobjs, len(sobjs), skymask
def compute_offsets(self):
objid_bri, slitidx_bri, spatid_bri, snr_bar_bri = self.get_brightest_obj(self.stack_dict['specobjs_list'],
self.spat_ids)
msgs.info('Determining offsets using brightest object on slit: {:d} with avg SNR={:5.2f}'.format(spatid_bri,np.mean(snr_bar_bri)))
thismask_stack = self.stack_dict['slitmask_stack'] == spatid_bri
trace_stack_bri = np.zeros((self.nspec, self.nexp))
# TODO Need to think abbout whether we have multiple tslits_dict for each exposure or a single one
for iexp in range(self.nexp):
trace_stack_bri[:,iexp] = self.stack_dict['slits_list'][iexp].center[:,slitidx_bri]
# trace_stack_bri[:,iexp] = (self.stack_dict['tslits_dict_list'][iexp]['slit_left'][:,slitid_bri] +
# self.stack_dict['tslits_dict_list'][iexp]['slit_righ'][:,slitid_bri])/2.0
# Determine the wavelength grid that we will use for the current slit/order
wave_bins = coadd.get_wave_bins(thismask_stack, self.stack_dict['waveimg_stack'], self.wave_grid)
dspat_bins, dspat_stack = coadd.get_spat_bins(thismask_stack, trace_stack_bri)
sci_list = [self.stack_dict['sciimg_stack'] - self.stack_dict['skymodel_stack']]
var_list = []
msgs.info('Rebinning Images')
sci_list_rebin, var_list_rebin, norm_rebin_stack, nsmp_rebin_stack = coadd.rebin2d(
wave_bins, dspat_bins, self.stack_dict['waveimg_stack'], dspat_stack, thismask_stack,
(self.stack_dict['mask_stack'] == 0), sci_list, var_list)
thismask = np.ones_like(sci_list_rebin[0][0,:,:],dtype=bool)
nspec_pseudo, nspat_pseudo = thismask.shape
slit_left = np.full(nspec_pseudo, 0.0)
slit_righ = np.full(nspec_pseudo, nspat_pseudo)
inmask = norm_rebin_stack > 0
traces_rect = np.zeros((nspec_pseudo, self.nexp))
sobjs = specobjs.SpecObjs()
#specobj_dict = {'setup': 'unknown', 'slitid': 999, 'orderindx': 999, 'det': self.det, 'objtype': 'unknown',
# 'pypeline': 'MultiSLit' + '_coadd_2d'}
for iexp in range(self.nexp):
sobjs_exp, _ = extract.objfind(sci_list_rebin[0][iexp,:,:], thismask, slit_left, slit_righ,
inmask=inmask[iexp,:,:], ir_redux=self.ir_redux,
fwhm=self.par['reduce']['findobj']['find_fwhm'],
trim_edg=self.par['reduce']['findobj']['find_trim_edge'],
npoly_cont=self.par['reduce']['findobj']['find_npoly_cont'],
maxdev=self.par['reduce']['findobj']['find_maxdev'],
ncoeff=3, sig_thresh=self.par['reduce']['findobj']['sig_thresh'], nperslit=1,
find_min_max=self.par['reduce']['findobj']['find_min_max'],
show_trace=self.debug_offsets, show_peaks=self.debug_offsets)
sobjs.add_sobj(sobjs_exp)
traces_rect[:, iexp] = sobjs_exp.TRACE_SPAT
# Now deterimine the offsets. Arbitrarily set the zeroth trace to the reference
med_traces_rect = np.median(traces_rect,axis=0)
offsets = med_traces_rect[0] - med_traces_rect
# Print out a report on the offsets
msg_string = msgs.newline() + '---------------------------------------------'
msg_string += msgs.newline() + ' Summary of offsets for highest S/N object '
msg_string += msgs.newline() + ' found on slitid = {:d} '.format(spatid_bri)
msg_string += msgs.newline() + '---------------------------------------------'
msg_string += msgs.newline() + ' exp# offset '
for iexp, off in enumerate(offsets):
msg_string += msgs.newline() + ' {:d} {:5.2f}'.format(iexp, off)
msg_string += msgs.newline() + '-----------------------------------------------'
msgs.info(msg_string)
if self.debug_offsets:
for iexp in range(self.nexp):
plt.plot(traces_rect[:, iexp], linestyle='--', label='original trace')
plt.plot(traces_rect[:, iexp] + offsets[iexp], label='shifted traces')
plt.legend()
plt.show()
return objid_bri, spatid_bri, snr_bar_bri, offsets
def extinction_correction_tester(algorithm):
spec1d_file = data_path('spec1d_test.fits')
sens_file = data_path('sens_test.fits')
if os.path.isfile(spec1d_file):
os.remove(spec1d_file)
if os.path.isfile(sens_file):
os.remove(sens_file)
# make a bogus spectrum that has N_lam = 1
wave = np.linspace(4000, 6000)
counts = np.ones_like(wave)
ivar = np.ones_like(wave)
sobj = specobj.SpecObj.from_arrays('MultiSlit', wave, counts, ivar)
sobjs = specobjs.SpecObjs([sobj])
# choice of PYP_SPEC, DISPNAME and EXPTIME are unimportant here
# AIRMASS must be > 1
sobjs.write_to_fits(
{
'PYP_SPEC': 'p200_dbsp_blue',
'DISPNAME': '600/4000',
'EXPTIME': 1.0,
'AIRMASS': 1.1
}, spec1d_file)
par = pypeitpar.PypeItPar()
# set the senfunc algorithm
par['sensfunc']['algorithm'] = algorithm
# needed to initiate SensFunc (dummy standard star Feige34)
par['sensfunc']['star_ra'] = 159.9042
par['sensfunc']['star_dec'] = 43.1025
sensobj = sensfunc.SensFunc.get_instance(spec1d_file, sens_file,
par['sensfunc'])
sensobj.wave = np.linspace(3000, 6000, 300).reshape((300, 1))
sensobj.sens = sensobj.empty_sensfunc_table(*sensobj.wave.T.shape)
# make the zeropoint such that the sensfunc is flat
sensobj.zeropoint = 30 - np.log10(sensobj.wave**2) / 0.4
sensobj.to_file(sens_file)
# now flux our N_lam = 1 specobj
par['fluxcalib']['extinct_correct'] = None
fluxCalibrate = fluxcalibrate.MultiSlitFC([spec1d_file], [sens_file],
par=par['fluxcalib'])
# without extinction correction, we should get constant F_lam
# with extinction correction, the spectrum will be blue
# make sure that the appropriate default behavior occurred
sobjs = specobjs.SpecObjs.from_fitsfile(spec1d_file)
print(sobjs[0].keys())
if algorithm == 'UVIS':
assert sobjs[0]['OPT_FLAM'][0] > sobjs[0]['OPT_FLAM'][-1], \
"UVIS sensfunc was not extinction corrected by default, but should have been"
elif algorithm == 'IR':
assert np.isclose(sobjs[0]['OPT_FLAM'][0], sobjs[0]['OPT_FLAM'][-1]), \
"IR sensfunc was extinction corrected by default, but shouldn't have been"
# clean up
os.remove(spec1d_file)
os.remove(sens_file)
def load_specobjs(fname, order=None):
""" Load a spec1d file into a list of SpecObjExp objects
Parameters
----------
fname : str
Returns
-------
specObjs : list of SpecObjExp
head0
"""
sobjs = specobjs.SpecObjs()
speckeys = [
'WAVE', 'WAVE_GRID_MASK', 'WAVE_GRID', 'WAVE_GRID_MIN',
'WAVE_GRID_MAX', 'SKY', 'MASK', 'FLAM', 'FLAM_IVAR', 'FLAM_SIG',
'COUNTS_IVAR', 'COUNTS', 'COUNTS_SIG'
]
# sobjs_keys gives correspondence between header cards and sobjs attribute name
sobjs_key = specobjs.SpecObj.sobjs_key()
hdulist = fits.open(fname)
head0 = hdulist[0].header
#pypeline = head0['PYPELINE']
# Is this an Echelle reduction?
#if 'Echelle' in pypeline:
# echelle = True
#else:
# echelle = False
for hdu in hdulist:
if hdu.name == 'PRIMARY':
continue
# Parse name
idx = hdu.name
objp = idx.split('-')
if objp[-2][:5] == 'ORDER':
iord = int(objp[-2][5:])
else:
msgs.warn('Loading longslit data ?')
iord = int(-1)
if (order is not None) and (iord != order):
continue
specobj = specobjs.SpecObj(None, None, None, idx=idx)
# Assign specobj attributes from header cards
for attr, hdrcard in sobjs_key.items():
try:
value = hdu.header[hdrcard]
except:
continue
setattr(specobj, attr, value)
# Load data
spec = Table(hdu.data)
shape = (len(spec), 1024) # 2nd number is dummy
specobj.shape = shape
specobj.trace_spat = spec['TRACE']
# Add spectrum
if 'BOX_COUNTS' in spec.keys():
for skey in speckeys:
try:
specobj.boxcar[skey] = spec['BOX_{:s}'.format(skey)].data
except KeyError:
pass
# Add units on wave
specobj.boxcar['WAVE'] = specobj.boxcar['WAVE'] * units.AA
if 'OPT_COUNTS' in spec.keys():
for skey in speckeys:
try:
specobj.optimal[skey] = spec['OPT_{:s}'.format(skey)].data
except KeyError:
pass
# Add units on wave
specobj.optimal['WAVE'] = specobj.optimal['WAVE'] * units.AA
# Append
sobjs.add_sobj(specobj)
# Return
return sobjs, head0
def find_objects_pypeline(self, image, ivar, std=False, std_trace = None, maskslits=None,
manual_extract_dict=None,
show_peaks=False, show_fits=False, show_trace=False, show=False, debug=False):
"""
Find objects in the slits. This is currently setup only for ARMS
Wrapper to extract.objfind
Parameters
----------
tslits_dict: dict
Dictionary containing information on the slits traced for this image
Optional Parameters
-------------------
SHOW_PEAKS: bool
Generate QA showing peaks identified by object finding
SHOW_FITS: bool
Generate QA showing fits to traces
SHOW_TRACE: bool
Generate QA showing traces identified. Requires an open ginga RC modules window
Returns
-------
specobjs : Specobjs object
Container holding Specobj objects
nobj:
Number of objects identified
self.skymask : ndarray
Boolean image indicating which pixels are useful for global sky subtraction
"""
self.maskslits = self.maskslits if maskslits is None else maskslits
gdslits = np.where(np.invert(self.maskslits))[0]
# create the ouptut image for skymask
skymask = np.zeros_like(image, dtype=bool)
# Instantiate the specobjs container
sobjs = specobjs.SpecObjs()
# Loop on slits
for slit in gdslits:
qa_title ="Finding objects on slit # {:d}".format(slit)
msgs.info(qa_title)
thismask = (self.slitmask == slit)
inmask = (self.mask == 0) & thismask
# Find objects
specobj_dict = {'setup': self.setup, 'slitid': slit, 'orderindx': 999,
'det': self.det, 'objtype': self.objtype, 'pypeline': self.pypeline}
# TODO we need to add QA paths and QA hooks. QA should be
# done through objfind where all the relevant information
# is. This will be a png file(s) per slit.
sig_thresh = 30.0 if std else self.redux_par['sig_thresh']
#
sobjs_slit, skymask[thismask] = \
extract.objfind(image, thismask, self.tslits_dict['slit_left'][:,slit],self.tslits_dict['slit_righ'][:,slit],
inmask=inmask, ncoeff=self.redux_par['trace_npoly'],
std_trace=std_trace, sig_thresh=sig_thresh, hand_extract_dict=manual_extract_dict, #self.redux_par['manual'],
specobj_dict=specobj_dict, show_peaks=show_peaks,show_fits=show_fits, show_trace=show_trace,
qa_title=qa_title, nperslit=self.redux_par['maxnumber'])
sobjs.add_sobj(sobjs_slit)
# Steps
self.steps.append(inspect.stack()[0][3])
if show:
self.show('image', image=image*(self.mask == 0), chname = 'objfind',
sobjs=sobjs, slits=True)
# Return
return sobjs, len(sobjs), skymask
def ech_objfind(image,
ivar,
ordermask,
slit_left,
slit_righ,
inmask=None,
plate_scale=0.2,
npca=2,
ncoeff=5,
min_snr=0.0,
nabove_min_snr=0,
pca_percentile=20.0,
snr_pca=3.0,
box_radius=2.0,
show_peaks=False,
show_fits=False,
show_trace=False):
if inmask is None:
inmask = (ordermask > 0)
frameshape = image.shape
nspec = frameshape[0]
norders = slit_left.shape[1]
if isinstance(plate_scale, (float, int)):
plate_scale_ord = np.full(
norders, plate_scale) # 0.12 binned by 3 spatially for HIRES
elif isinstance(plate_scale, (np.ndarray, list, tuple)):
if len(plate_scale) == norders:
plate_scale_ord = plate_scale
elif len(plate_scale) == 1:
plate_scale_ord = np.full(norders, plate_scale[0])
else:
msgs.error(
'Invalid size for plate_scale. It must either have one element or norders elements'
)
else:
msgs.error('Invalid type for plate scale')
specmid = nspec // 2
slit_width = slit_righ - slit_left
spec_vec = np.arange(nspec)
slit_spec_pos = nspec / 2.0
slit_spat_pos = np.zeros((norders, 2))
for iord in range(norders):
slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec,
slit_left[:, iord]),
np.interp(slit_spec_pos, spec_vec,
slit_righ[:, iord]))
# Loop over orders and find objects
sobjs = specobjs.SpecObjs()
show_peaks = True
show_fits = True
# ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx!
for iord in range(norders):
msgs.info('Finding objects on slit # {:d}'.format(iord + 1))
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
specobj_dict = {
'setup': 'HIRES',
'slitid': iord + 1,
'scidx': 0,
'det': 1,
'objtype': 'science'
}
sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \
extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks,
show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0)
# ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord
for spec in sobjs_slit:
spec.ech_orderindx = iord
sobjs.add_sobj(sobjs_slit)
nfound = len(sobjs)
# Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0"
FOFSEP = 1.0 # separation of FOF algorithm in arcseconds
FOF_frac = FOFSEP / (np.median(slit_width) * np.median(plate_scale_ord))
# Feige: made the code also works for only one object found in one order
# Run the FOF. We use fake coordinaes
fracpos = sobjs.spat_fracpos
ra_fake = fracpos / 1000.0 # Divide all angles by 1000 to make geometry euclidian
dec_fake = 0.0 * fracpos
if nfound > 1:
(ingroup, multgroup, firstgroup,
nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac / 1000.0)
group = ingroup.copy()
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(uni_group)
msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects')
elif nfound == 1:
group = np.zeros(1, dtype='int')
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(group)
msgs.warn('Only find one object no FOF matching is needed')
gfrac = np.zeros(nfound)
for jj in range(nobj):
this_group = group == uni_group[jj]
gfrac[this_group] = np.median(fracpos[this_group])
uni_frac = gfrac[uni_ind]
sobjs_align = sobjs.copy()
# Now fill in the missing objects and their traces
for iobj in range(nobj):
for iord in range(norders):
# Is there an object on this order that grouped into the current group in question?
on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx
== iord)
if not np.any(on_slit):
# Add this to the sobjs_align, and assign required tags
thisobj = specobjs.SpecObj(frameshape,
slit_spat_pos[iord, :],
slit_spec_pos,
det=sobjs_align[0].det,
setup=sobjs_align[0].setup,
slitid=(iord + 1),
scidx=sobjs_align[0].scidx,
objtype=sobjs_align[0].objtype)
thisobj.ech_orderindx = iord
thisobj.spat_fracpos = uni_frac[iobj]
thisobj.trace_spat = slit_left[:,
iord] + slit_width[:, iord] * uni_frac[
iobj] # new trace
thisobj.trace_spec = spec_vec
thisobj.spat_pixpos = thisobj.trace_spat[specmid]
thisobj.set_idx()
# Use the real detections of this objects for the FWHM
this_group = group == uni_group[iobj]
# Assign to the fwhm of the nearest detected order
imin = np.argmin(
np.abs(sobjs_align[this_group].ech_orderindx - iord))
thisobj.fwhm = sobjs_align[imin].fwhm
thisobj.maskwidth = sobjs_align[imin].maskwidth
thisobj.ech_fracpos = uni_frac[iobj]
thisobj.ech_group = uni_group[iobj]
thisobj.ech_usepca = True
sobjs_align.add_sobj(thisobj)
group = np.append(group, uni_group[iobj])
gfrac = np.append(gfrac, uni_frac[iobj])
else:
# ToDo fix specobjs to get rid of these crappy loops!
for spec in sobjs_align[on_slit]:
spec.ech_fracpos = uni_frac[iobj]
spec.ech_group = uni_group[iobj]
spec.ech_usepca = False
# Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order
# respectively
sobjs_sort = specobjs.SpecObjs()
for iobj in range(nobj):
this_group = group == uni_group[iobj]
this_sobj = sobjs_align[this_group]
sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)])
# Loop over the objects and perform a quick and dirty extraction to assess S/N.
varimg = utils.calc_ivar(ivar)
flux_box = np.zeros((nspec, norders, nobj))
ivar_box = np.zeros((nspec, norders, nobj))
mask_box = np.zeros((nspec, norders, nobj))
SNR_arr = np.zeros((norders, nobj))
for iobj in range(nobj):
for iord in range(norders):
indx = (sobjs_sort.ech_group
== uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord)
spec = sobjs_sort[indx]
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
box_rad_pix = box_radius / plate_scale_ord[iord]
flux_tmp = extract.extract_boxcar(image * inmask_iord,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
var_tmp = extract.extract_boxcar(varimg * inmask_iord,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
ivar_tmp = utils.calc_ivar(var_tmp)
pixtot = extract.extract_boxcar(ivar * 0 + 1.0,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
mask_tmp = (extract.extract_boxcar(ivar * inmask_iord == 0.0,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec) != pixtot)
flux_box[:, iord, iobj] = flux_tmp * mask_tmp
ivar_box[:, iord, iobj] = np.fmax(ivar_tmp * mask_tmp, 0.0)
mask_box[:, iord, iobj] = mask_tmp
(mean, med_sn, stddev) = sigma_clipped_stats(
flux_box[mask_tmp, iord, iobj] *
np.sqrt(ivar_box[mask_tmp, iord, iobj]),
sigma_lower=5.0,
sigma_upper=5.0)
SNR_arr[iord, iobj] = med_sn
# Purge objects with low SNR and that don't show up in enough orders
keep_obj = np.zeros(nobj, dtype=bool)
sobjs_trim = specobjs.SpecObjs()
uni_group_trim = np.array([], dtype=int)
uni_frac_trim = np.array([], dtype=float)
for iobj in range(nobj):
if (np.sum(SNR_arr[:, iobj] > min_snr) >= nabove_min_snr):
keep_obj[iobj] = True
ikeep = sobjs_sort.ech_group == uni_group[iobj]
sobjs_trim.add_sobj(sobjs_sort[ikeep])
uni_group_trim = np.append(uni_group_trim, uni_group[iobj])
uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj])
else:
msgs.info(
'Purging object #{:d}'.format(iobj) +
' which does not satisfy min_snr > {:5.2f}'.format(min_snr) +
' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) +
' orders')
nobj_trim = np.sum(keep_obj)
if nobj_trim == 0:
return specobjs.SpecObjs()
SNR_arr_trim = SNR_arr[:, keep_obj]
# Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA
for iobj in range(nobj_trim):
SNR_now = SNR_arr_trim[:, iobj]
indx = (sobjs_trim.ech_group == uni_group_trim[iobj])
# PCA interp/extrap if:
# (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca)
# OR
# (if this order was not originally traced by the object finding, see above)
usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) &
(SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca
# ToDo fix specobjs to get rid of these crappy loops!
for iord, spec in enumerate(sobjs_trim[indx]):
spec.ech_usepca = usepca[iord]
if usepca[iord]:
msgs.info('Using PCA to predict trace for object #{:d}'.format(
iobj) + ' on order #{:d}'.format(iord))
sobjs_final = sobjs_trim.copy()
# Loop over the objects one by one and adjust/predict the traces
npoly_cen = 3
pca_fits = np.zeros((nspec, norders, nobj_trim))
for iobj in range(nobj_trim):
igroup = sobjs_final.ech_group == uni_group_trim[iobj]
# PCA predict the masked orders which were not traced
pca_fits[:, :, iobj] = pca_trace((sobjs_final[igroup].trace_spat).T,
usepca=None,
npca=npca,
npoly_cen=npoly_cen)
# usepca = sobjs_final[igroup].ech_usepca,
# Perform iterative flux weighted centroiding using new PCA predictions
xinit_fweight = pca_fits[:, :, iobj].copy()
inmask_now = inmask & (ordermask > 0)
xfit_fweight = extract.iter_tracefit(image,
xinit_fweight,
ncoeff,
inmask=inmask_now,
show_fits=show_fits)
# Perform iterative Gaussian weighted centroiding
xinit_gweight = xfit_fweight.copy()
xfit_gweight = extract.iter_tracefit(image,
xinit_gweight,
ncoeff,
inmask=inmask_now,
gweight=True,
show_fits=show_fits)
# Assign the new traces
for iord, spec in enumerate(sobjs_final[igroup]):
spec.trace_spat = xfit_gweight[:, iord]
spec.spat_pixpos = spec.trace_spat[specmid]
# Set the IDs
sobjs_final.set_idx()
if show_trace:
viewer, ch = ginga.show_image(objminsky * (ordermask > 0))
for iobj in range(nobj_trim):
for iord in range(norders):
ginga.show_trace(viewer,
ch,
pca_fits[:, iord, iobj],
str(uni_frac[iobj]),
color='yellow')
for spec in sobjs_trim:
color = 'green' if spec.ech_usepca else 'magenta'
ginga.show_trace(viewer,
ch,
spec.trace_spat,
spec.idx,
color=color)
#for spec in sobjs_final:
# color = 'red' if spec.ech_usepca else 'green'
# ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)
return sobjs_final
def main(args):
""" Executes 2d coadding
"""
msgs.warn('PATH =' + os.getcwd())
# Load the file
if args.file is not None:
spectrograph_name, config_lines, spec2d_files = io.read_spec2d_file(
args.file, filetype="coadd2d")
spectrograph = load_spectrograph(spectrograph_name)
# Parameters
# TODO: Shouldn't this reinstantiate the same parameters used in
# the PypeIt run that extracted the objects? Why are we not
# just passing the pypeit file?
# JFH: The reason is that the coadd2dfile may want different reduction parameters
spectrograph_def_par = spectrograph.default_pypeit_par()
parset = par.PypeItPar.from_cfg_lines(
cfg_lines=spectrograph_def_par.to_config(),
merge_with=config_lines)
elif args.obj is not None:
# TODO: We should probably be reading the pypeit file and using those parameters here rather than using the
# default parset.
# TODO: This needs to define the science path
spec2d_files = glob.glob('./Science/spec2d_*' + args.obj + '*')
head0 = fits.getheader(spec2d_files[0])
spectrograph_name = head0['PYP_SPEC']
spectrograph = load_spectrograph(spectrograph_name)
parset = spectrograph.default_pypeit_par()
else:
msgs.error(
'You must either input a coadd2d file with --file or an object name with --obj'
)
# Update with configuration specific parameters (which requires science file) and initialize spectrograph
spectrograph_cfg_lines = spectrograph.config_specific_par(
spec2d_files[0]).to_config()
parset = par.PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines,
merge_with=parset.to_config())
# If detector was passed as an argument override whatever was in the coadd2d_file
if args.det is not None:
msgs.info("Restricting reductions to detector={}".format(args.det))
parset['rdx']['detnum'] = int(args.det)
# Get headers (if possible) and base names
spec1d_files = [
files.replace('spec2d', 'spec1d') for files in spec2d_files
]
head1d = None
for spec1d_file in spec1d_files:
if os.path.isfile(spec1d_file):
head1d = fits.getheader(spec1d_file)
break
if head1d is None:
msgs.warn("No 1D spectra so am generating a dummy header for output")
head1d = io.initialize_header()
head2d = fits.getheader(spec2d_files[0])
if args.basename is None:
filename = os.path.basename(spec2d_files[0])
basename = filename.split('_')[2]
else:
basename = args.basename
# Write the par to disk
par_outfile = basename + '_coadd2d.par'
print("Writing the parameters to {}".format(par_outfile))
parset.to_config(par_outfile)
# Now run the coadds
skysub_mode = head2d['SKYSUB']
ir_redux = True if 'DIFF' in skysub_mode else False
# Print status message
msgs_string = 'Reducing target {:s}'.format(basename) + msgs.newline()
msgs_string += 'Performing coadd of frames reduce with {:s} imaging'.format(
skysub_mode)
msgs_string += msgs.newline(
) + 'Combining frames in 2d coadd:' + msgs.newline()
for file in spec2d_files:
msgs_string += '{0:s}'.format(os.path.basename(file)) + msgs.newline()
msgs.info(msgs_string)
# TODO: This needs to be added to the parameter list for rdx
redux_path = os.getcwd()
master_dirname = os.path.basename(head2d['PYPMFDIR']) + '_coadd'
master_dir = os.path.join(redux_path, master_dirname)
# Make the new Master dir
if not os.path.isdir(master_dir):
msgs.info(
'Creating directory for Master output: {0}'.format(master_dir))
os.makedirs(master_dir)
# Instantiate the sci_dict
sci_dict = OrderedDict() # This needs to be ordered
sci_dict['meta'] = {}
sci_dict['meta']['vel_corr'] = 0.
sci_dict['meta']['ir_redux'] = ir_redux
# Find the detectors to reduce
detectors = PypeIt.select_detectors(detnum=parset['rdx']['detnum'],
ndet=spectrograph.ndet)
if len(detectors) != spectrograph.ndet:
msgs.warn('Not reducing detectors: {0}'.format(' '.join([
str(d)
for d in set(np.arange(spectrograph.ndet) + 1) - set(detectors)
])))
# Loop on detectors
for det in detectors:
msgs.info("Working on detector {0}".format(det))
sci_dict[det] = {}
# Instantiate Coadd2d
coadd = coadd2d.CoAdd2D.get_instance(
spec2d_files,
spectrograph,
parset,
det=det,
offsets=parset['coadd2d']['offsets'],
weights=parset['coadd2d']['weights'],
ir_redux=ir_redux,
debug_offsets=args.debug_offsets,
debug=args.debug,
samp_fact=args.samp_fact)
# Coadd the slits
coadd_dict_list = coadd.coadd(
only_slits=None) # TODO implement only_slits later
# Create the pseudo images
pseudo_dict = coadd.create_pseudo_image(coadd_dict_list)
# Reduce
msgs.info('Running the extraction')
# TODO -- This should mirror what is in pypeit.extract_one
# TODO -- JFH :: This ought to return a Spec2DObj and SpecObjs which would be slurped into
# AllSpec2DObj and all_specobsj, as below.
# TODO -- JFH -- Check that the slits we are using are correct
sci_dict[det]['sciimg'], sci_dict[det]['sciivar'], sci_dict[det]['skymodel'], sci_dict[det]['objmodel'], \
sci_dict[det]['ivarmodel'], sci_dict[det]['outmask'], sci_dict[det]['specobjs'], sci_dict[det]['detector'], \
sci_dict[det]['slits'], sci_dict[det]['tilts'], sci_dict[det]['waveimg'] = coadd.reduce(
pseudo_dict, show = args.show, show_peaks = args.peaks)
# Save pseudo image master files
#coadd.save_masters()
# Make the new Science dir
# TODO: This needs to be defined by the user
scipath = os.path.join(redux_path, 'Science_coadd')
if not os.path.isdir(scipath):
msgs.info('Creating directory for Science output: {0}'.format(scipath))
os.makedirs(scipath)
# THE FOLLOWING MIMICS THE CODE IN pypeit.save_exposure()
# TODO -- These lines should be above once reduce() passes back something sensible
all_specobjs = specobjs.SpecObjs()
for det in detectors:
all_specobjs.add_sobj(sci_dict[det]['specobjs'])
# Write
outfile1d = os.path.join(scipath, 'spec1d_{:s}.fits'.format(basename))
subheader = spectrograph.subheader_for_spec(head2d, head2d)
all_specobjs.write_to_fits(subheader, outfile1d)
# 2D spectra
# TODO -- These lines should be above once reduce() passes back something sensible
all_spec2d = spec2dobj.AllSpec2DObj()
all_spec2d['meta']['ir_redux'] = ir_redux
for det in detectors:
all_spec2d[det] = spec2dobj.Spec2DObj(
det=det,
sciimg=sci_dict[det]['sciimg'],
ivarraw=sci_dict[det]['sciivar'],
skymodel=sci_dict[det]['skymodel'],
objmodel=sci_dict[det]['objmodel'],
ivarmodel=sci_dict[det]['ivarmodel'],
scaleimg=np.array([1.0], dtype=np.float),
bpmmask=sci_dict[det]['outmask'],
detector=sci_dict[det]['detector'],
slits=sci_dict[det]['slits'],
waveimg=sci_dict[det]['waveimg'],
tilts=sci_dict[det]['tilts'],
sci_spat_flexure=None,
sci_spec_flexure=None,
vel_corr=None,
vel_type=None)
# Build header
outfile2d = os.path.join(scipath, 'spec2d_{:s}.fits'.format(basename))
pri_hdr = all_spec2d.build_primary_hdr(
head2d,
spectrograph,
subheader=subheader,
# TODO -- JFH :: Decide if we need any of these
redux_path=None,
master_key_dict=None,
master_dir=None)
# Write
all_spec2d.write_to_fits(outfile2d, pri_hdr=pri_hdr)
def save_all(sci_dict,
master_key_dict,
master_dir,
spectrograph,
head1d,
head2d,
scipath,
basename,
update_det=None,
binning='None'):
"""
Routine to save PypeIt 1d and 2d outputs
Args:
sci_dict: dict
Dictionary containing extraction outputs
master_key_dict: dict
Dictionary with master key information for this reduction
master_dir: str
Directory where the master files live
spectrograph: object, spectrograph
Spectrograph object for the spectorgraph that was used
head1d: dict
fitstbl meta data dictionary that will become the header for the spec1d files
head2d: dict
rawfile header that will become the header for the spec2d files
scipath: str
path to which the outputs should be written
basename: str
the object basename
refframe: str, default = 'heliocentric'
Reference frame for the wavelengths
update_det : int or list, default=None
If provided, do not clobber the existing file but only update
the indicated detectors. Useful for re-running on a subset of detectors
Returns:
"""
# Check for the directory
if not os.path.isdir(scipath):
os.makedirs(scipath)
# Filenames to write out
# TODO: These should be centrally defined so that they don't become
# out of sync with what's in pypeit.PypeIt
outfile1d = os.path.join(scipath, 'spec1d_{:s}.fits'.format(basename))
outfile2d = os.path.join(scipath, 'spec2d_{:s}.fits'.format(basename))
outfiletxt = os.path.join(scipath, 'spec1d_{:s}.txt'.format(basename))
# TODO: Need some checks here that the exposure has been reduced
# Build the final list of specobjs and vel_corr
all_specobjs = specobjs.SpecObjs()
for key in sci_dict:
try:
all_specobjs.add_sobj(sci_dict[key]['specobjs'])
except KeyError: # No object extracted
continue
if len(all_specobjs) == 0:
msgs.warn('No objects to save. Only writing spec2d files!')
else:
# Build the spec1d output header.
header = all_specobjs.build_header(head1d, head2d, spectrograph)
all_specobjs.write_to_fits(header, outfile1d, update_det=update_det)
# Txt file
# TODO JFH: Make this a method in the specobjs class.
save_obj_info(all_specobjs, spectrograph, outfiletxt, binning=binning)
# Write 2D images for the Science Frame
save_2d_images(sci_dict,
head2d,
spectrograph,
master_key_dict,
master_dir,
outfile2d,
update_det=update_det)
return
def main(args):
""" Executes 2d coadding
"""
msgs.info('PATH =' + os.getcwd())
# Load the file
if args.file is not None:
spectrograph_name, config_lines, spec2d_files \
= io.read_spec2d_file(args.file, filetype="coadd2d")
spectrograph = load_spectrograph(spectrograph_name)
# Parameters
# TODO: Shouldn't this reinstantiate the same parameters used in
# the PypeIt run that extracted the objects? Why are we not
# just passing the pypeit file?
# JFH: The reason is that the coadd2dfile may want different reduction parameters
# DP: I think config_specific_par() is more appropriate here. default_pypeit_par()
# is included in config_specific_par()
# NOTE `config_specific_par` works with the spec2d files because we construct the header
# of those files to include all the relevant keywords from the raw file.
spectrograph_cfg_lines = spectrograph.config_specific_par(
spec2d_files[0]).to_config()
parset = par.PypeItPar.from_cfg_lines(
cfg_lines=spectrograph_cfg_lines, merge_with=config_lines)
elif args.obj is not None:
# TODO: We should probably be reading the pypeit file and using
# those parameters here rather than using the default parset.
# TODO: This needs to define the science path
spec2d_files = glob.glob('./Science/spec2d_*' + args.obj + '*')
head0 = fits.getheader(spec2d_files[0])
spectrograph_name = head0['PYP_SPEC']
spectrograph = load_spectrograph(spectrograph_name)
# NOTE `config_specific_par` works with the spec2d files because we construct the header
# of those files to include all the relevant keywords from the raw file.
spectrograph_cfg_lines = spectrograph.config_specific_par(
spec2d_files[0]).to_config()
parset = par.PypeItPar.from_cfg_lines(
cfg_lines=spectrograph_cfg_lines)
else:
return msgs.error(
'You must provide either a coadd2d file (--file) or an object name (--obj)'
)
# If detector was passed as an argument override whatever was in the coadd2d_file
if args.det is not None:
msgs.info("Restricting reductions to detector={}".format(args.det))
# parset['rdx']['detnum'] = par.util.eval_tuple(args.det.split(','))
# TODO this needs to be adjusted if we want to pass (as inline command) mosaic detectors
parset['rdx']['detnum'] = [int(d) for d in args.det.split(',')]
# Get headers (if possible) and base names
spec1d_files = [
files.replace('spec2d', 'spec1d') for files in spec2d_files
]
head1d = None
for spec1d_file in spec1d_files:
if os.path.isfile(spec1d_file):
head1d = fits.getheader(spec1d_file)
break
if head1d is None:
msgs.warn(
"No 1D spectra so am generating a dummy header for output")
head1d = io.initialize_header()
head2d = fits.getheader(spec2d_files[0])
if args.basename is None:
#TODO Fix this, currently does not work if target names have - or _
filename_first = os.path.basename(spec2d_files[0])
filename_last = os.path.basename(spec2d_files[-1])
prefix_first = (filename_first.split('_')[1]).split('-')[0]
prefix_last = (filename_last.split('_')[1]).split('-')[0]
objname = (filename_first.split('-')[1]).split('_')[0]
basename = '{:s}-{:s}-{:s}'.format(prefix_first, prefix_last,
objname)
else:
basename = args.basename
# TODO Heliocentric for coadd2d needs to be thought through. Currently turning it off.
parset['calibrations']['wavelengths']['refframe'] = 'observed'
# TODO Flexure correction for coadd2d needs to be thought through. Currently turning it off.
parset['flexure']['spec_method'] = 'skip'
# Write the par to disk
par_outfile = basename + '_coadd2d.par'
print("Writing the parameters to {}".format(par_outfile))
parset.to_config(par_outfile,
exclude_defaults=True,
include_descr=False)
# Now run the coadds
skysub_mode = head2d['SKYSUB']
findobj_mode = head2d['FINDOBJ']
bkg_redux = True if 'DIFF' in skysub_mode else False
find_negative = True if 'NEG' in findobj_mode else False
# Print status message
msgs_string = 'Reducing target {:s}'.format(basename) + msgs.newline()
msgs_string += 'Coadding frame sky-subtraced with {:s}'.format(
skysub_mode)
msgs_string += 'Searching for objects that are {:s}'.format(
findobj_mode)
msgs_string += msgs.newline(
) + 'Combining frames in 2d coadd:' + msgs.newline()
for f, file in enumerate(spec2d_files):
msgs_string += 'Exp {0}: {1:s}'.format(
f, os.path.basename(file)) + msgs.newline()
msgs.info(msgs_string)
# TODO: This needs to be added to the parameter list for rdx
redux_path = os.getcwd()
master_dirname = os.path.basename(head2d['PYPMFDIR']) + '_coadd'
master_dir = os.path.join(redux_path, master_dirname)
# Make the new Master dir
if not os.path.isdir(master_dir):
msgs.info(
'Creating directory for Master output: {0}'.format(master_dir))
os.makedirs(master_dir)
# Instantiate the sci_dict
sci_dict = OrderedDict() # This needs to be ordered
sci_dict['meta'] = {}
sci_dict['meta']['vel_corr'] = 0.
sci_dict['meta']['bkg_redux'] = bkg_redux
sci_dict['meta']['find_negative'] = find_negative
# Make QA coadd directory
parset['rdx']['qadir'] += '_coadd'
qa_path = os.path.join(parset['rdx']['redux_path'],
parset['rdx']['qadir'], 'PNGs')
if not os.path.isdir(qa_path):
os.makedirs(qa_path)
# Find the detectors to reduce
# detectors = PypeIt.select_detectors(detnum=parset['rdx']['detnum'], ndet=spectrograph.ndet)
detectors = spectrograph.select_detectors(
subset=parset['rdx']['detnum'])
# if len(detectors) != spectrograph.ndet:
# msgs.warn('Not reducing detectors: {0}'.format(' '.join([str(d) for d in
# set(np.arange(spectrograph.ndet) + 1) - set(detectors)])))
msgs.info(f'Detectors to work on: {detectors}')
# Only_slits?
if args.only_slits:
parset['coadd2d']['only_slits'] = [
int(item) for item in args.only_slits.split(',')
]
# container for specobjs
all_specobjs = specobjs.SpecObjs()
# container for spec2dobj
all_spec2d = spec2dobj.AllSpec2DObj()
# set some meta
all_spec2d['meta']['bkg_redux'] = bkg_redux
all_spec2d['meta']['find_negative'] = find_negative
# Loop on detectors
for det in detectors:
msgs.info("Working on detector {0}".format(det))
# Instantiate Coadd2d
coadd = coadd2d.CoAdd2D.get_instance(
spec2d_files,
spectrograph,
parset,
det=det,
offsets=parset['coadd2d']['offsets'],
weights=parset['coadd2d']['weights'],
spec_samp_fact=args.spec_samp_fact,
spat_samp_fact=args.spat_samp_fact,
bkg_redux=bkg_redux,
find_negative=find_negative,
debug_offsets=args.debug_offsets,
debug=args.debug)
# TODO Add this stuff to a run method in coadd2d
# Coadd the slits
coadd_dict_list = coadd.coadd(
only_slits=parset['coadd2d']['only_slits'])
# Create the pseudo images
pseudo_dict = coadd.create_pseudo_image(coadd_dict_list)
# Reduce
msgs.info('Running the extraction')
# TODO -- This should mirror what is in pypeit.extract_one
# TODO -- JFH :: This ought to return a Spec2DObj and SpecObjs which
# would be slurped into AllSpec2DObj and all_specobsj, as below.
# TODO -- JFH -- Check that the slits we are using are correct
sci_dict[coadd.detname] = {}
sci_dict[coadd.detname]['sciimg'], sci_dict[coadd.detname]['sciivar'], \
sci_dict[coadd.detname]['skymodel'], sci_dict[coadd.detname]['objmodel'], \
sci_dict[coadd.detname]['ivarmodel'], sci_dict[coadd.detname]['outmask'], \
sci_dict[coadd.detname]['specobjs'], sci_dict[coadd.detname]['detector'], \
sci_dict[coadd.detname]['slits'], sci_dict[coadd.detname]['tilts'], \
sci_dict[coadd.detname]['waveimg'] \
= coadd.reduce(pseudo_dict, show=args.show, show_peaks=args.peaks, basename=basename)
# Tack on detector (similarly to pypeit.extract_one)
for sobj in sci_dict[coadd.detname]['specobjs']:
sobj.DETECTOR = sci_dict[coadd.detname]['detector']
# fill the specobjs container
all_specobjs.add_sobj(sci_dict[coadd.detname]['specobjs'])
# fill the spec2dobj container but first ...
# pull out maskdef_designtab from sci_dict[det]['slits']
maskdef_designtab = sci_dict[
coadd.detname]['slits'].maskdef_designtab
slits = copy.deepcopy(sci_dict[coadd.detname]['slits'])
slits.maskdef_designtab = None
# fill up
all_spec2d[coadd.detname] = spec2dobj.Spec2DObj(
sciimg=sci_dict[coadd.detname]['sciimg'],
ivarraw=sci_dict[coadd.detname]['sciivar'],
skymodel=sci_dict[coadd.detname]['skymodel'],
objmodel=sci_dict[coadd.detname]['objmodel'],
ivarmodel=sci_dict[coadd.detname]['ivarmodel'],
scaleimg=np.array([1.0], dtype=np.float),
bpmmask=sci_dict[coadd.detname]['outmask'],
detector=sci_dict[coadd.detname]['detector'],
slits=slits,
waveimg=sci_dict[coadd.detname]['waveimg'],
tilts=sci_dict[coadd.detname]['tilts'],
sci_spat_flexure=None,
sci_spec_flexure=None,
vel_corr=None,
vel_type=None,
maskdef_designtab=maskdef_designtab)
# Save pseudo image master files
#coadd.save_masters()
# SAVE TO DISK
# Make the new Science dir
# TODO: This needs to be defined by the user
scipath = os.path.join(redux_path, 'Science_coadd')
if not os.path.isdir(scipath):
msgs.info(
'Creating directory for Science output: {0}'.format(scipath))
os.makedirs(scipath)
# THE FOLLOWING MIMICS THE CODE IN pypeit.save_exposure()
subheader = spectrograph.subheader_for_spec(head2d, head2d)
# Write spec1D
if all_specobjs.nobj > 0:
outfile1d = os.path.join(scipath,
'spec1d_{:s}.fits'.format(basename))
all_specobjs.write_to_fits(subheader, outfile1d)
# Info
outfiletxt = os.path.join(scipath,
'spec1d_{:s}.txt'.format(basename))
sobjs = specobjs.SpecObjs.from_fitsfile(outfile1d,
chk_version=False)
sobjs.write_info(outfiletxt, spectrograph.pypeline)
# Build header for spec2d
outfile2d = os.path.join(scipath, 'spec2d_{:s}.fits'.format(basename))
pri_hdr = all_spec2d.build_primary_hdr(
head2d,
spectrograph,
subheader=subheader,
# TODO -- JFH :: Decide if we need any of these
redux_path=None,
master_key_dict=None,
master_dir=None)
# Write spec2d
all_spec2d.write_to_fits(outfile2d, pri_hdr=pri_hdr)
(crmask_A == False) & (crmask_B == False)
ivar_AB = utils.calc_ivar(var_AB) * (mask_AB == True)
# Sky subtraction
from pypeit.core import skysub
from pypeit.core import extract
slitpix = tslits_dict['slitpix']
lcen = tslits_dict['lcen']
rcen = tslits_dict['rcen']
ximg = tslits_dict['ximg']
edgmask = tslits_dict['edge_mask']
FWHM = 5.0
bsp = 0.8
residual_img = np.zeros_like(sciframe_A)
specobjs_pos = specobjs.SpecObjs()
specobjs_neg = specobjs.SpecObjs()
skymask = (slitpix > 0)
for islit in range(1, nslits + 1):
thismask = (slitpix == islit)
residual_img[thismask] = skysub.global_skysub(diff_AB,
ivar_AB,
mstilts,
thismask,
lcen[:,islit-1],
rcen[:,islit-1],
inmask=((edgmask == False) & (mask_AB == True)),
bsp=bsp,
pos_mask=False,
show_fit=False)
image = diff_AB - residual_img