def test_assign_maskinfo():
# Spectrograph
keck_deimos = KeckDEIMOSSpectrograph()
par = keck_deimos.default_pypeit_par()
# working only on detector 3
det = 3
# Built trace image
traceImage = buildimage.buildimage_fromlist(
keck_deimos, det, par['calibrations']['traceframe'],
deimos_flat_files())
msbpm = keck_deimos.bpm(traceImage.files[0], det)
# load specific config parameters
par = keck_deimos.config_specific_par(traceImage.files[0])
trace_par = par['calibrations']['slitedges']
# Run edge trace
edges = edgetrace.EdgeTraceSet(traceImage,
keck_deimos,
trace_par,
bpm=msbpm,
auto=True,
debug=False,
show_stages=False,
qa_path=None)
slits = edges.get_slits()
# Test that the maskfile is saved properly
hdul = fits.open(slits.maskfile)
det_par = keck_deimos.get_detector_par(hdul, det=det)
specobjs_file = os.path.join(
os.getenv('PYPEIT_DEV'), 'Cooked', 'Science',
'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_2010Sep13T061231.334.fits')
# specobjs_file = os.path.join(os.getenv('PYPEIT_DEV'), 'REDUX_OUT', 'keck_deimos',
# '830G_M_8500', 'Science',
# 'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_2010Sep13T061231.334.fits')
sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file)
# Init at null
for sobj in sobjs:
sobj.MASKDEF_OBJNAME = None
sobj.RA = None
sobj.DEC = None
# Run me
slits.assign_maskinfo(sobjs, det_par['platescale'])
# Test
assert sobjs[sobjs.SLITID ==
496].MASKDEF_OBJNAME == 'ero89', 'Wrong MASKDEF_OBJNAME'
assert sobjs[sobjs.SLITID == 496].RA == 352.27471667, 'Wrong object RA'
assert sobjs[sobjs.SLITID == 496].DEC == -3.09223056, 'Wrong object DEC'
# Write sobjs
sobjs.write_to_fits({}, data_path('tst_sobjs.fits'))
os.remove(data_path('tst_sobjs.fits'))
def get_slits(self):
"""
Load or generate the definition of the slit boundaries.
Internals that must be available are :attr:`fitstbl`,
:attr:`calib_ID`, :attr:`det`.
Returns:
:class:`pypeit.slittrace.SlitTraceSet`: Traces of the
slit edges; also kept internally as :attr:`slits`.
"""
# Check for existing data
if not self._chk_objs(['msbpm']):
return
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
trace_image_files, self.master_key_dict[
'trace'] = self._prep_calibrations('trace')
# Reuse master frame?
slit_masterframe_name = masterframe.construct_file_name(
slittrace.SlitTraceSet,
self.master_key_dict['trace'],
master_dir=self.master_dir)
if os.path.isfile(slit_masterframe_name) and self.reuse_masters:
self.slits = slittrace.SlitTraceSet.from_file(
slit_masterframe_name)
# Reset the bitmask
self.slits.mask = self.slits.mask_init.copy()
else:
# Slits don't exist or we're not resusing them
edge_masterframe_name = masterframe.construct_file_name(
edgetrace.EdgeTraceSet,
self.master_key_dict['trace'],
master_dir=self.master_dir)
# Reuse master frame?
if os.path.isfile(edge_masterframe_name) and self.reuse_masters:
self.edges = edgetrace.EdgeTraceSet.from_file(
edge_masterframe_name)
elif len(trace_image_files) == 0:
msgs.warn("No frametype=trace files to build slits")
return None
else:
# Build the trace image
self.traceImage = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['traceframe'],
trace_image_files,
bias=self.msbias,
bpm=self.msbpm,
dark=self.msdark)
self.edges = edgetrace.EdgeTraceSet(self.traceImage,
self.spectrograph,
self.par['slitedges'],
bpm=self.msbpm,
auto=True)
self.edges.save(edge_masterframe_name,
master_dir=self.master_dir,
master_key=self.master_key_dict['trace'])
# Show the result if requested
if self.show:
self.edges.show(in_ginga=True)
# Get the slits from the result of the edge tracing, delete
# the edges object, and save the slits, if requested
self.slits = self.edges.get_slits()
self.edges = None
self.slits.to_master_file(slit_masterframe_name)
# User mask?
if self.slitspat_num is not None:
self.slits.user_mask(self.det, self.slitspat_num)
return self.slits
def main(args):
import time
import os
import numpy as np
from pypeit.spectrographs.util import load_spectrograph
from pypeit import traceimage, edgetrace, biasframe
from pypeit.pypeit import PypeIt
from pypeit.core import parse
from IPython import embed
if args.pypeit_file is not None:
pypeit_file = args.pypeit_file
if not os.path.isfile(pypeit_file):
raise FileNotFoundError(
'File does not exist: {0}'.format(pypeit_file))
pypeit_file = os.path.abspath(pypeit_file)
redux_path = os.path.abspath(
os.path.split(pypeit_file)[0] if args.redux_path is None else args.
redux_path)
rdx = PypeIt(pypeit_file, redux_path=redux_path)
# Save the spectrograph
spec = rdx.spectrograph
# Get the calibration group to use
group = np.unique(
rdx.fitstbl['calib'])[0] if args.group is None else args.group
if group not in np.unique(rdx.fitstbl['calib']):
raise ValueError(
'Not a valid calibration group: {0}'.format(group))
# Find the rows in the metadata table with trace frames in the
# specified calibration group
tbl_rows = rdx.fitstbl.find_frames('trace',
calib_ID=int(group),
index=True)
# Master keyword
master_key_base = '_'.join(
rdx.fitstbl.master_key(tbl_rows[0]).split('_')[:2])
# Save the binning
binning = rdx.fitstbl['binning'][tbl_rows[0]]
# Save the full file paths
files = rdx.fitstbl.frame_paths(tbl_rows)
# Trace image processing parameters
proc_par = rdx.caliBrate.par['traceframe']
# Slit tracing parameters
trace_par = rdx.caliBrate.par['slitedges']
# Get the bias files, if requested
bias_rows = rdx.fitstbl.find_frames('bias',
calib_ID=int(group),
index=True)
bias_files = rdx.fitstbl.frame_paths(bias_rows)
bias_par = rdx.caliBrate.par['biasframe']
# Set the QA path
qa_path = rdx.qa_path
else:
spec = load_spectrograph(args.spectrograph)
master_key_base = 'A_1'
binning = '1,1' if args.binning is None else args.binning
if not os.path.isfile(args.trace_file):
raise FileNotFoundError('File does not exist: {0}'.format(
args.trace_file))
files = [os.path.abspath(args.trace_file)]
redux_path = os.path.abspath(
os.path.split(files[0])[0] if args.redux_path is None else args.
redux_path)
par = spec.default_pypeit_par()
proc_par = par['calibrations']['traceframe']
trace_par = par['calibrations']['slitedges']
bias_files = None
bias_par = None
# Set the QA path
qa_path = os.path.join(os.path.abspath(os.path.split(files[0])[0]),
'QA')
detectors = np.arange(spec.ndet) + 1 if args.detector is None else [
args.detector
]
master_dir = os.path.join(redux_path, args.master_dir)
for det in detectors:
# Master keyword for output file name
master_key = '{0}_{1}'.format(master_key_base, str(det).zfill(2))
# Get the bias frame if requested
if bias_files is None:
proc_par['process']['bias'] = 'skip'
msbias = None
else:
biasFrame = biasframe.BiasFrame(spec,
files=bias_files,
det=det,
par=bias_par,
master_key=master_key,
master_dir=master_dir)
msbias = biasFrame.build_image()
msbpm = spec.bpm(files[0], det)
# Build the trace image
traceImage = traceimage.TraceImage(spec,
files=files,
det=det,
par=proc_par,
bias=msbias)
traceImage.build_image(bias=msbias, bpm=msbpm)
# Trace the slit edges
t = time.perf_counter()
edges = edgetrace.EdgeTraceSet(spec,
trace_par,
master_key=master_key,
master_dir=master_dir,
img=traceImage,
det=det,
bpm=msbpm,
auto=True,
debug=args.debug,
show_stages=args.show,
qa_path=qa_path)
print('Tracing for detector {0} finished in {1} s.'.format(
det,
time.perf_counter() - t))
edges.save()
return 0
def test_assign_maskinfo_add_missing():
instr_names = ['keck_deimos', 'keck_mosfire']
for name in instr_names:
# Spectrograph
instrument = load_spectrograph(name)
par = instrument.default_pypeit_par()
# working only on detector 3 (det=3 for DEIMOS. For MOSFIRE does not matter because we have only one det)
det = 3 if name == 'keck_deimos' else 1
# Built trace image
traceImage = buildimage.buildimage_fromlist(
instrument, det, par['calibrations']['traceframe'],
flat_files(instr=name))
# load specific config parameters
par = instrument.config_specific_par(traceImage.files[0])
# Run edge trace
edges = edgetrace.EdgeTraceSet(traceImage,
instrument,
par['calibrations']['slitedges'],
auto=True,
debug=False,
show_stages=False,
qa_path=None)
slits = edges.get_slits()
# Test that the maskfile is saved properly
hdul = fits.open(slits.maskfile)
det_par = instrument.get_detector_par(det, hdu=hdul)
if name == 'keck_deimos':
specobjs_file = os.path.join(
os.getenv('PYPEIT_DEV'), 'Cooked', 'Science',
'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_20100913T061231.334.fits'
)
sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file)
# correct value
slitid = sobjs[sobjs.MASKDEF_OBJNAME == 'ero89'].SLITID[0]
true_maskdef_objname = sobjs[sobjs.SLITID ==
slitid].MASKDEF_OBJNAME[0]
true_ra = round(sobjs[sobjs.SLITID == slitid].RA[0], 6)
true_dec = round(sobjs[sobjs.SLITID == slitid].DEC[0], 6)
true_spat_pixpos = round(
sobjs[sobjs.MASKDEF_OBJNAME == 'ero884'].SPAT_PIXPOS[0])
true_spat_pixpos_2 = round(
sobjs[sobjs.MASKDEF_OBJNAME == 'ero191'].SPAT_PIXPOS[0])
elif name == 'keck_mosfire':
specobjs_file = os.path.join(
os.getenv('PYPEIT_DEV'), 'Cooked', 'Science',
'spec1d_m191014_0170-2M2228_12_MOSFIRE_20191014T095212.598.fits'
)
sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file)
# correct value
slitid = sobjs[sobjs.MASKDEF_OBJNAME == '18'].SLITID[0]
true_maskdef_objname = sobjs[sobjs.SLITID ==
slitid].MASKDEF_OBJNAME[0]
true_ra = round(sobjs[sobjs.SLITID == slitid].RA[0], 6)
true_dec = round(sobjs[sobjs.SLITID == slitid].DEC[0], 6)
true_spat_pixpos = round(
sobjs[sobjs.MASKDEF_OBJNAME == '7'].SPAT_PIXPOS[0])
# Init at null and remove the force extraction
idx_remove = []
for i, sobj in enumerate(sobjs):
if sobj.MASKDEF_EXTRACT:
idx_remove.append(i)
else:
sobj.MASKDEF_ID = None
sobj.MASKDEF_OBJNAME = None
sobj.RA = None
sobj.DEC = None
sobj.MASKDEF_EXTRACT = None
sobjs.remove_sobj(idx_remove)
# get the dither offset if available
if name == 'keck_deimos':
dither_off = None
elif name == 'keck_mosfire':
dither_off = instrument.parse_dither_pattern([
os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA',
'keck_mosfire', 'J_multi', 'm191014_0170.fits')
])[2][0]
# get object positions from slitmask design and slitmask offsets
calib_slits = slittrace.get_maskdef_objpos_offset_alldets(
sobjs, [slits], [None], [det_par['platescale']],
par['calibrations']['slitedges']['det_buffer'],
par['reduce']['slitmask'],
dither_off=dither_off)
# determine if slitmask offsets exist and compute an average offsets over all the detectors
calib_slits = slittrace.average_maskdef_offset(
calib_slits, det_par['platescale'], instrument.list_detectors())
# slitmask design matching and add undetected objects
sobjs = slittrace.assign_addobjs_alldets(
sobjs, calib_slits, [None], [det_par['platescale']],
par['reduce']['slitmask'], par['reduce']['findobj']['find_fwhm'])
# Test
if name == 'keck_deimos':
# Check if recover the maskdef assignment
assert sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[
0] == true_maskdef_objname, 'Wrong DEIMOS MASKDEF_OBJNAME'
assert round(sobjs[sobjs.SLITID == slitid].RA[0],
6) == true_ra, 'Wrong object DEIMOS RA'
assert round(sobjs[sobjs.SLITID == slitid].DEC[0],
6) == true_dec, 'Wrong object DEIMOS DEC'
# Test that undetected objects are found at the correct location (the correct location is
# verified by visual inspection)
assert round(sobjs[sobjs.MASKDEF_OBJNAME == 'ero884'].SPAT_PIXPOS[0]) == true_spat_pixpos, \
'Wrong object (ero884) location on the DEIMOS slit'
assert round(sobjs[sobjs.MASKDEF_OBJNAME == 'ero191'].SPAT_PIXPOS[0]) == true_spat_pixpos_2, \
'Wrong object (ero191) location on the DEIMOS slit'
elif name == 'keck_mosfire':
# Check if recover the maskdef assignment
assert sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[
0] == true_maskdef_objname, 'Wrong MOSFIRE MASKDEF_OBJNAME'
assert round(sobjs[sobjs.SLITID == slitid].RA[0],
6) == true_ra, 'Wrong object MOSFIRE RA'
assert round(sobjs[sobjs.SLITID == slitid].DEC[0],
6) == true_dec, 'Wrong object MOSFIRE DEC'
# Test that undetected object are found at the correct location (the correct location is
# verified by visual inspection)
assert round(sobjs[sobjs.MASKDEF_OBJNAME == '7'].SPAT_PIXPOS[0]) == true_spat_pixpos, \
'Wrong object (7) location on the MOSFIRE slit'
# Write sobjs
sobjs.write_to_fits({}, data_path('tst_sobjs.fits'))
os.remove(data_path('tst_sobjs.fits'))
def get_slits(self, redo=False, write_qa=True):
"""
Load or generate the definition of the slit boundaries.
Internals that must be available are :attr:`fitstbl`,
:attr:`calib_ID`, :attr:`det`.
Args:
redo (bool): Redo
write_qa (bool, optional):
Generate the QA? Turn off for testing..
Returns:
Returns the trace-slits dictionary (also kept internally as
:attr:`tslits_dict`) and the slit mask array (numpy.ndarray;
also kept internally as :attr:`maskslits`)
"""
# Check for existing data
if not self._chk_objs(['msbpm']):
self.tslits_dict = None
return self.tslits_dict
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
trace_rows = self.fitstbl.find_frames('trace',
calib_ID=self.calib_ID,
index=True)
self.trace_image_files = self.fitstbl.frame_paths(trace_rows)
self.master_key_dict['trace'] \
= self.fitstbl.master_key(trace_rows[0] if len(trace_rows) > 0 else self.frame,
det=self.det)
# Return already generated data
if self._cached('trace', self.master_key_dict['trace']) and not redo:
self.tslits_dict = self.calib_dict[
self.master_key_dict['trace']]['trace']
return self.tslits_dict
# Instantiate
self.edges = edgetrace.EdgeTraceSet(
self.spectrograph,
self.par['slitedges'],
master_key=self.master_key_dict['trace'],
master_dir=self.master_dir,
qa_path=self.qa_path if write_qa else None)
if self.reuse_masters and self.edges.exists():
self.edges.load()
self.tslits_dict = self.edges.convert_to_tslits_dict()
else:
# Build the trace image
self.traceImage = traceimage.TraceImage(
self.spectrograph,
files=self.trace_image_files,
det=self.det,
par=self.par['traceframe'],
bias=self.msbias)
self.traceImage.build_image(bias=self.msbias, bpm=self.msbpm)
try:
self.edges.auto_trace(
self.traceImage,
bpm=self.msbpm,
det=self.det,
save=self.save_masters) #, debug=True, show_stages=True)
except:
self.edges.save()
msgs.error(
'Crashed out of finding the slits. Have saved the work done to disk '
'but it needs fixing.')
return None
# Show the result if requested
if self.show:
self.edges.show(thin=10, in_ginga=True)
# TODO: Stop-gap until we can get rid of tslits_dict
self.tslits_dict = self.edges.convert_to_tslits_dict()
# Save, initialize maskslits, and return
self._update_cache('trace', 'trace', self.tslits_dict)
return self.tslits_dict
def main(args):
import time
import os
import numpy as np
from pypeit.spectrographs.util import load_spectrograph
from pypeit import edgetrace
from pypeit import slittrace
from pypeit.pypeit import PypeIt
from pypeit.images import buildimage
from pypeit import masterframe
from IPython import embed
if args.pypeit_file is not None:
pypeit_file = args.pypeit_file
if not os.path.isfile(pypeit_file):
raise FileNotFoundError('File does not exist: {0}'.format(pypeit_file))
pypeit_file = os.path.abspath(pypeit_file)
redux_path = os.path.abspath(os.path.split(pypeit_file)[0]
if args.redux_path is None else args.redux_path)
rdx = PypeIt(pypeit_file, redux_path=redux_path)
detectors = rdx.par['rdx']['detnum'] if args.detector is None else args.detector
# Save the spectrograph
spec = rdx.spectrograph
# Get the calibration group to use
group = np.unique(rdx.fitstbl['calib'])[0] if args.group is None else args.group
if group not in np.unique(rdx.fitstbl['calib']):
raise ValueError('Not a valid calibration group: {0}'.format(group))
# Find the rows in the metadata table with trace frames in the
# specified calibration group
tbl_rows = rdx.fitstbl.find_frames('trace', calib_ID=int(group), index=True)
# Master keyword
master_key_base = '_'.join(rdx.fitstbl.master_key(tbl_rows[0]).split('_')[:2])
# Save the binning
binning = rdx.fitstbl['binning'][tbl_rows[0]]
# Save the full file paths
files = rdx.fitstbl.frame_paths(tbl_rows)
# Trace image processing parameters
proc_par = rdx.par['calibrations']['traceframe']
# Slit tracing parameters
trace_par = rdx.par['calibrations']['slitedges']
# Get the bias files, if requested
bias_rows = rdx.fitstbl.find_frames('bias', calib_ID=int(group), index=True)
bias_files = rdx.fitstbl.frame_paths(bias_rows)
bias_par = rdx.par['calibrations']['biasframe']
if len(bias_files) == 0:
bias_files = None
# Get the dark files, if requested
dark_rows = rdx.fitstbl.find_frames('dark', calib_ID=int(group), index=True)
dark_files = rdx.fitstbl.frame_paths(dark_rows)
dark_par = rdx.par['calibrations']['darkframe']
if len(dark_files) == 0:
dark_files = None
# Set the QA path
qa_path = rdx.qa_path
else:
detectors = args.detector
spec = load_spectrograph(args.spectrograph)
master_key_base = 'A_1'
binning = '1,1' if args.binning is None else args.binning
if not os.path.isfile(args.trace_file):
raise FileNotFoundError('File does not exist: {0}'.format(args.trace_file))
files = [os.path.abspath(args.trace_file)]
redux_path = os.path.abspath(os.path.split(files[0])[0]
if args.redux_path is None else args.redux_path)
par = spec.default_pypeit_par()
proc_par = par['calibrations']['traceframe']
trace_par = par['calibrations']['slitedges']
bias_files = None
bias_par = None
dark_files = None
dark_par = None
# Set the QA path
qa_path = os.path.join(os.path.abspath(os.path.split(files[0])[0]), 'QA')
if detectors is None:
detectors = np.arange(spec.ndet)+1
elif isinstance(detectors, (int, tuple)):
detectors = [detectors]
elif any([isinstance(d,str) for d in detectors]):
detectors = [eval(d) for d in detectors]
master_dir = os.path.join(redux_path, args.master_dir)
for det in detectors:
# Master keyword for output file name
master_key = f'{master_key_base}_{spec.get_det_name(det)}'
# Get the bias frame if requested
if bias_files is None:
proc_par['process']['use_biasimage'] = False
msbias = None
else:
msbias = buildimage.buildimage_fromlist(spec, det, bias_par, bias_files)
# Get the dark frame if requested
if dark_files is None:
proc_par['process']['use_darkimage'] = False
msdark = None
else:
msdark = buildimage.buildimage_fromlist(spec, det, dark_par, dark_files)
msbpm = spec.bpm(files[0], det)
# Build the trace image
traceImage = buildimage.buildimage_fromlist(spec, det, proc_par, files, bias=msbias,
bpm=msbpm, dark=msdark)
# Trace the slit edges
t = time.perf_counter()
edges = edgetrace.EdgeTraceSet(traceImage, spec, trace_par, auto=True,
debug=args.debug, show_stages=args.show,
qa_path=qa_path)
print('Tracing for detector {0} finished in {1} s.'.format(det, time.perf_counter()-t))
# Write the MasterEdges file
edge_masterframe_name = masterframe.construct_file_name(edgetrace.EdgeTraceSet,
master_key,
master_dir=master_dir)
edges.to_master_file(edge_masterframe_name)
# Write the MasterSlits file
slit_masterframe_name = masterframe.construct_file_name(slittrace.SlitTraceSet,
master_key,
master_dir=master_dir)
edges.get_slits().to_master_file(slit_masterframe_name)
return 0