def test_proc_diff(nires_sci_files, nires_bg_files):
"""
Run on near-IR frames
"""
# Setup
det = 1
bpm = np.zeros((2048, 1024))
pixelflat = np.ones_like(bpm)
# Sci image
flatImages = flatfield.FlatImages(pixelflat_norm=pixelflat)
sciImg = buildimage.buildimage_fromlist(keck_nires,
det,
nires_par['scienceframe'],
nires_sci_files,
bias=None,
bpm=bpm,
flatimages=flatImages)
# Bg image
bgImg = buildimage.buildimage_fromlist(keck_nires,
det,
nires_par['scienceframe'],
nires_bg_files,
bias=None,
bpm=bpm,
flatimages=flatImages)
# Difference
sciImg = sciImg.sub(bgImg, nires_par['scienceframe']['process'])
# Test
assert isinstance(sciImg, pypeitimage.PypeItImage)
def test_proc_diff(nires_sci_files, nires_bg_files):
"""
Run on near-IR frames
"""
# Setup
det = 1
bpm = np.zeros((2048, 1024), dtype=int)
pixelflat = np.ones(bpm.shape, dtype=float)
nires_par = keck_nires.default_pypeit_par()
# Sci image
flatImages = flatfield.FlatImages(pixelflat_norm=pixelflat)
nires_par['scienceframe']['process']['use_illumflat'] = False
nires_par['scienceframe']['process']['use_specillum'] = False
sciImg = buildimage.buildimage_fromlist(keck_nires,
det,
nires_par['scienceframe'],
nires_sci_files,
bias=None,
bpm=bpm,
flatimages=flatImages)
# Bg image
bgImg = buildimage.buildimage_fromlist(keck_nires,
det,
nires_par['scienceframe'],
nires_bg_files,
bias=None,
bpm=bpm,
flatimages=flatImages)
# Difference
sciImg = sciImg.sub(bgImg, nires_par['scienceframe']['process'])
# Test
assert isinstance(sciImg, pypeitimage.PypeItImage)
def test_maskdef_id():
# Load instrument
keck_deimos = load_spectrograph('keck_deimos')
par = keck_deimos.default_pypeit_par()
# working only on detector 1
det = 1
# Built trace image
traceImage = buildimage.buildimage_fromlist(
keck_deimos, det, par['calibrations']['traceframe'],
deimos_flat_files())
msbpm = keck_deimos.bpm(traceImage.files[0], det)
# load specific config parameters
par = keck_deimos.config_specific_par(traceImage.files[0])
trace_par = par['calibrations']['slitedges']
# Run edge trace
edges = EdgeTraceSet(traceImage,
keck_deimos,
trace_par,
bpm=msbpm,
auto=True,
debug=False,
show_stages=False,
qa_path=None)
slits = edges.get_slits()
# Check that the `maskdef_id` assigned to the first and last slits is correct
assert slits.maskdef_id[0] == 1098247, 'maskdef_id not found or wrong'
assert slits.maskdef_id[-1] == 1098226, 'maskdef_id not found or wrong'
def test_maskdef_id():
instr_names = ['keck_deimos', 'keck_mosfire']
for name in instr_names:
# Load instrument
instrument = load_spectrograph(name)
par = instrument.default_pypeit_par()
# working only on detector 1
det=1
# Built trace image
traceImage = buildimage.buildimage_fromlist(instrument, det, par['calibrations']['traceframe'],
flat_files(instr=name))
# load specific config parameters
par = instrument.config_specific_par(traceImage.files[0])
trace_par = par['calibrations']['slitedges']
# Run edge trace
edges = EdgeTraceSet(traceImage, instrument, trace_par, auto=True, debug=False,
show_stages=False,qa_path=None)
slits = edges.get_slits()
# Check that the `maskdef_id` assigned to the first and last slits is correct
if name == 'keck_deimos':
assert slits.maskdef_id[0] == 1098247, 'DEIMOS maskdef_id not found or wrong'
assert slits.maskdef_id[-1] == 1098226, 'DEIMOS maskdef_id not found or wrong'
# `maskdef_id` is the slit_id ("dSlitId") from the DEIMOS slit-mask design. If the matching is done
# correctly these are the slit_id values that the first and last slits should have. These values are
# coming from a reduction run with DEEP2 IDL-based pipeline.
elif name == 'keck_mosfire':
assert slits.maskdef_id[0] == 11, 'MOSFIRE maskdef_id not found or wrong'
assert slits.maskdef_id[-1] == 1, 'MOSFIRE maskdef_id not found or wrong'
def test_io(kast_blue_bias_files):
#
outfile = masterframe.construct_file_name(buildimage.BiasImage, master_key,
master_dir=master_dir)
if os.path.isfile(outfile):
os.remove(outfile)
# Build
msbias = buildimage.buildimage_fromlist(shane_kast_blue, 1, frame_par,
kast_blue_bias_files)
# Save as a master frame
master_filename = masterframe.construct_file_name(msbias, master_key, master_dir=master_dir)
msbias.to_master_file(master_filename)#master_dir, master_key, # Naming
#shane_kast_blue.spectrograph, # Header
#steps=msbias.process_steps,
#raw_files=kast_blue_bias_files)
assert os.path.isfile(outfile), 'Error writing MasterBias'
# Load master frame
biasImage = buildimage.BiasImage.from_file(outfile)
assert np.array_equal(biasImage.image, msbias.image)
# Instantiate from master frame
#bias_frame2 = biasframe.BiasFrame.from_master_file(bias_frame.master_file_path)
#assert np.array_equal(pypeitImage.image, bias_frame2.pypeitImage.image)
# Clean up
os.remove(outfile)
def get_align(self):
"""
Load or generate the alignment frame
Requires: :attr:`slits`, :attr:`det`, :attr:`par`
Returns:
:class:`pypeit.alignframe.Alignments`:
"""
# Check for existing data
if not self._chk_objs(['msbpm', 'slits']):
msgs.error('Must have the bpm and slits to make the alignments!')
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
align_files, self.master_key_dict['align'] = self._prep_calibrations(
'align')
masterframe_filename = masterframe.construct_file_name(
alignframe.Alignments,
self.master_key_dict['align'],
master_dir=self.master_dir)
# Reuse master frame?
if os.path.isfile(masterframe_filename) and self.reuse_masters:
self.alignments = alignframe.Alignments.from_file(
masterframe_filename)
self.alignments.is_synced(self.slits)
return self.alignments
msalign = buildimage.buildimage_fromlist(self.spectrograph,
self.det,
self.par['alignframe'],
align_files,
bias=self.msbias,
bpm=self.msbpm,
dark=self.msdark)
# Extract some header info needed by the algorithm
binning = self.spectrograph.get_meta_value(align_files[0], 'binning')
# Instantiate
# TODO: From JFH: Do we need the bpm here? Check that this was in the previous code.
alignment = alignframe.TraceAlignment(msalign,
self.slits,
self.spectrograph,
self.par['alignment'],
det=self.det,
binning=binning,
qa_path=self.qa_path,
msbpm=self.msbpm)
# Run
self.alignments = alignment.run(show=self.show)
# Save to Masters
self.alignments.to_master_file(masterframe_filename)
return self.alignments
def test_overlapped_slits():
# Load flats frames that have overlapping alignment slits.
deimos_flats = [os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'keck_deimos',
'1200G_alignslit', ifile)
for ifile in ['DE.20110529.11732.fits', 'DE.20110529.11813.fits',
'DE.20110529.11895.fits']]
# Load instrument
keck_deimos = load_spectrograph('keck_deimos')
par = keck_deimos.default_pypeit_par()
# working only on detector 2
det = 2
# Built trace image
traceImage = buildimage.buildimage_fromlist(keck_deimos, det,
par['calibrations']['traceframe'], deimos_flats)
# load specific config parameters
par = keck_deimos.config_specific_par(traceImage.files[0])
trace_par = par['calibrations']['slitedges']
# Run edge trace
edges = EdgeTraceSet(traceImage, keck_deimos, trace_par, auto=True, debug=False,
show_stages=False, qa_path=None)
slits = edges.get_slits()
# Check that the total number of expected slits and the number of alignment slits are correct.
assert len(slits.maskdef_designtab['MASKDEF_ID'].data) == 22, \
'wrong number of slits for this detector'
assert np.sum(slits.maskdef_designtab['ALIGN'].data) == 3, 'wrong number of alignment slits'
def test_assign_maskinfo():
# Spectrograph
keck_deimos = KeckDEIMOSSpectrograph()
par = keck_deimos.default_pypeit_par()
# working only on detector 3
det = 3
# Built trace image
traceImage = buildimage.buildimage_fromlist(
keck_deimos, det, par['calibrations']['traceframe'],
deimos_flat_files())
msbpm = keck_deimos.bpm(traceImage.files[0], det)
# load specific config parameters
par = keck_deimos.config_specific_par(traceImage.files[0])
trace_par = par['calibrations']['slitedges']
# Run edge trace
edges = edgetrace.EdgeTraceSet(traceImage,
keck_deimos,
trace_par,
bpm=msbpm,
auto=True,
debug=False,
show_stages=False,
qa_path=None)
slits = edges.get_slits()
# Test that the maskfile is saved properly
hdul = fits.open(slits.maskfile)
det_par = keck_deimos.get_detector_par(hdul, det=det)
specobjs_file = os.path.join(
os.getenv('PYPEIT_DEV'), 'Cooked', 'Science',
'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_2010Sep13T061231.334.fits')
# specobjs_file = os.path.join(os.getenv('PYPEIT_DEV'), 'REDUX_OUT', 'keck_deimos',
# '830G_M_8500', 'Science',
# 'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_2010Sep13T061231.334.fits')
sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file)
# Init at null
for sobj in sobjs:
sobj.MASKDEF_OBJNAME = None
sobj.RA = None
sobj.DEC = None
# Run me
slits.assign_maskinfo(sobjs, det_par['platescale'])
# Test
assert sobjs[sobjs.SLITID ==
496].MASKDEF_OBJNAME == 'ero89', 'Wrong MASKDEF_OBJNAME'
assert sobjs[sobjs.SLITID == 496].RA == 352.27471667, 'Wrong object RA'
assert sobjs[sobjs.SLITID == 496].DEC == -3.09223056, 'Wrong object DEC'
# Write sobjs
sobjs.write_to_fits({}, data_path('tst_sobjs.fits'))
os.remove(data_path('tst_sobjs.fits'))
def test_process_multidet():
files = glob.glob(
os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'gemini_gmos',
'GN_HAM_R400_885', 'N20190205S024*.fits'))
files.sort()
spec = load_spectrograph('gemini_gmos_north_ham')
frame_par = spec.default_pypeit_par()['calibrations']['biasframe']
det = 1
bias_img_det1 = buildimage.buildimage_fromlist(spec, det, frame_par, files)
det = (1, 2, 3)
bias_img = buildimage.buildimage_fromlist(spec, det, frame_par, files)
assert np.array_equal(bias_img_det1.image, bias_img.image[0]) \
and not np.array_equal(bias_img_det1.image, bias_img.image[1]) \
and not np.array_equal(bias_img_det1.image, bias_img.image[2]), \
'Bad multi-detector processing'
def test_process(deimos_flat_files):
keck_deimos = load_spectrograph('keck_deimos')
par = keck_deimos.default_pypeit_par()
par['calibrations']['traceframe']['process']['use_biasimage'] = False
# Instantiate
traceImage = buildimage.buildimage_fromlist(keck_deimos, 1, par['calibrations']['traceframe'],
deimos_flat_files)
# Run
assert isinstance(traceImage.image, np.ndarray)
for key in ['subtract_overscan', 'apply_gain']:
assert key in traceImage.process_steps
def test_combine(deimos_flat_files):
spectograph = load_spectrograph('keck_deimos')
par = spectograph.default_pypeit_par()
par['calibrations']['pixelflatframe']['process']['use_biasimage'] = False
# DEIMOS
deimos_flat = buildimage.buildimage_fromlist(
spectograph, 3, par['calibrations']['pixelflatframe'],
deimos_flat_files)
# Process steps
# Test
assert deimos_flat.image.shape == (4096, 2048)
def get_tiltimg(self):
"""
Load or generate the Tilt image
Requirements:
master_key, det, par
Args:
Returns:
`numpy.ndarray`_: :attr:`mstilt` image
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
tilt_files, self.master_key_dict['tilt'] = self._prep_calibrations(
'tilt')
masterframe_name = masterframe.construct_file_name(
buildimage.TiltImage,
self.master_key_dict['tilt'],
master_dir=self.master_dir)
# Reuse master frame?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.mstilt = buildimage.TiltImage.from_file(masterframe_name)
elif len(tilt_files) == 0:
msgs.warn("No frametype=tilt files to build tiltimg")
return
else: # Build
msgs.info("Preparing a master {0:s} frame".format(
buildimage.TiltImage.master_type))
# NOTE: Slits passed for the spatial flexure correction
self.mstilt = buildimage.buildimage_fromlist(self.spectrograph,
self.det,
self.par['tiltframe'],
tilt_files,
bias=self.msbias,
bpm=self.msbpm,
dark=self.msdark,
slits=self.slits)
# Save to Masters
self.mstilt.to_master_file(masterframe_name)
# TODO in the future add in a tilt_inmask
#self._update_cache('tilt', 'tilt_inmask', self.mstilt_inmask)
# Return
return self.mstilt
def get_dark(self):
"""
Load or generate the dark image
Returns:
:class:`~pypeit.images.buildimage.DarkImage`: The combined dark
image.
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
dark_files, self.master_key_dict['dark'] = self._prep_calibrations(
'dark')
# Construct the name, in case we need it
masterframe_name = masterframe.construct_file_name(
buildimage.DarkImage,
self.master_key_dict['dark'],
master_dir=self.master_dir)
# Try to load?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.msdark = buildimage.DarkImage.from_file(masterframe_name)
elif len(dark_files) == 0:
self.msdark = None
else:
# TODO: If a bias has been constructed and it will be subtracted
# from the science images, it should also be subtracted from this
# image. If it isn't, subtracting the dark will effectively lead to
# subtracting the bias twice.
# TODO: The order is such that the bpm doesn't exist yet. But
# calling buildimage_fromlist will create the bpm if it isn't
# passed. So calling get_dark then get_bpm unnecessarily creates
# the bpm twice. Is there any reason why creation of the bpm should
# come after the dark, or can we change the order?
# Build and save it
self.msdark = buildimage.buildimage_fromlist(self.spectrograph,
self.det,
self.par['darkframe'],
dark_files,
bias=self.msbias)
self.msdark.to_master_file(masterframe_name)
# Return
return self.msdark
def test_lris_red_biases():
path = pathlib.Path(os.getenv('PYPEIT_DEV')).absolute() / 'RAW_DATA' / 'keck_lris_red' \
/ 'multi_600_5000_d560'
files = [
str(path / f) for f in [
'LR.20170324.06731.fits.gz', 'LR.20170324.06791.fits.gz',
'LR.20170324.06849.fits.gz', 'LR.20170324.06908.fits.gz',
'LR.20170324.06967.fits.gz'
]
]
spectrograph = load_spectrograph('keck_lris_red')
par = spectrograph.default_pypeit_par()
bias = buildimage.buildimage_fromlist(spectrograph, 1,
par['calibrations']['biasframe'],
files)
def get_arc(self):
"""
Load or generate the Arc image
Requirements:
master_key, det, par
Args:
Returns:
`numpy.ndarray`_: :attr:`msarc` image
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
arc_files, self.master_key_dict['arc'] = self._prep_calibrations('arc')
masterframe_name = masterframe.construct_file_name(
buildimage.ArcImage,
self.master_key_dict['arc'],
master_dir=self.master_dir)
# Reuse master frame?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.msarc = buildimage.ArcImage.from_file(masterframe_name)
elif len(arc_files) == 0:
msgs.warn("No frametype=arc files to build arc")
return
else: # Build it
msgs.info("Preparing a master {0:s} frame".format(
buildimage.ArcImage.master_type))
self.msarc = buildimage.buildimage_fromlist(self.spectrograph,
self.det,
self.par['arcframe'],
arc_files,
bias=self.msbias,
bpm=self.msbpm,
dark=self.msdark)
# Save
self.msarc.to_master_file(masterframe_name)
# Return
return self.msarc
def test_combine_deimos_flats():
deimos_flat_files = [
os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'keck_deimos',
'830G_L_8400', ifile)
for ifile in ['d0914_0014.fits.gz', 'd0914_0015.fits.gz']
]
assert len(deimos_flat_files) == 2, 'Incorrect number of files.'
spectrograph = load_spectrograph('keck_deimos')
par = spectrograph.default_pypeit_par()
par['calibrations']['pixelflatframe']['process']['use_biasimage'] = False
# DEIMOS
deimos_flat = buildimage.buildimage_fromlist(
spectrograph, 3, par['calibrations']['pixelflatframe'],
deimos_flat_files)
# Process steps
# Test
assert deimos_flat.image.shape == (4096, 2048)
def get_bias(self):
"""
Load or generate the bias frame/command
Requirements:
master_key, det, par
Returns:
:class:`pypeit.images.buildimage.BiasImage`:
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
bias_files, self.master_key_dict['bias'] = self._prep_calibrations(
'bias')
# Construct the name, in case we need it
masterframe_name = masterframe.construct_file_name(
buildimage.BiasImage,
self.master_key_dict['bias'],
master_dir=self.master_dir)
if self.par['biasframe']['useframe'] is not None:
msgs.error("Not ready to load from disk")
# Try to load?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.msbias = buildimage.BiasImage.from_file(masterframe_name)
elif len(bias_files) == 0:
self.msbias = None
else:
# Build it
self.msbias = buildimage.buildimage_fromlist(
self.spectrograph, self.det, self.par['biasframe'], bias_files)
# Save it?
self.msbias.to_master_file(masterframe_name)
# Return
return self.msbias
def test_from_list(shane_kast_blue_sci_files):
"""
Run on two frames
"""
det = 1
# Load calibrations
pixelflat = load_kast_blue_masters(pixflat=True)[0]
bpm = kast_blue.empty_bpm(shane_kast_blue_sci_files[0], det)
# Do it
flatImages = flatfield.FlatImages(pixelflat_norm=pixelflat)
kast_par['scienceframe']['process']['use_illumflat'] = False
kast_par['scienceframe']['process']['use_biasimage'] = False
sciImg = buildimage.buildimage_fromlist(kast_blue,
det,
kast_par['scienceframe'],
shane_kast_blue_sci_files,
bpm=bpm,
bias=None,
flatimages=flatImages)
# Test
assert isinstance(sciImg, pypeitimage.PypeItImage)
def get_dark(self):
"""
Load or generate the dark image
Requirements:
master_key, det, par
Returns:
:class:`pypeit.images.buildimage.DarkImage`:
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
dark_files, self.master_key_dict['dark'] = self._prep_calibrations(
'dark')
# Construct the name, in case we need it
masterframe_name = masterframe.construct_file_name(
buildimage.DarkImage,
self.master_key_dict['dark'],
master_dir=self.master_dir)
# Try to load?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.msdark = buildimage.DarkImage.from_file(masterframe_name)
elif len(dark_files) == 0:
self.msdark = None
else:
# TODO: Should this include the bias?
# Build it
self.msdark = buildimage.buildimage_fromlist(
self.spectrograph, self.det, self.par['darkframe'], dark_files)
# Save it?
self.msdark.to_master_file(masterframe_name)
# Return
return self.msdark
def test_mosaic_io():
outfile = data_path('test_bias_mosaic.fits')
if os.path.isfile(outfile):
os.remove(outfile)
files = glob.glob(
os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'gemini_gmos',
'GN_HAM_R400_885', 'N20190205S024*.fits'))
files.sort()
spec = load_spectrograph('gemini_gmos_north_ham')
frame_par = spec.default_pypeit_par()['calibrations']['biasframe']
det = (1, 2, 3)
bias = buildimage.buildimage_fromlist(spec, det, frame_par, files)
bias.to_file(outfile)
_bias = buildimage.BiasImage.from_file(outfile)
assert isinstance(_bias.detector, Mosaic), 'Bad detector type'
assert _bias.detector.ndet == bias.detector.ndet, 'Bad number of detectors'
assert np.array_equal(_bias.detector.tform,
bias.detector.tform), 'Bad number of detectors'
# Clean up
os.remove(outfile)
def test_io(kast_blue_bias_files):
#
outfile = masterframe.construct_file_name(buildimage.BiasImage,
master_key,
master_dir=master_dir)
if os.path.isfile(outfile):
os.remove(outfile)
# Build
msbias = buildimage.buildimage_fromlist(shane_kast_blue, 1, frame_par,
kast_blue_bias_files)
# Save as a master frame
master_filename = masterframe.construct_file_name(msbias,
master_key,
master_dir=master_dir)
msbias.to_master_file(master_filename)
assert os.path.isfile(outfile), 'Error writing MasterBias'
# Load master frame
biasImage = buildimage.BiasImage.from_file(outfile)
assert np.array_equal(biasImage.image, msbias.image), 'Image changed'
assert np.array_equal(biasImage.ivar,
msbias.ivar), 'Inverse-variance changed'
# Clean up
os.remove(outfile)
def get_flats(self):
"""
Load or generate a normalized pixel flat and slit illumination
flat.
Requires :attr:`slits`, :attr:`wavetilts`, :attr:`det`,
:attr:`par`.
Constructs :attr:`flatimages`.
"""
# Check for existing data
if not self._chk_objs(['msarc', 'msbpm', 'slits', 'wv_calib']):
msgs.warn(
'Must have the arc, bpm, slits, and wv_calib defined to make flats! Skipping and may crash down the line'
)
self.flatimages = flatfield.FlatImages()
return
# Slit and tilt traces are required to flat-field the data
if not self._chk_objs(['slits', 'wavetilts']):
# TODO: Why doesn't this fault?
msgs.warn(
'Flats were requested, but there are quantities missing necessary to '
'create flats. Proceeding without flat fielding....')
self.flatimages = flatfield.FlatImages()
return
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
illum_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('illumflat')
pixflat_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('pixelflat')
masterframe_filename = masterframe.construct_file_name(
flatfield.FlatImages,
self.master_key_dict['flat'],
master_dir=self.master_dir)
# The following if-elif-else does:
# 1. Try to load a MasterFrame (if reuse_masters is True). If successful, pass it back
# 2. Build from scratch
# 3. Load any user-supplied images to over-ride any built
# Load MasterFrame?
if os.path.isfile(masterframe_filename) and self.reuse_masters:
flatimages = flatfield.FlatImages.from_file(masterframe_filename)
flatimages.is_synced(self.slits)
# Load user defined files
if self.par['flatfield']['pixelflat_file'] is not None:
# Load
msgs.info(
'Using user-defined file: {0}'.format('pixelflat_file'))
with fits.open(self.par['flatfield']['pixelflat_file']) as hdu:
nrm_image = flatfield.FlatImages(
pixelflat_norm=hdu[self.det].data)
flatimages = flatfield.merge(flatimages, nrm_image)
self.flatimages = flatimages
# update slits
self.slits.mask_flats(self.flatimages)
return self.flatimages
# Generate the image
pixelflatImages, illumflatImages = None, None
# Check if the image files are the same
pix_is_illum = Counter(illum_image_files) == Counter(
pixflat_image_files)
if len(pixflat_image_files) > 0:
pixel_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['pixelflatframe'],
pixflat_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
# Initialise the pixel flat
pixelFlatField = flatfield.FlatField(pixel_flat, self.spectrograph,
self.par['flatfield'],
self.slits, self.wavetilts,
self.wv_calib)
# Generate
pixelflatImages = pixelFlatField.run(show=self.show)
# Only build illum_flat if the input files are different from the pixel flat
if (not pix_is_illum) and len(illum_image_files) > 0:
illum_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['illumflatframe'],
illum_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
# Initialise the pixel flat
illumFlatField = flatfield.FlatField(illum_flat,
self.spectrograph,
self.par['flatfield'],
self.slits,
self.wavetilts,
self.wv_calib,
spat_illum_only=True)
# Generate
illumflatImages = illumFlatField.run(show=self.show)
# Merge the illum flat with the pixel flat
if pixelflatImages is not None:
# Combine the pixelflat and illumflat parameters into flatimages.
# This will merge the attributes of pixelflatImages that are not None
# with the attributes of illflatImages that are not None. Default is
# to take pixelflatImages.
flatimages = flatfield.merge(pixelflatImages, illumflatImages)
else:
# No pixel flat, but there might be an illumflat. This will mean that
# the attributes prefixed with 'pixelflat_' will all be None.
flatimages = illumflatImages
# Save flat images
if flatimages is not None:
flatimages.to_master_file(masterframe_filename)
# Save slits too, in case they were tweaked
self.slits.to_master_file()
# 3) Load user-supplied images
# NOTE: This is the *final* images, not just a stack
# And it will over-ride what is generated below (if generated)
if self.par['flatfield']['pixelflat_file'] is not None:
# Load
msgs.info('Using user-defined file: {0}'.format('pixelflat_file'))
with fits.open(self.par['flatfield']['pixelflat_file']) as hdu:
flatimages = flatfield.merge(
flatimages,
flatfield.FlatImages(pixelflat_norm=hdu[self.det].data))
self.flatimages = flatimages
# Return
return self.flatimages
def test_assign_maskinfo_add_missing():
instr_names = ['keck_deimos', 'keck_mosfire']
for name in instr_names:
# Spectrograph
instrument = load_spectrograph(name)
par = instrument.default_pypeit_par()
# working only on detector 3 (det=3 for DEIMOS. For MOSFIRE does not matter because we have only one det)
det = 3 if name == 'keck_deimos' else 1
# Built trace image
traceImage = buildimage.buildimage_fromlist(
instrument, det, par['calibrations']['traceframe'],
flat_files(instr=name))
# load specific config parameters
par = instrument.config_specific_par(traceImage.files[0])
# Run edge trace
edges = edgetrace.EdgeTraceSet(traceImage,
instrument,
par['calibrations']['slitedges'],
auto=True,
debug=False,
show_stages=False,
qa_path=None)
slits = edges.get_slits()
# Test that the maskfile is saved properly
hdul = fits.open(slits.maskfile)
det_par = instrument.get_detector_par(det, hdu=hdul)
if name == 'keck_deimos':
specobjs_file = os.path.join(
os.getenv('PYPEIT_DEV'), 'Cooked', 'Science',
'spec1d_DE.20100913.22358-CFHQS1_DEIMOS_20100913T061231.334.fits'
)
sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file)
# correct value
slitid = sobjs[sobjs.MASKDEF_OBJNAME == 'ero89'].SLITID[0]
true_maskdef_objname = sobjs[sobjs.SLITID ==
slitid].MASKDEF_OBJNAME[0]
true_ra = round(sobjs[sobjs.SLITID == slitid].RA[0], 6)
true_dec = round(sobjs[sobjs.SLITID == slitid].DEC[0], 6)
true_spat_pixpos = round(
sobjs[sobjs.MASKDEF_OBJNAME == 'ero884'].SPAT_PIXPOS[0])
true_spat_pixpos_2 = round(
sobjs[sobjs.MASKDEF_OBJNAME == 'ero191'].SPAT_PIXPOS[0])
elif name == 'keck_mosfire':
specobjs_file = os.path.join(
os.getenv('PYPEIT_DEV'), 'Cooked', 'Science',
'spec1d_m191014_0170-2M2228_12_MOSFIRE_20191014T095212.598.fits'
)
sobjs = specobjs.SpecObjs.from_fitsfile(specobjs_file)
# correct value
slitid = sobjs[sobjs.MASKDEF_OBJNAME == '18'].SLITID[0]
true_maskdef_objname = sobjs[sobjs.SLITID ==
slitid].MASKDEF_OBJNAME[0]
true_ra = round(sobjs[sobjs.SLITID == slitid].RA[0], 6)
true_dec = round(sobjs[sobjs.SLITID == slitid].DEC[0], 6)
true_spat_pixpos = round(
sobjs[sobjs.MASKDEF_OBJNAME == '7'].SPAT_PIXPOS[0])
# Init at null and remove the force extraction
idx_remove = []
for i, sobj in enumerate(sobjs):
if sobj.MASKDEF_EXTRACT:
idx_remove.append(i)
else:
sobj.MASKDEF_ID = None
sobj.MASKDEF_OBJNAME = None
sobj.RA = None
sobj.DEC = None
sobj.MASKDEF_EXTRACT = None
sobjs.remove_sobj(idx_remove)
# get the dither offset if available
if name == 'keck_deimos':
dither_off = None
elif name == 'keck_mosfire':
dither_off = instrument.parse_dither_pattern([
os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA',
'keck_mosfire', 'J_multi', 'm191014_0170.fits')
])[2][0]
# get object positions from slitmask design and slitmask offsets
calib_slits = slittrace.get_maskdef_objpos_offset_alldets(
sobjs, [slits], [None], [det_par['platescale']],
par['calibrations']['slitedges']['det_buffer'],
par['reduce']['slitmask'],
dither_off=dither_off)
# determine if slitmask offsets exist and compute an average offsets over all the detectors
calib_slits = slittrace.average_maskdef_offset(
calib_slits, det_par['platescale'], instrument.list_detectors())
# slitmask design matching and add undetected objects
sobjs = slittrace.assign_addobjs_alldets(
sobjs, calib_slits, [None], [det_par['platescale']],
par['reduce']['slitmask'], par['reduce']['findobj']['find_fwhm'])
# Test
if name == 'keck_deimos':
# Check if recover the maskdef assignment
assert sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[
0] == true_maskdef_objname, 'Wrong DEIMOS MASKDEF_OBJNAME'
assert round(sobjs[sobjs.SLITID == slitid].RA[0],
6) == true_ra, 'Wrong object DEIMOS RA'
assert round(sobjs[sobjs.SLITID == slitid].DEC[0],
6) == true_dec, 'Wrong object DEIMOS DEC'
# Test that undetected objects are found at the correct location (the correct location is
# verified by visual inspection)
assert round(sobjs[sobjs.MASKDEF_OBJNAME == 'ero884'].SPAT_PIXPOS[0]) == true_spat_pixpos, \
'Wrong object (ero884) location on the DEIMOS slit'
assert round(sobjs[sobjs.MASKDEF_OBJNAME == 'ero191'].SPAT_PIXPOS[0]) == true_spat_pixpos_2, \
'Wrong object (ero191) location on the DEIMOS slit'
elif name == 'keck_mosfire':
# Check if recover the maskdef assignment
assert sobjs[sobjs.SLITID == slitid].MASKDEF_OBJNAME[
0] == true_maskdef_objname, 'Wrong MOSFIRE MASKDEF_OBJNAME'
assert round(sobjs[sobjs.SLITID == slitid].RA[0],
6) == true_ra, 'Wrong object MOSFIRE RA'
assert round(sobjs[sobjs.SLITID == slitid].DEC[0],
6) == true_dec, 'Wrong object MOSFIRE DEC'
# Test that undetected object are found at the correct location (the correct location is
# verified by visual inspection)
assert round(sobjs[sobjs.MASKDEF_OBJNAME == '7'].SPAT_PIXPOS[0]) == true_spat_pixpos, \
'Wrong object (7) location on the MOSFIRE slit'
# Write sobjs
sobjs.write_to_fits({}, data_path('tst_sobjs.fits'))
os.remove(data_path('tst_sobjs.fits'))
def main(args):
# Build the fitstable since we currently need it for output. This should not be the case!
A_files = [os.path.join(args.full_rawpath, file) for file in args.Afiles]
B_files = [os.path.join(args.full_rawpath, file) for file in args.Bfiles]
data_files = A_files + B_files
ps = pypeitsetup.PypeItSetup(A_files,
path='./',
spectrograph_name='keck_mosfire')
ps.build_fitstbl()
fitstbl = ps.fitstbl
# Read in the spectrograph, config the parset
spectrograph = load_spectrograph('keck_mosfire')
spectrograph_def_par = spectrograph.default_pypeit_par()
parset = par.PypeItPar.from_cfg_lines(
cfg_lines=spectrograph_def_par.to_config(),
merge_with=config_lines(args))
science_path = os.path.join(parset['rdx']['redux_path'],
parset['rdx']['scidir'])
# Calibration Master directory
if args.master_dir is None:
msgs.error(
"You need to set an Environmental variable MOSFIRE_MASTERS that points at the Master Calibs"
)
# Define some hard wired master files here to be later parsed out of the directory
slit_masterframe_name = os.path.join(args.master_dir,
'MasterSlits_E_15_01.fits.gz')
tilts_masterframe_name = os.path.join(args.master_dir,
'MasterTilts_E_1_01.fits')
wvcalib_masterframe_name = os.path.join(args.master_dir,
'MasterWaveCalib_E_1_01.fits')
# For now don't require a standard
std_outfile = None
#std_outfile = os.path.join('/Users/joe/Dropbox/PypeIt_Redux/MOSFIRE/Nov19/quicklook/Science/',
# 'spec1d_m191118_0064-GD71_MOSFIRE_2019Nov18T104704.507.fits')
# make the get_std from pypeit a utility function or class method
det = 1 # MOSFIRE has a single detector
if std_outfile is not None:
# Get the standard trace if need be
sobjs = specobjs.SpecObjs.from_fitsfile(std_outfile)
this_det = sobjs.DET == det
if np.any(this_det):
sobjs_det = sobjs[this_det]
sobjs_std = sobjs_det.get_std()
std_trace = None if sobjs_std is None else sobjs_std.TRACE_SPAT.flatten(
)
else:
std_trace = None
else:
std_trace = None
# Read in the msbpm
sdet = get_dnum(det, prefix=False)
msbpm = spectrograph.bpm(A_files[0], det)
# Read in the slits
slits = slittrace.SlitTraceSet.from_file(slit_masterframe_name)
# Reset the bitmask
slits.mask = slits.mask_init.copy()
# Read in the wv_calib
wv_calib = wavecalib.WaveCalib.from_file(wvcalib_masterframe_name)
wv_calib.is_synced(slits)
slits.mask_wvcalib(wv_calib)
# Read in the tilts
tilts_obj = wavetilts.WaveTilts.from_file(tilts_masterframe_name)
tilts_obj.is_synced(slits)
slits.mask_wavetilts(tilts_obj)
# Build Science image
sciImg = buildimage.buildimage_fromlist(spectrograph,
det,
parset['scienceframe'],
A_files,
bpm=msbpm,
slits=slits,
ignore_saturation=False)
# Background Image?
sciImg = sciImg.sub(
buildimage.buildimage_fromlist(spectrograph,
det,
parset['scienceframe'],
B_files,
bpm=msbpm,
slits=slits,
ignore_saturation=False),
parset['scienceframe']['process'])
# Build the Calibrate object
caliBrate = calibrations.Calibrations(None, parset['calibrations'],
spectrograph, None)
caliBrate.slits = slits
caliBrate.wavetilts = tilts_obj
caliBrate.wv_calib = wv_calib
# Instantiate Reduce object
# Required for pypeline specific object
# At instantiaton, the fullmask in self.sciImg is modified
redux = reduce.Reduce.get_instance(sciImg,
spectrograph,
parset,
caliBrate,
'science',
ir_redux=True,
show=args.show,
det=det,
std_outfile=std_outfile)
manual_extract_dict = None
skymodel, objmodel, ivarmodel, outmask, sobjs, waveImg, tilts = redux.run(
std_trace=std_trace,
return_negative=True,
manual_extract_dict=manual_extract_dict,
show_peaks=args.show)
# TODO -- Do this upstream
# Tack on detector
for sobj in sobjs:
sobj.DETECTOR = sciImg.detector
# Construct the Spec2DObj with the positive image
spec2DObj_A = spec2dobj.Spec2DObj(det=det,
sciimg=sciImg.image,
ivarraw=sciImg.ivar,
skymodel=skymodel,
objmodel=objmodel,
ivarmodel=ivarmodel,
waveimg=waveImg,
bpmmask=outmask,
detector=sciImg.detector,
sci_spat_flexure=sciImg.spat_flexure,
tilts=tilts,
slits=copy.deepcopy(caliBrate.slits))
spec2DObj_A.process_steps = sciImg.process_steps
all_spec2d = spec2dobj.AllSpec2DObj()
all_spec2d['meta']['ir_redux'] = True
all_spec2d[det] = spec2DObj_A
# Save image A but with all the objects extracted, i.e. positive and negative
#outfile2d, outfile1d = save_exposure(fitstbl, 0, spectrograph, science_path, parset, caliBrate, all_spec2d, sobjs)
# Construct the Spec2DObj with the negative image
spec2DObj_B = spec2dobj.Spec2DObj(det=det,
sciimg=-sciImg.image,
ivarraw=sciImg.ivar,
skymodel=-skymodel,
objmodel=-objmodel,
ivarmodel=ivarmodel,
waveimg=waveImg,
bpmmask=outmask,
detector=sciImg.detector,
sci_spat_flexure=sciImg.spat_flexure,
tilts=tilts,
slits=copy.deepcopy(caliBrate.slits))
# Parse the offset information out of the headers. TODO in the future get this out of fitstable
dither_pattern_A, dither_id_A, offset_arcsec_A = parse_dither_pattern(
A_files, spectrograph.primary_hdrext)
dither_pattern_B, dither_id_B, offset_arcsec_B = parse_dither_pattern(
B_files, spectrograph.primary_hdrext)
# Print out a report on the offsets
msg_string = msgs.newline(
) + '****************************************************'
msg_string += msgs.newline(
) + ' Summary of offsets for dither pattern: {:s}'.format(
dither_pattern_A[0])
msg_string += msgs.newline(
) + '****************************************************'
msg_string += msgs.newline(
) + 'Position filename arcsec pixels '
msg_string += msgs.newline(
) + '----------------------------------------------------'
for iexp, file in enumerate(A_files):
msg_string += msgs.newline(
) + ' A {:s} {:6.2f} {:6.2f}'.format(
os.path.basename(file), offset_arcsec_A[iexp],
offset_arcsec_A[iexp] / sciImg.detector.platescale)
for iexp, file in enumerate(B_files):
msg_string += msgs.newline(
) + ' B {:s} {:6.2f} {:6.2f}'.format(
os.path.basename(file), offset_arcsec_B[iexp],
offset_arcsec_B[iexp] / sciImg.detector.platescale)
msg_string += msgs.newline(
) + '****************************************************'
msgs.info(msg_string)
#offset_dith_pix = offset_dith_pix = offset_arcsec_A[0]/sciImg.detector.platescale
offsets_dith_pix = (np.array([
0.0, np.mean(offset_arcsec_B) - np.mean(offset_arcsec_A)
])) / sciImg.detector.platescale
if args.offset is not None:
offsets_pixels = np.array([0.0, args.offset])
msgs.info('Using user specified offsets instead: {:5.2f}'.format(
args.offset))
else:
offsets_pixels = offsets_dith_pix
spec2d_list = [spec2DObj_A, spec2DObj_B]
# Instantiate Coadd2d
coadd = coadd2d.CoAdd2D.get_instance(spec2d_list,
spectrograph,
parset,
det=det,
offsets=offsets_pixels,
weights='uniform',
ir_redux=True,
debug=args.show,
samp_fact=args.samp_fact)
# Coadd the slits
coadd_dict_list = coadd.coadd(
only_slits=None, interp_dspat=False) # TODO implement only_slits later
# Create the pseudo images
pseudo_dict = coadd.create_pseudo_image(coadd_dict_list)
##########################
# Now display the images #
##########################
display.display.connect_to_ginga(raise_err=True, allow_new=True)
# Bug in ginga prevents me from using cuts here for some reason
#mean, med, sigma = sigma_clipped_stats(pseudo_dict['imgminsky'][pseudo_dict['inmask']], sigma_lower=5.0,sigma_upper=5.0)
#cut_min = mean - 4.0 * sigma
#cut_max = mean + 4.0 * sigma
chname_skysub = 'skysub-det{:s}'.format(sdet)
# Clear all channels at the beginning
# TODO: JFH For some reason Ginga crashes when I try to put cuts in here.
viewer, ch = ginga.show_image(pseudo_dict['imgminsky'],
chname=chname_skysub,
waveimg=pseudo_dict['waveimg'],
clear=True) # cuts=(cut_min, cut_max),
slit_left, slit_righ, _ = pseudo_dict['slits'].select_edges()
slit_id = slits.slitord_id[0]
ginga.show_slits(viewer, ch, slit_left, slit_righ, slit_ids=slit_id)
# SKRESIDS
chname_skyresids = 'sky_resid-det{:s}'.format(sdet)
image = pseudo_dict['imgminsky'] * np.sqrt(
pseudo_dict['sciivar']) * pseudo_dict['inmask'] # sky residual map
viewer, ch = ginga.show_image(
image,
chname_skyresids,
waveimg=pseudo_dict['waveimg'],
cuts=(-5.0, 5.0),
)
ginga.show_slits(viewer,
ch,
slit_left,
slit_righ,
slit_ids=slits.slitord_id[0])
shell = viewer.shell()
out = shell.start_global_plugin('WCSMatch')
out = shell.call_global_plugin_method('WCSMatch', 'set_reference_channel',
[chname_skyresids], {})
if args.embed:
embed()
return 0
def get_flats(self):
"""
Load or generate a normalized pixel flat and slit illumination
flat.
Requires :attr:`slits`, :attr:`wavetilts`, :attr:`det`,
:attr:`par`.
Returns:
:class:`pypeit.flatfield.FlatImages`:
"""
# Check for existing data
if not self._chk_objs(['msarc', 'msbpm', 'slits', 'wv_calib']):
msgs.warn(
'Must have the arc, bpm, slits, and wv_calib defined to make flats! Skipping and may crash down the line'
)
self.flatimages = flatfield.FlatImages(None, None, None, None)
return
# Slit and tilt traces are required to flat-field the data
if not self._chk_objs(['slits', 'wavetilts']):
# TODO: Why doesn't this fault?
msgs.warn(
'Flats were requested, but there are quantities missing necessary to '
'create flats. Proceeding without flat fielding....')
self.flatimages = flatfield.FlatImages(None, None, None, None)
return
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
illum_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('illumflat')
pixflat_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('pixelflat')
masterframe_filename = masterframe.construct_file_name(
flatfield.FlatImages,
self.master_key_dict['flat'],
master_dir=self.master_dir)
# The following if-elif-else does:
# 1. Try to load a MasterFrame (if reuse_masters is True). If successful, pass it back
# 2. Build from scratch
# 3. Load any user-supplied images to over-ride any built
# Load MasterFrame?
if os.path.isfile(masterframe_filename) and self.reuse_masters:
self.flatimages = flatfield.FlatImages.from_file(
masterframe_filename)
self.flatimages.is_synced(self.slits)
self.slits.mask_flats(self.flatimages)
return self.flatimages
# TODO -- Allow for separate pixelflat and illumflat images
# Generate the image
stacked_flat = None
if len(illum_image_files) > 0:
stacked_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['illumflatframe'],
illum_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
if stacked_flat is None and len(pixflat_image_files) > 0:
stacked_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['pixelflatframe'],
pixflat_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
if stacked_flat is not None:
# Create pixelflat and illumination flat from illumination flat stack
flatField = flatfield.FlatField(stacked_flat, self.spectrograph,
self.par['flatfield'], self.slits,
self.wavetilts)
# Run
self.flatimages = flatField.run(show=self.show)
# Save to Masters
self.flatimages.to_master_file(masterframe_filename)
# Save slits too, in case they were tweaked
self.slits.to_master_file()
else:
self.flatimages = flatfield.FlatImages(None, None, None, None)
# 3) Load user-supplied images
# NOTE: This is the *final* images, not just a stack
# And it will over-ride what is generated below (if generated)
if self.par['flatfield']['pixelflat_file'] is not None:
# - Name is explicitly correct?
# THIS IS DONE IN PYPEITPAR
#if os.path.isfile(self.par['flatfield']['pixelflat_file']):
# flat_file = self.par['flatfield']['pixelflat_file']
#else:
# msgs.error('Could not find user-defined flatfield file: {0}'.format(
# self.par['flatfield']['pixelflat_file']))
# Load
msgs.info('Using user-defined file: {0}'.format('pixelflat_file'))
with fits.open(self.par['flatfield']['pixelflat_file']) as hdu:
self.flatimages.pixelflat = hdu[self.det].data
# Return
return self.flatimages
def get_slits(self):
"""
Load or generate the definition of the slit boundaries.
Internals that must be available are :attr:`fitstbl`,
:attr:`calib_ID`, :attr:`det`.
Returns:
:class:`pypeit.slittrace.SlitTraceSet`: Traces of the
slit edges; also kept internally as :attr:`slits`.
"""
# Check for existing data
if not self._chk_objs(['msbpm']):
return
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
trace_image_files, self.master_key_dict[
'trace'] = self._prep_calibrations('trace')
# Reuse master frame?
slit_masterframe_name = masterframe.construct_file_name(
slittrace.SlitTraceSet,
self.master_key_dict['trace'],
master_dir=self.master_dir)
if os.path.isfile(slit_masterframe_name) and self.reuse_masters:
self.slits = slittrace.SlitTraceSet.from_file(
slit_masterframe_name)
# Reset the bitmask
self.slits.mask = self.slits.mask_init.copy()
else:
# Slits don't exist or we're not resusing them
edge_masterframe_name = masterframe.construct_file_name(
edgetrace.EdgeTraceSet,
self.master_key_dict['trace'],
master_dir=self.master_dir)
# Reuse master frame?
if os.path.isfile(edge_masterframe_name) and self.reuse_masters:
self.edges = edgetrace.EdgeTraceSet.from_file(
edge_masterframe_name)
elif len(trace_image_files) == 0:
msgs.warn("No frametype=trace files to build slits")
return None
else:
# Build the trace image
self.traceImage = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['traceframe'],
trace_image_files,
bias=self.msbias,
bpm=self.msbpm,
dark=self.msdark)
self.edges = edgetrace.EdgeTraceSet(self.traceImage,
self.spectrograph,
self.par['slitedges'],
bpm=self.msbpm,
auto=True)
self.edges.save(edge_masterframe_name,
master_dir=self.master_dir,
master_key=self.master_key_dict['trace'])
# Show the result if requested
if self.show:
self.edges.show(in_ginga=True)
# Get the slits from the result of the edge tracing, delete
# the edges object, and save the slits, if requested
self.slits = self.edges.get_slits()
self.edges = None
self.slits.to_master_file(slit_masterframe_name)
# User mask?
if self.slitspat_num is not None:
self.slits.user_mask(self.det, self.slitspat_num)
return self.slits
def main(args):
import time
import os
import numpy as np
from pypeit.spectrographs.util import load_spectrograph
from pypeit import edgetrace
from pypeit import slittrace
from pypeit.pypeit import PypeIt
from pypeit.images import buildimage
from pypeit import masterframe
from IPython import embed
if args.pypeit_file is not None:
pypeit_file = args.pypeit_file
if not os.path.isfile(pypeit_file):
raise FileNotFoundError('File does not exist: {0}'.format(pypeit_file))
pypeit_file = os.path.abspath(pypeit_file)
redux_path = os.path.abspath(os.path.split(pypeit_file)[0]
if args.redux_path is None else args.redux_path)
rdx = PypeIt(pypeit_file, redux_path=redux_path)
detectors = rdx.par['rdx']['detnum'] if args.detector is None else args.detectors
# Save the spectrograph
spec = rdx.spectrograph
# Get the calibration group to use
group = np.unique(rdx.fitstbl['calib'])[0] if args.group is None else args.group
if group not in np.unique(rdx.fitstbl['calib']):
raise ValueError('Not a valid calibration group: {0}'.format(group))
# Find the rows in the metadata table with trace frames in the
# specified calibration group
tbl_rows = rdx.fitstbl.find_frames('trace', calib_ID=int(group), index=True)
# Master keyword
master_key_base = '_'.join(rdx.fitstbl.master_key(tbl_rows[0]).split('_')[:2])
# Save the binning
binning = rdx.fitstbl['binning'][tbl_rows[0]]
# Save the full file paths
files = rdx.fitstbl.frame_paths(tbl_rows)
# Trace image processing parameters
proc_par = rdx.par['calibrations']['traceframe']
# Slit tracing parameters
trace_par = rdx.par['calibrations']['slitedges']
# Get the bias files, if requested
bias_rows = rdx.fitstbl.find_frames('bias', calib_ID=int(group), index=True)
bias_files = rdx.fitstbl.frame_paths(bias_rows)
bias_par = rdx.par['calibrations']['biasframe']
if len(bias_files) == 0:
bias_files = None
# Set the QA path
qa_path = rdx.qa_path
else:
detectors = args.detector
spec = load_spectrograph(args.spectrograph)
master_key_base = 'A_1'
binning = '1,1' if args.binning is None else args.binning
if not os.path.isfile(args.trace_file):
raise FileNotFoundError('File does not exist: {0}'.format(args.trace_file))
files = [os.path.abspath(args.trace_file)]
redux_path = os.path.abspath(os.path.split(files[0])[0]
if args.redux_path is None else args.redux_path)
par = spec.default_pypeit_par()
proc_par = par['calibrations']['traceframe']
trace_par = par['calibrations']['slitedges']
bias_files = None
bias_par = None
# Set the QA path
qa_path = os.path.join(os.path.abspath(os.path.split(files[0])[0]), 'QA')
if detectors is None:
detectors = np.arange(spec.ndet)+1
if isinstance(detectors, int):
detectors = [detectors]
master_dir = os.path.join(redux_path, args.master_dir)
for det in detectors:
# Master keyword for output file name
master_key = '{0}_{1}'.format(master_key_base, str(det).zfill(2))
# Get the bias frame if requested
if bias_files is None:
proc_par['process']['bias'] = 'skip'
msbias = None
else:
#biasFrame = biasframe.BiasFrame(spec, files=bias_files, det=det, par=bias_par,
# master_key=master_key, master_dir=master_dir)
#msbias = biasFrame.build_image()
msbias = buildimage.buildimage_fromlist(spec, det, bias_par, bias_files)
msbpm = spec.bpm(files[0], det)
# Build the trace image
#traceImage = traceimage.TraceImage(spec, files=files, det=det, par=proc_par, bias=msbias)
#traceImage.build_image(bias=msbias, bpm=msbpm)
traceImage = buildimage.buildimage_fromlist(spec, det, proc_par, files, bias=msbias,
bpm=msbpm)
# Trace the slit edges
t = time.perf_counter()
edges = edgetrace.EdgeTraceSet(traceImage, spec, trace_par, det=det, bpm=msbpm,
auto=True, debug=args.debug, show_stages=args.show,
qa_path=qa_path)
print('Tracing for detector {0} finished in {1} s.'.format(det, time.perf_counter()-t))
# Write the MasterEdges file
edge_masterframe_name = masterframe.construct_file_name(edgetrace.EdgeTraceSet,
master_key,
master_dir=master_dir)
edges.save(edge_masterframe_name, master_dir=master_dir, master_key=master_key)
# Write the MasterSlits file
slit_masterframe_name = masterframe.construct_file_name(slittrace.SlitTraceSet,
master_key, master_dir=master_dir)
edges.get_slits().to_master_file(slit_masterframe_name) #master_dir, master_key, # Naming
#spec.spectrograph) # Header
return 0
def get_align(self):
"""
Load or generate the alignment frame
Requirements:
master_key, det, par
Returns:
ndarray or str: :attr:`align`
"""
# Check for existing data
if not self._chk_objs(['msbpm', 'tslits_dict']):
msgs.error("Don't have all the objects")
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
align_files = self._prep_calibrations('align')
#align_rows = self.fitstbl.find_frames('align', calib_ID=self.calib_ID, index=True)
#self.align_files = self.fitstbl.frame_paths(align_rows)
#self.master_key_dict['align'] \
# = self.fitstbl.master_key(align_rows[0] if len(align_rows) > 0 else self.frame,
# det=self.det)
masterframe_name = masterframe.construct_file_name(
buildimage.AlignImage, self.master_key_dict['align'], master_dir=self.master_dir)
# Previously cahded?
if self._cached('align', self.master_key_dict['align']):
# Previously calculated
self.msalign = self.calib_dict[self.master_key_dict['align']]['align']
elif os.path.isfile(masterframe_name) and self.reuse_masters:
self.msalign = buildimage.AlignImage.from_file(masterframe_name)
else:
# Instantiate with everything needed to generate the image (in case we do)
#self.alignFrame = alignframe.AlignFrame(self.spectrograph, files=self.align_files,
# det=self.det, msbias=self.msbias,
# par=self.par['alignframe'],
# master_key=self.master_key_dict['align'],
# master_dir=self.master_dir,
# reuse_masters=self.reuse_masters)
self.align = buildimage.buildimage_fromlist(self.spectrograph, self.det,
self.par['alignframe'],
align_files, bias=self.msbias, bpm=self.msbpm)
# Load the MasterFrame (if it exists and is desired)?
#self.msalign = self.alignFrame.load()
#if self.msalign is None: # Otherwise build it
# msgs.info("Preparing a master {0:s} frame".format(self.alignFrame.master_type))
# self.msalign = self.alignFrame.build_image(bias=self.msbias, bpm=self.msbpm)
# # Need to set head0 here, since a master align frame loaded from file will have head0 set.
# self.msalign.head0 = self.alignFrame.build_master_header(steps=self.alignFrame.process_steps,
# raw_files=self.alignFrame.file_list)
# # Save to Masters
# Save to Masters
self.msalign.to_master_file(self.master_dir, self.master_key_dict['align'], # Naming
self.spectrograph.spectrograph, # Header
steps=self.msalign.process_steps,
raw_files=align_files)
# Store the alignment frame
self._update_cache('align', 'align', self.msalign)
# Check if the alignment dictionary exists
if self._cached('align_dict', self.master_key_dict['align']) \
and self._cached('wtmask', self.master_key_dict['align']):
self.align_dict = self.calib_dict[self.master_key_dict['align']]['align_dict']
else:
# Extract some header info needed by the algorithm
binning = self.spectrograph.get_meta_value(self.align_files[0], 'binning')
# Instantiate
self.alignment = alignframe.Alignment(self.msalign, self.tslits_dict, self.spectrograph,
self.par['alignment'],
det=self.det, binning=binning,
master_key=self.master_key_dict['align'],
master_dir=self.master_dir,
reuse_masters=self.reuse_masters,
qa_path=self.qa_path, msbpm=self.msbpm)
# Master
self.align_dict = self.alignment.load()
if self.align_dict is None:
self.align_dict = self.alignment.run(self.show)
self.alignment.save()
# Save & return
self._update_cache('align', 'align_dict', self.align_dict)
return self.msalign, self.align_dict
def test_add_missing_slits():
# Load instrument
keck_deimos = load_spectrograph('keck_deimos')
par = keck_deimos.default_pypeit_par()
# working only on detector 1
det = 1
# Built trace image
traceImage = buildimage.buildimage_fromlist(
keck_deimos, det, par['calibrations']['traceframe'],
deimos_flat_files())
msbpm = keck_deimos.bpm(traceImage.files[0], det)
# load specific config parameters
par = keck_deimos.config_specific_par(traceImage.files[0])
trace_par = par['calibrations']['slitedges']
# Running the EdgeTraceSet steps (one-by-one)
# Initialize EdgeTraceSet
edges = EdgeTraceSet(traceImage,
keck_deimos,
trace_par,
bpm=msbpm,
auto=False,
debug=False,
show_stages=False,
qa_path=None)
# Perform the initial edge detection and trace identification
edges.initial_trace(bpm=msbpm)
# Initial trace can result in no edges found
if not edges.is_empty:
# Refine the locations of the trace using centroids of the
# features in the Sobel-filtered image.
edges.centroid_refine()
# Initial trace can result in no edges found, or centroid
# refinement could have removed all traces (via `check_traces`)
if not edges.is_empty:
# Fit the trace locations with a polynomial
edges.fit_refine(debug=False)
# Use the fits to determine if there are any discontinous
# trace centroid measurements that are actually components
# of the same slit edge
edges.merge_traces(debug=False)
# Check if the PCA decomposition is possible; this should catch
# long slits
if edges.par['auto_pca'] and edges.can_pca():
# Use a PCA decomposition to parameterize the trace
# functional forms
edges.pca_refine(debug=False)
# Use the results of the PCA decomposition to rectify and
# detect peaks/troughs in the spectrally collapsed
# Sobel-filtered image, then use those peaks to further
# refine the edge traces
edges.peak_refine(rebuild_pca=True, debug=False)
# Check the values of the traces that will be removed
# Two traces NOT from the same slit
assert round(edges.edge_fit[edges.pca.reference_row, :]
[8]) == 458, 'wrong left trace position'
assert round(edges.edge_fit[edges.pca.reference_row, :]
[-12]) == 1686, 'wrong right trace position'
# Two traces from the same slit
assert round(edges.edge_fit[edges.pca.reference_row, :]
[10]) == 496, 'wrong left trace position'
assert round(edges.edge_fit[edges.pca.reference_row, :]
[11]) == 561, 'wrong right trace position'
# Remove two left traces and two right traces
# NOT form the same slit
indx = np.zeros(edges.ntrace, dtype=bool)
indx[8] = True
indx[-12] = True
# Form the same slit
indx[10] = True
indx[11] = True
edges.remove_traces(edges.synced_selection(indx, mode='ignore'),
rebuild_pca=True)
# Run slitmask matching algorithm
edges.maskdesign_matching(debug=False)
# Sync left and right edges
edges.sync()
# Check the values of the traces that have been recovered
# Two traces NOT from the same slit
assert round(edges.edge_fit[edges.pca.reference_row, :]
[8]) == 459, 'left trace position not recovered'
assert round(edges.edge_fit[edges.pca.reference_row, :]
[-12]) == 1690, 'right trace position not recovered'
# Two traces from the same slit
assert round(edges.edge_fit[edges.pca.reference_row, :]
[10]) == 497, 'wrong left trace position'
assert round(edges.edge_fit[edges.pca.reference_row, :]
[11]) == 561, 'wrong right trace position'
def test_process(kast_blue_bias_files):
# Instantiate
bias_img = buildimage.buildimage_fromlist(shane_kast_blue, 1, frame_par,
kast_blue_bias_files)
assert isinstance(bias_img.image, np.ndarray)