def test_io(kast_blue_bias_files):
#
outfile = masterframe.construct_file_name(buildimage.BiasImage, master_key,
master_dir=master_dir)
if os.path.isfile(outfile):
os.remove(outfile)
# Build
msbias = buildimage.buildimage_fromlist(shane_kast_blue, 1, frame_par,
kast_blue_bias_files)
# Save as a master frame
master_filename = masterframe.construct_file_name(msbias, master_key, master_dir=master_dir)
msbias.to_master_file(master_filename)#master_dir, master_key, # Naming
#shane_kast_blue.spectrograph, # Header
#steps=msbias.process_steps,
#raw_files=kast_blue_bias_files)
assert os.path.isfile(outfile), 'Error writing MasterBias'
# Load master frame
biasImage = buildimage.BiasImage.from_file(outfile)
assert np.array_equal(biasImage.image, msbias.image)
# Instantiate from master frame
#bias_frame2 = biasframe.BiasFrame.from_master_file(bias_frame.master_file_path)
#assert np.array_equal(pypeitImage.image, bias_frame2.pypeitImage.image)
# Clean up
os.remove(outfile)
def test_masterframe_methods():
master_key = 'A_1_DET01'
master_dir = data_path('')
# Filename
Aimg = buildimage.ArcImage(None)
Aimg.PYP_SPEC = 'shane_kast_blue'
filename = masterframe.construct_file_name(Aimg,
master_key,
master_dir=master_dir)
assert isinstance(filename, str)
assert filename == data_path(
f'Master{Aimg.master_type}_{master_key}.{Aimg.master_file_format}')
# Header
hdr = masterframe.build_master_header(Aimg, master_key, master_dir)
assert isinstance(hdr, fits.Header)
# Get those keys!
_master_key, _master_dir = masterframe.grab_key_mdir(hdr)
assert _master_key == master_key
_master_key2, _master_dir2 = masterframe.grab_key_mdir(filename,
from_filename=True)
assert _master_key2 == master_key
def test_masterframe_methods():
master_key = 'A_1_01'
master_dir = data_root()
# Filename
Aimg = buildimage.ArcImage(None)
Aimg.PYP_SPEC = 'shane_kast_blue'
filename = masterframe.construct_file_name(Aimg,
master_key,
master_dir=master_dir)
assert isinstance(filename, str)
assert filename == os.path.join(
master_dir, 'Master' + Aimg.master_type + '_' + master_key + '.' +
Aimg.master_file_format)
# Header
hdr = masterframe.build_master_header(Aimg, master_key, master_dir)
assert isinstance(hdr, fits.Header)
# Get those keys!
_master_key, _master_dir = masterframe.grab_key_mdir(hdr)
assert _master_key == master_key
_master_key2, _master_dir2 = masterframe.grab_key_mdir(filename,
from_filename=True)
assert _master_key2 == master_key
def get_align(self):
"""
Load or generate the alignment frame
Requires: :attr:`slits`, :attr:`det`, :attr:`par`
Returns:
:class:`pypeit.alignframe.Alignments`:
"""
# Check for existing data
if not self._chk_objs(['msbpm', 'slits']):
msgs.error('Must have the bpm and slits to make the alignments!')
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
align_files, self.master_key_dict['align'] = self._prep_calibrations(
'align')
masterframe_filename = masterframe.construct_file_name(
alignframe.Alignments,
self.master_key_dict['align'],
master_dir=self.master_dir)
# Reuse master frame?
if os.path.isfile(masterframe_filename) and self.reuse_masters:
self.alignments = alignframe.Alignments.from_file(
masterframe_filename)
self.alignments.is_synced(self.slits)
return self.alignments
msalign = buildimage.buildimage_fromlist(self.spectrograph,
self.det,
self.par['alignframe'],
align_files,
bias=self.msbias,
bpm=self.msbpm,
dark=self.msdark)
# Extract some header info needed by the algorithm
binning = self.spectrograph.get_meta_value(align_files[0], 'binning')
# Instantiate
# TODO: From JFH: Do we need the bpm here? Check that this was in the previous code.
alignment = alignframe.TraceAlignment(msalign,
self.slits,
self.spectrograph,
self.par['alignment'],
det=self.det,
binning=binning,
qa_path=self.qa_path,
msbpm=self.msbpm)
# Run
self.alignments = alignment.run(show=self.show)
# Save to Masters
self.alignments.to_master_file(masterframe_filename)
return self.alignments
def show(self, slits=None, wcs_match=True):
"""
Simple wrapper to show_flats()
Args:
slits:
wcs_match:
Returns:
"""
illumflat = None
# Try to grab the slits
if slits is None:
# Warning: This parses the filename, not the Header!
master_key, master_dir = masterframe.grab_key_mdir(self.filename, from_filename=True)
try:
slit_masterframe_name = masterframe.construct_file_name(slittrace.SlitTraceSet, master_key,
master_dir=master_dir)
slits = slittrace.SlitTraceSet.from_file(slit_masterframe_name)
except:
msgs.warn('Could not load slits to show with flat-field images. Did you provide the master info??')
if slits is not None:
slits.mask_flats(self)
illumflat = self.fit2illumflat(slits)
# Show
show_flats(self.pixelflat, illumflat, self.procflat, self.flat_model,
wcs_match=wcs_match, slits=slits)
def get_wv_calib(self):
"""
Load or generate the 1D wavelength calibrations
Requirements:
msarc, msbpm, slits, det, par
Returns:
dict: :attr:`wv_calib` calibration dict and the updated slit mask array
"""
# Check for existing data
if not self._chk_objs(['msarc', 'msbpm', 'slits']):
msgs.warn(
'Not enough information to load/generate the wavelength calibration. Skipping and may crash down the line'
)
return None
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
if 'arc' not in self.master_key_dict.keys():
msgs.error('Arc master key not set. First run get_arc.')
# No wavelength calibration requested
if self.par['wavelengths']['reference'] == 'pixel':
msgs.info("A wavelength calibration will not be performed")
self.wv_calib = None
return self.wv_calib
# Grab arc binning (may be different from science!)
# TODO : Do this internally when we have a wv_calib DataContainer
binspec, binspat = parse.parse_binning(self.msarc.detector.binning)
# Instantiate
self.waveCalib = wavecalib.WaveCalib(
self.msarc,
self.slits,
self.spectrograph,
self.par['wavelengths'],
binspectral=binspec,
det=self.det,
master_key=self.master_key_dict['arc'], # For QA naming
qa_path=self.qa_path,
msbpm=self.msbpm)
# Load from disk (MasterFrame)?
masterframe_name = masterframe.construct_file_name(
wavecalib.WaveCalib,
self.master_key_dict['arc'],
master_dir=self.master_dir)
if os.path.isfile(masterframe_name) and self.reuse_masters:
# Load from disk
self.wv_calib = self.waveCalib.load(masterframe_name)
self.slits.mask_wvcalib(self.wv_calib)
else:
self.wv_calib = self.waveCalib.run(skip_QA=(not self.write_qa))
# Save to Masters
self.waveCalib.save(outfile=masterframe_name)
# Return
return self.wv_calib
def get_tilts(self):
"""
Load or generate the tilts image
Requirements:
mstilt, slits, wv_calib
det, par, spectrograph
Returns:
:class:`pypeit.wavetilts.WaveTilts`:
"""
# Check for existing data
#TODO add mstilt_inmask to this list when it gets implemented.
if not self._chk_objs(['mstilt', 'msbpm', 'slits', 'wv_calib']):
msgs.warn(
'dont have all the objects for tilts. Skipping and may crash down the line..'
)
return None
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
if 'tilt' not in self.master_key_dict.keys():
msgs.error('Tilt master key not set. First run get_tiltimage.')
# Load up?
masterframe_name = masterframe.construct_file_name(
wavetilts.WaveTilts,
self.master_key_dict['tilt'],
master_dir=self.master_dir)
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.wavetilts = wavetilts.WaveTilts.from_file(masterframe_name)
self.wavetilts.is_synced(self.slits)
self.slits.mask_wavetilts(self.wavetilts)
else: # Build
# Flexure
_spat_flexure = self.mstilt.spat_flexure \
if self.par['tiltframe']['process']['spat_flexure_correct'] else None
# Instantiate
buildwaveTilts = wavetilts.BuildWaveTilts(
self.mstilt,
self.slits,
self.spectrograph,
self.par['tilts'],
self.par['wavelengths'],
det=self.det,
qa_path=self.qa_path,
master_key=self.master_key_dict['tilt'],
spat_flexure=_spat_flexure)
# TODO still need to deal with syntax for LRIS ghosts. Maybe we don't need it
self.wavetilts = buildwaveTilts.run(doqa=self.write_qa,
show=self.show)
# Save?
self.wavetilts.to_master_file(masterframe_name)
return self.wavetilts
def main(args):
import os
from pypeit.core.gui.skysub_regions import SkySubGUI
from pypeit.core import flexure
from pypeit.scripts import utils
from pypeit import masterframe
from pypeit.images import buildimage
# Generate a utilities class
info = utils.Utilities(None, pypeit_file=args.file, det=args.det)
# Interactively select a science frame
sciIdx = info.select_science_frame(standard=args.standard)
# Load the spectrograph and parset
info.load_spectrograph_parset(sciIdx)
# Get the master key and directory
mdir, mkey = info.get_master_dirkey()
# Load the image data
frame = info.load_frame(sciIdx)
# Load the slits information
slits = utils.get_slits(mkey, mdir)
spat_flexure = None
if args.flexure:
spat_flexure = flexure.spat_flexure_shift(frame, slits)
# Derive an appropriate output filename
file_base = info.get_basename(sciIdx)
prefix = os.path.splitext(file_base)
if prefix[1] == ".gz":
outname = os.path.splitext(prefix[0])[0]
else:
outname = prefix[0]
ext = buildimage.SkyRegions.master_file_format
regfile = masterframe.construct_file_name(buildimage.SkyRegions, master_key=mkey,
master_dir=mdir)
regfile = regfile.replace(".{0:s}".format(ext), "_{0:s}.{1:s}".format(outname, ext))
#outname = "{0:s}/MasterSkyRegions_{1:s}_{2:s}.fits.gz".format(mdir, mkey, outname)
# Finally, initialise the GUI
skyreg = SkySubGUI.initialize(args.det, frame, slits, info.spectrograph.pypeline,
info.spectrograph.name, outname=regfile,
overwrite=args.overwrite, runtime=False, printout=True,
initial=args.initial, flexure=spat_flexure)
# Get the results
skyreg.get_result()
def get_tiltimg(self):
"""
Load or generate the Tilt image
Requirements:
master_key, det, par
Args:
Returns:
`numpy.ndarray`_: :attr:`mstilt` image
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
tilt_files, self.master_key_dict['tilt'] = self._prep_calibrations(
'tilt')
masterframe_name = masterframe.construct_file_name(
buildimage.TiltImage,
self.master_key_dict['tilt'],
master_dir=self.master_dir)
# Reuse master frame?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.mstilt = buildimage.TiltImage.from_file(masterframe_name)
elif len(tilt_files) == 0:
msgs.warn("No frametype=tilt files to build tiltimg")
return
else: # Build
msgs.info("Preparing a master {0:s} frame".format(
buildimage.TiltImage.master_type))
# NOTE: Slits passed for the spatial flexure correction
self.mstilt = buildimage.buildimage_fromlist(self.spectrograph,
self.det,
self.par['tiltframe'],
tilt_files,
bias=self.msbias,
bpm=self.msbpm,
dark=self.msdark,
slits=self.slits)
# Save to Masters
self.mstilt.to_master_file(masterframe_name)
# TODO in the future add in a tilt_inmask
#self._update_cache('tilt', 'tilt_inmask', self.mstilt_inmask)
# Return
return self.mstilt
def test_master_io():
# Instantiate a simple pypeitImage
pypeitImage = pypeitimage.PypeItImage(np.ones((1000, 1000)))
pypeitImage.fullmask = np.zeros((1000, 1000), dtype=np.int64)
pypeitImage.detector = test_detector.detector_container.DetectorContainer(**test_detector.def_det)
pypeitImage.PYP_SPEC = 'shane_kast_blue'
# Now the arcimage
arcImage = buildimage.ArcImage.from_pypeitimage(pypeitImage)
# Write
master_filename = masterframe.construct_file_name(arcImage, 'A_01_22', master_dir=data_path(''))
arcImage.to_master_file(master_filename)
# Read
_arcImage = buildimage.ArcImage.from_file(data_path('MasterArc_A_01_22.fits'))
assert isinstance(_arcImage.detector, test_detector.detector_container.DetectorContainer)
def test_io():
slits = SlitTraceSet(np.full((1000, 3), 2, dtype=float),
np.full((1000, 3), 8, dtype=float),
'MultiSlit',
nspat=10,
PYP_SPEC='dummy')
master_file = masterframe.construct_file_name(slits,
master_key,
master_dir=master_dir)
# Remove any existing file from previous runs that were interrupted
if os.path.isfile(master_file):
os.remove(master_file)
# Try to save it
slits.to_master_file(
master_file) #master_dir, master_key, 'dummy_spectrograph_name')
assert os.path.isfile(master_file), 'File not written'
# Instantiate an empty SlitTraceSet with the same master file, and
# indicate it should be reused
#_slits = SlitTraceSet.from_master('dummy', os.getcwd())
_slits = SlitTraceSet.from_file(master_file)
assert np.array_equal(_slits.left_init, np.full(
(1000, 3), 2, dtype=float)), 'Bad left read'
# And that it's the same as the existing one
assert np.array_equal(_slits.left_init, slits.left_init), 'Bad left read'
# Try to read/write to a custom file name
# Remove existing file from previous runs that were interrupted
other_ofile = 'test.fits.gz'
if os.path.isfile(other_ofile):
os.remove(other_ofile)
# Test write
slits.to_file(other_ofile)
# Test overwrite
slits.to_file(other_ofile, overwrite=True)
# Test from_file
_slits = SlitTraceSet.from_file(other_ofile)
assert np.array_equal(slits.right_init,
_slits.right_init), 'Bad read from_file'
# Clean up
os.remove(master_file)
os.remove(other_ofile)
def test_io(kast_blue_bias_files):
#
outfile = masterframe.construct_file_name(buildimage.BiasImage,
master_key,
master_dir=master_dir)
if os.path.isfile(outfile):
os.remove(outfile)
# Build
msbias = buildimage.buildimage_fromlist(shane_kast_blue, 1, frame_par,
kast_blue_bias_files)
# Save as a master frame
master_filename = masterframe.construct_file_name(msbias,
master_key,
master_dir=master_dir)
msbias.to_master_file(master_filename)
assert os.path.isfile(outfile), 'Error writing MasterBias'
# Load master frame
biasImage = buildimage.BiasImage.from_file(outfile)
assert np.array_equal(biasImage.image, msbias.image), 'Image changed'
assert np.array_equal(biasImage.ivar,
msbias.ivar), 'Inverse-variance changed'
# Clean up
os.remove(outfile)
def get_dark(self):
"""
Load or generate the dark image
Returns:
:class:`~pypeit.images.buildimage.DarkImage`: The combined dark
image.
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
dark_files, self.master_key_dict['dark'] = self._prep_calibrations(
'dark')
# Construct the name, in case we need it
masterframe_name = masterframe.construct_file_name(
buildimage.DarkImage,
self.master_key_dict['dark'],
master_dir=self.master_dir)
# Try to load?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.msdark = buildimage.DarkImage.from_file(masterframe_name)
elif len(dark_files) == 0:
self.msdark = None
else:
# TODO: If a bias has been constructed and it will be subtracted
# from the science images, it should also be subtracted from this
# image. If it isn't, subtracting the dark will effectively lead to
# subtracting the bias twice.
# TODO: The order is such that the bpm doesn't exist yet. But
# calling buildimage_fromlist will create the bpm if it isn't
# passed. So calling get_dark then get_bpm unnecessarily creates
# the bpm twice. Is there any reason why creation of the bpm should
# come after the dark, or can we change the order?
# Build and save it
self.msdark = buildimage.buildimage_fromlist(self.spectrograph,
self.det,
self.par['darkframe'],
dark_files,
bias=self.msbias)
self.msdark.to_master_file(masterframe_name)
# Return
return self.msdark
def get_arc(self):
"""
Load or generate the Arc image
Requirements:
master_key, det, par
Args:
Returns:
`numpy.ndarray`_: :attr:`msarc` image
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
arc_files, self.master_key_dict['arc'] = self._prep_calibrations('arc')
masterframe_name = masterframe.construct_file_name(
buildimage.ArcImage,
self.master_key_dict['arc'],
master_dir=self.master_dir)
# Reuse master frame?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.msarc = buildimage.ArcImage.from_file(masterframe_name)
elif len(arc_files) == 0:
msgs.warn("No frametype=arc files to build arc")
return
else: # Build it
msgs.info("Preparing a master {0:s} frame".format(
buildimage.ArcImage.master_type))
self.msarc = buildimage.buildimage_fromlist(self.spectrograph,
self.det,
self.par['arcframe'],
arc_files,
bias=self.msbias,
bpm=self.msbpm,
dark=self.msdark)
# Save
self.msarc.to_master_file(masterframe_name)
# Return
return self.msarc
def get_bias(self):
"""
Load or generate the bias frame/command
Requirements:
master_key, det, par
Returns:
:class:`pypeit.images.buildimage.BiasImage`:
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
bias_files, self.master_key_dict['bias'] = self._prep_calibrations(
'bias')
# Construct the name, in case we need it
masterframe_name = masterframe.construct_file_name(
buildimage.BiasImage,
self.master_key_dict['bias'],
master_dir=self.master_dir)
if self.par['biasframe']['useframe'] is not None:
msgs.error("Not ready to load from disk")
# Try to load?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.msbias = buildimage.BiasImage.from_file(masterframe_name)
elif len(bias_files) == 0:
self.msbias = None
else:
# Build it
self.msbias = buildimage.buildimage_fromlist(
self.spectrograph, self.det, self.par['biasframe'], bias_files)
# Save it?
self.msbias.to_master_file(masterframe_name)
# Return
return self.msbias
def get_dark(self):
"""
Load or generate the dark image
Requirements:
master_key, det, par
Returns:
:class:`pypeit.images.buildimage.DarkImage`:
"""
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
dark_files, self.master_key_dict['dark'] = self._prep_calibrations(
'dark')
# Construct the name, in case we need it
masterframe_name = masterframe.construct_file_name(
buildimage.DarkImage,
self.master_key_dict['dark'],
master_dir=self.master_dir)
# Try to load?
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.msdark = buildimage.DarkImage.from_file(masterframe_name)
elif len(dark_files) == 0:
self.msdark = None
else:
# TODO: Should this include the bias?
# Build it
self.msdark = buildimage.buildimage_fromlist(
self.spectrograph, self.det, self.par['darkframe'], dark_files)
# Save it?
self.msdark.to_master_file(masterframe_name)
# Return
return self.msdark
def get_slits(self):
"""
Load or generate the definition of the slit boundaries.
Internals that must be available are :attr:`fitstbl`,
:attr:`calib_ID`, :attr:`det`.
Returns:
:class:`pypeit.slittrace.SlitTraceSet`: Traces of the
slit edges; also kept internally as :attr:`slits`.
"""
# Check for existing data
if not self._chk_objs(['msbpm']):
return
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
trace_image_files, self.master_key_dict[
'trace'] = self._prep_calibrations('trace')
# Reuse master frame?
slit_masterframe_name = masterframe.construct_file_name(
slittrace.SlitTraceSet,
self.master_key_dict['trace'],
master_dir=self.master_dir)
if os.path.isfile(slit_masterframe_name) and self.reuse_masters:
self.slits = slittrace.SlitTraceSet.from_file(
slit_masterframe_name)
# Reset the bitmask
self.slits.mask = self.slits.mask_init.copy()
else:
# Slits don't exist or we're not resusing them
edge_masterframe_name = masterframe.construct_file_name(
edgetrace.EdgeTraceSet,
self.master_key_dict['trace'],
master_dir=self.master_dir)
# Reuse master frame?
if os.path.isfile(edge_masterframe_name) and self.reuse_masters:
self.edges = edgetrace.EdgeTraceSet.from_file(
edge_masterframe_name)
elif len(trace_image_files) == 0:
msgs.warn("No frametype=trace files to build slits")
return None
else:
# Build the trace image
self.traceImage = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['traceframe'],
trace_image_files,
bias=self.msbias,
bpm=self.msbpm,
dark=self.msdark)
self.edges = edgetrace.EdgeTraceSet(self.traceImage,
self.spectrograph,
self.par['slitedges'],
bpm=self.msbpm,
auto=True)
self.edges.save(edge_masterframe_name,
master_dir=self.master_dir,
master_key=self.master_key_dict['trace'])
# Show the result if requested
if self.show:
self.edges.show(in_ginga=True)
# Get the slits from the result of the edge tracing, delete
# the edges object, and save the slits, if requested
self.slits = self.edges.get_slits()
self.edges = None
self.slits.to_master_file(slit_masterframe_name)
# User mask?
if self.slitspat_num is not None:
self.slits.user_mask(self.det, self.slitspat_num)
return self.slits
def get_flats(self):
"""
Load or generate a normalized pixel flat and slit illumination
flat.
Requires :attr:`slits`, :attr:`wavetilts`, :attr:`det`,
:attr:`par`.
Constructs :attr:`flatimages`.
"""
# Check for existing data
if not self._chk_objs(['msarc', 'msbpm', 'slits', 'wv_calib']):
msgs.warn(
'Must have the arc, bpm, slits, and wv_calib defined to make flats! Skipping and may crash down the line'
)
self.flatimages = flatfield.FlatImages()
return
# Slit and tilt traces are required to flat-field the data
if not self._chk_objs(['slits', 'wavetilts']):
# TODO: Why doesn't this fault?
msgs.warn(
'Flats were requested, but there are quantities missing necessary to '
'create flats. Proceeding without flat fielding....')
self.flatimages = flatfield.FlatImages()
return
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
illum_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('illumflat')
pixflat_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('pixelflat')
masterframe_filename = masterframe.construct_file_name(
flatfield.FlatImages,
self.master_key_dict['flat'],
master_dir=self.master_dir)
# The following if-elif-else does:
# 1. Try to load a MasterFrame (if reuse_masters is True). If successful, pass it back
# 2. Build from scratch
# 3. Load any user-supplied images to over-ride any built
# Load MasterFrame?
if os.path.isfile(masterframe_filename) and self.reuse_masters:
flatimages = flatfield.FlatImages.from_file(masterframe_filename)
flatimages.is_synced(self.slits)
# Load user defined files
if self.par['flatfield']['pixelflat_file'] is not None:
# Load
msgs.info(
'Using user-defined file: {0}'.format('pixelflat_file'))
with fits.open(self.par['flatfield']['pixelflat_file']) as hdu:
nrm_image = flatfield.FlatImages(
pixelflat_norm=hdu[self.det].data)
flatimages = flatfield.merge(flatimages, nrm_image)
self.flatimages = flatimages
# update slits
self.slits.mask_flats(self.flatimages)
return self.flatimages
# Generate the image
pixelflatImages, illumflatImages = None, None
# Check if the image files are the same
pix_is_illum = Counter(illum_image_files) == Counter(
pixflat_image_files)
if len(pixflat_image_files) > 0:
pixel_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['pixelflatframe'],
pixflat_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
# Initialise the pixel flat
pixelFlatField = flatfield.FlatField(pixel_flat, self.spectrograph,
self.par['flatfield'],
self.slits, self.wavetilts,
self.wv_calib)
# Generate
pixelflatImages = pixelFlatField.run(show=self.show)
# Only build illum_flat if the input files are different from the pixel flat
if (not pix_is_illum) and len(illum_image_files) > 0:
illum_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['illumflatframe'],
illum_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
# Initialise the pixel flat
illumFlatField = flatfield.FlatField(illum_flat,
self.spectrograph,
self.par['flatfield'],
self.slits,
self.wavetilts,
self.wv_calib,
spat_illum_only=True)
# Generate
illumflatImages = illumFlatField.run(show=self.show)
# Merge the illum flat with the pixel flat
if pixelflatImages is not None:
# Combine the pixelflat and illumflat parameters into flatimages.
# This will merge the attributes of pixelflatImages that are not None
# with the attributes of illflatImages that are not None. Default is
# to take pixelflatImages.
flatimages = flatfield.merge(pixelflatImages, illumflatImages)
else:
# No pixel flat, but there might be an illumflat. This will mean that
# the attributes prefixed with 'pixelflat_' will all be None.
flatimages = illumflatImages
# Save flat images
if flatimages is not None:
flatimages.to_master_file(masterframe_filename)
# Save slits too, in case they were tweaked
self.slits.to_master_file()
# 3) Load user-supplied images
# NOTE: This is the *final* images, not just a stack
# And it will over-ride what is generated below (if generated)
if self.par['flatfield']['pixelflat_file'] is not None:
# Load
msgs.info('Using user-defined file: {0}'.format('pixelflat_file'))
with fits.open(self.par['flatfield']['pixelflat_file']) as hdu:
flatimages = flatfield.merge(
flatimages,
flatfield.FlatImages(pixelflat_norm=hdu[self.det].data))
self.flatimages = flatimages
# Return
return self.flatimages
def main(args):
import os
import sys
import astropy.io.fits as fits
from pypeit import masterframe
from pypeit.spectrographs.util import load_spectrograph
from pypeit.core import parse
from pypeit.core import gui
from pypeit.core.wavecal import waveio, templates
from pypeit.wavecalib import WaveCalib
from pypeit import slittrace
from pypeit.images.buildimage import ArcImage
# Load the MasterArc file
if os.path.exists(args.arc_file):
arcfil = args.arc_file
else:
try:
arcfil = "Masters/{0:s}".format(args.arc_file)
except FileNotFoundError:
print("Could not find MasterArc file.")
sys.exit()
msarc = ArcImage.from_file(arcfil)
mdir = msarc.head0['MSTRDIR']
mkey = msarc.head0['MSTRKEY']
# Load the spectrograph
specname = msarc.head0['PYP_SPEC']
spec = load_spectrograph(specname)
par = spec.default_pypeit_par()['calibrations']['wavelengths']
# Get the lamp list
if args.lamps == '':
lamplist = par['lamps']
if lamplist is None:
print("ERROR :: Cannot determine the lamps")
sys.exit()
else:
lamplist = args.lamps.split(",")
par['lamps'] = lamplist
# Load the slits
slits = slittrace.SlitTraceSet.from_file(args.slits_file)
# Reset the mask
slits.mask = slits.mask_init
# Check if a solution exists
solnname = os.path.join(mdir,
masterframe.construct_file_name(WaveCalib, mkey))
wv_calib = waveio.load_wavelength_calibration(solnname) if os.path.exists(
solnname) else None
# Load the MasterFrame (if it exists and is desired)?
wavecal = WaveCalib(msarc,
slits,
spec,
par,
binspectral=slits.binspec,
det=args.det,
master_key=mkey,
msbpm=msarc.fullmask)
arccen, arc_maskslit = wavecal.extract_arcs()
# Launch the identify window
arcfitter = gui.identify.initialise(arccen,
slit=int(args.slit),
par=par,
wv_calib_all=wv_calib,
wavelim=[args.wmin, args.wmax],
nonlinear_counts=spec.nonlinear_counts(
msarc.detector))
final_fit = arcfitter.get_results()
# Ask the user if they wish to store the result in PypeIt calibrations
if final_fit['rms'] < args.rmstol:
ans = ''
while ans != 'y' and ans != 'n':
ans = input(
"Would you like to store this wavelength solution in the archive? (y/n): "
)
if ans == 'y':
gratname = fits.getheader(
msarc.head0['F1'])[spec.meta['dispname']['card']].replace(
"/", "_")
dispangl = "UNKNOWN"
templates.pypeit_identify_record(final_fit, slits.binspec,
specname, gratname, dispangl)
print("Your wavelength solution has been stored")
print("Please consider sending your solution to the PypeIt team!")
else:
print(
"Final fit RMS: {0:0.3f} is larger than the allowed tolerance: {1:0.3f}"
.format(final_fit['rms'], args.rmstol))
print(
"Set the variable --rmstol on the command line to allow a more flexible RMS tolerance"
)
ans = ''
while ans != 'y' and ans != 'n':
ans = input("Would you like to store the line IDs? (y/n): ")
if ans == 'y':
arcfitter.save_IDs()
def calib_all(self, run=True):
"""
Create calibrations for all setups
This will not crash if not all of the standard set of files are not provided
Args:
run (bool, optional): If False, only print the calib names and do
not actually run. Only used with the pypeit_parse_calib_id script
Returns:
dict: A simple dict summarizing the calibration names
"""
calib_dict = {}
self.tstart = time.time()
# Frame indices
frame_indx = np.arange(len(self.fitstbl))
for i in range(self.fitstbl.n_calib_groups):
# 1-indexed calib number
calib_grp = str(i + 1)
# Find all the frames in this calibration group
in_grp = self.fitstbl.find_calib_group(i)
grp_frames = frame_indx[in_grp]
# Find the detectors to reduce
# detectors = PypeIt.select_detectors(detnum=self.par['rdx']['detnum'],
# ndet=self.spectrograph.ndet)
detectors = self.spectrograph.select_detectors(
subset=self.par['rdx']['detnum'])
calib_dict[calib_grp] = {}
# Loop on Detectors
for self.det in detectors:
# Instantiate Calibrations class
self.caliBrate = calibrations.Calibrations.get_instance(
self.fitstbl,
self.par['calibrations'],
self.spectrograph,
self.calibrations_path,
qadir=self.qa_path,
reuse_masters=self.reuse_masters,
show=self.show,
user_slits=slittrace.merge_user_slit(
self.par['rdx']['slitspatnum'],
self.par['rdx']['maskIDs']))
# Do it
# TODO: Why isn't set_config part of the Calibrations.__init__ method?
self.caliBrate.set_config(grp_frames[0], self.det,
self.par['calibrations'])
# Allow skipping the run (e.g. parse_calib_id.py script)
if run:
self.caliBrate.run_the_steps()
key = self.caliBrate.master_key_dict['frame']
calib_dict[calib_grp][key] = {}
for step in self.caliBrate.steps:
if step in ['bpm', 'slits', 'wv_calib', 'tilts', 'flats']:
continue
elif step == 'tiltimg': # Annoying kludge
step = 'tilt'
# Prep
raw_files, self.caliBrate.master_key_dict[
step] = self.caliBrate._prep_calibrations(step)
masterframe_name = masterframe.construct_file_name(
buildimage.frame_image_classes[step],
self.caliBrate.master_key_dict[step],
master_dir=self.caliBrate.master_dir)
# Add to dict
if len(raw_files) > 0:
calib_dict[calib_grp][key][step] = {}
calib_dict[calib_grp][key][step][
'master_key'] = self.caliBrate.master_key_dict[
step]
calib_dict[calib_grp][key][step][
'master_name'] = os.path.basename(masterframe_name)
calib_dict[calib_grp][key][step]['raw_files'] = [
os.path.basename(ifile) for ifile in raw_files
]
# Print the results
print(json.dumps(calib_dict, sort_keys=True, indent=4))
# Write
msgs.info('Writing calib file')
calib_file = self.pypeit_file.replace('.pypeit', '.calib_ids')
ltu.savejson(calib_file, calib_dict, overwrite=True, easy_to_read=True)
# Finish
self.print_end_time()
# Return
return calib_dict
def load_kast_blue_masters(aimg=False,
mstilt=False,
edges=False,
tilts=False,
wvcalib=False,
pixflat=False):
"""
Load up the set of shane_kast_blue master frames
Order is Arc, edges, tilts_dict, wv_calib, pixflat
Args:
get_spectrograph:
aimg:
edges (bool, optional):
Load the slit edges
tilts:
datasec:
wvcalib:
Returns:
list: List of calibration items
"""
spectrograph = load_spectrograph('shane_kast_blue')
spectrograph.naxis = (2112, 350) # Image shape with overscan
master_dir = os.path.join(os.getenv('PYPEIT_DEV'), 'Cooked',
'shane_kast_blue')
reuse_masters = True
# Load up the Masters
ret = []
master_key = 'A_1_01'
if aimg:
arc_file = masterframe.construct_file_name(buildimage.ArcImage,
master_key,
master_dir=master_dir)
AImg = buildimage.ArcImage.from_file(arc_file)
if mstilt:
# We use an arc
arc_file = masterframe.construct_file_name(buildimage.ArcImage,
master_key,
master_dir=master_dir)
AImg = buildimage.ArcImage.from_file(arc_file)
# Convert
mstilt = buildimage.TiltImage.from_pypeitimage(AImg)
ret.append(mstilt)
if edges:
#trace_file = '{0}.gz'.format(os.path.join(master_dir,
# MasterFrame.construct_file_name('Edges', master_key)))
trace_file = masterframe.construct_file_name(edgetrace.EdgeTraceSet,
master_key,
master_dir=master_dir)
ret.append(edgetrace.EdgeTraceSet.from_file(trace_file))
if tilts:
tilts_file = masterframe.construct_file_name(wavetilts.WaveTilts,
master_key,
master_dir=master_dir)
waveTilts = wavetilts.WaveTilts.from_file(tilts_file)
ret.append(waveTilts)
if wvcalib:
#calib_file = os.path.join(master_dir,
# MasterFrame.construct_file_name('WaveCalib', master_key,
# file_format='json'))
calib_file = masterframe.construct_file_name(wavecalib.WaveCalib,
master_key,
master_dir=master_dir)
wv_calib = waveio.load_wavelength_calibration(calib_file)
ret.append(wv_calib)
# Pixelflat
if pixflat:
#calib_file = os.path.join(master_dir,
# MasterFrame.construct_file_name('Flat', master_key))
flat_file = masterframe.construct_file_name(flatfield.FlatImages,
master_key,
master_dir=master_dir)
flatImages = flatfield.FlatImages.from_file(flat_file)
ret.append(flatImages.get_pixelflat())
# Return
return ret
def main(args):
import os
import sys
import numpy as np
from pypeit import msgs
from pypeit import masterframe
from pypeit.spectrographs.util import load_spectrograph
from pypeit.core.gui.identify import Identify
from pypeit.core.wavecal import waveio
from pypeit.wavecalib import BuildWaveCalib, WaveCalib
from pypeit import slittrace
from pypeit.images.buildimage import ArcImage
# Load the MasterArc file
msarc = ArcImage.from_file(args.arc_file)
# Load the spectrograph
spec = load_spectrograph(msarc.PYP_SPEC)
par = spec.default_pypeit_par()['calibrations']['wavelengths']
# Get the lamp list
if args.lamps is None:
lamps = par['lamps']
if lamps is None or lamps == ['use_header']:
msgs.error('Cannot determine the lamps; use --lamps argument')
else:
lamps = args.lamps.split(",")
par['lamps'] = lamps
# Load the slits
slits = slittrace.SlitTraceSet.from_file(args.slits_file)
# Reset the mask
slits.mask = slits.mask_init
# Check if a solution exists
solnname = masterframe.construct_file_name(WaveCalib,
msarc.master_key,
master_dir=msarc.master_dir)
wv_calib = waveio.load_wavelength_calibration(solnname) \
if os.path.exists(solnname) and args.solution else None
# Load the MasterFrame (if it exists and is desired). Bad-pixel mask
# set to any flagged pixel in MasterArc.
wavecal = BuildWaveCalib(msarc,
slits,
spec,
par,
lamps,
binspectral=slits.binspec,
det=args.det,
master_key=msarc.master_key,
msbpm=msarc.select_flag())
arccen, arc_maskslit = wavecal.extract_arcs(slitIDs=[args.slit])
# Launch the identify window
# TODO -- REMOVE THIS HACK
try:
nonlinear_counts = msarc.detector.nonlinear_counts()
except AttributeError:
nonlinear_counts = None
arcfitter = Identify.initialise(arccen,
lamps,
slits,
slit=int(args.slit),
par=par,
wv_calib_all=wv_calib,
wavelim=[args.wmin, args.wmax],
nonlinear_counts=nonlinear_counts,
pxtoler=args.pixtol,
test=args.test,
fwhm=args.fwhm,
sigdetect=args.sigdetect,
specname=spec.name,
y_log=not args.linear)
# Testing?
if args.test:
return arcfitter
final_fit = arcfitter.get_results()
# Build here to avoid circular import
# Note: This needs to be duplicated in test_scripts.py
# Wavecalib (wanted when dealing with multiple detectors, eg. GMOS)
if 'WaveFit' in arcfitter._fitdict.keys():
waveCalib = WaveCalib(
nslits=1,
wv_fits=np.atleast_1d(arcfitter._fitdict['WaveFit']),
arc_spectra=np.atleast_2d(arcfitter.specdata).T,
spat_ids=np.atleast_1d(int(arcfitter._spatid)),
PYP_SPEC=msarc.PYP_SPEC,
lamps=','.join(lamps))
else:
waveCalib = None
# Ask the user if they wish to store the result in PypeIt calibrations
arcfitter.store_solution(final_fit,
slits.binspec,
wvcalib=waveCalib,
rmstol=args.rmstol,
force_save=args.force_save)
def get_align(self):
"""
Load or generate the alignment frame
Requirements:
master_key, det, par
Returns:
ndarray or str: :attr:`align`
"""
# Check for existing data
if not self._chk_objs(['msbpm', 'tslits_dict']):
msgs.error("Don't have all the objects")
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
align_files = self._prep_calibrations('align')
#align_rows = self.fitstbl.find_frames('align', calib_ID=self.calib_ID, index=True)
#self.align_files = self.fitstbl.frame_paths(align_rows)
#self.master_key_dict['align'] \
# = self.fitstbl.master_key(align_rows[0] if len(align_rows) > 0 else self.frame,
# det=self.det)
masterframe_name = masterframe.construct_file_name(
buildimage.AlignImage, self.master_key_dict['align'], master_dir=self.master_dir)
# Previously cahded?
if self._cached('align', self.master_key_dict['align']):
# Previously calculated
self.msalign = self.calib_dict[self.master_key_dict['align']]['align']
elif os.path.isfile(masterframe_name) and self.reuse_masters:
self.msalign = buildimage.AlignImage.from_file(masterframe_name)
else:
# Instantiate with everything needed to generate the image (in case we do)
#self.alignFrame = alignframe.AlignFrame(self.spectrograph, files=self.align_files,
# det=self.det, msbias=self.msbias,
# par=self.par['alignframe'],
# master_key=self.master_key_dict['align'],
# master_dir=self.master_dir,
# reuse_masters=self.reuse_masters)
self.align = buildimage.buildimage_fromlist(self.spectrograph, self.det,
self.par['alignframe'],
align_files, bias=self.msbias, bpm=self.msbpm)
# Load the MasterFrame (if it exists and is desired)?
#self.msalign = self.alignFrame.load()
#if self.msalign is None: # Otherwise build it
# msgs.info("Preparing a master {0:s} frame".format(self.alignFrame.master_type))
# self.msalign = self.alignFrame.build_image(bias=self.msbias, bpm=self.msbpm)
# # Need to set head0 here, since a master align frame loaded from file will have head0 set.
# self.msalign.head0 = self.alignFrame.build_master_header(steps=self.alignFrame.process_steps,
# raw_files=self.alignFrame.file_list)
# # Save to Masters
# Save to Masters
self.msalign.to_master_file(self.master_dir, self.master_key_dict['align'], # Naming
self.spectrograph.spectrograph, # Header
steps=self.msalign.process_steps,
raw_files=align_files)
# Store the alignment frame
self._update_cache('align', 'align', self.msalign)
# Check if the alignment dictionary exists
if self._cached('align_dict', self.master_key_dict['align']) \
and self._cached('wtmask', self.master_key_dict['align']):
self.align_dict = self.calib_dict[self.master_key_dict['align']]['align_dict']
else:
# Extract some header info needed by the algorithm
binning = self.spectrograph.get_meta_value(self.align_files[0], 'binning')
# Instantiate
self.alignment = alignframe.Alignment(self.msalign, self.tslits_dict, self.spectrograph,
self.par['alignment'],
det=self.det, binning=binning,
master_key=self.master_key_dict['align'],
master_dir=self.master_dir,
reuse_masters=self.reuse_masters,
qa_path=self.qa_path, msbpm=self.msbpm)
# Master
self.align_dict = self.alignment.load()
if self.align_dict is None:
self.align_dict = self.alignment.run(self.show)
self.alignment.save()
# Save & return
self._update_cache('align', 'align_dict', self.align_dict)
return self.msalign, self.align_dict
def get_wv_calib(self):
"""
Load or generate the 1D wavelength calibrations
Requirements:
msarc, msbpm, slits, det, par
Returns:
dict: :attr:`wv_calib` calibration dict and the updated slit mask array
"""
# Check for existing data
if not self._chk_objs(['msarc', 'msbpm', 'slits']):
msgs.warn(
'Not enough information to load/generate the wavelength calibration. '
'Skipping and may crash down the line')
return None
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
if 'arc' not in self.master_key_dict.keys():
msgs.error('Arc master key not set. First run get_arc.')
# No wavelength calibration requested
if self.par['wavelengths']['reference'] == 'pixel':
msgs.info("A wavelength calibration will not be performed")
self.wv_calib = None
return self.wv_calib
# Grab arc binning (may be different from science!)
# TODO : Do this internally when we have a wv_calib DataContainer
binspec, binspat = parse.parse_binning(self.msarc.detector.binning)
masterframe_name = masterframe.construct_file_name(
wavecalib.WaveCalib,
self.master_key_dict['arc'],
master_dir=self.master_dir)
if os.path.isfile(masterframe_name) and self.reuse_masters:
self.wv_calib = wavecalib.WaveCalib.from_file(masterframe_name)
self.wv_calib.chk_synced(self.slits)
self.slits.mask_wvcalib(self.wv_calib)
else:
# Determine lamp list to use for wavecalib
# Find all the arc frames in this calibration group
is_arc = self.fitstbl.find_frames('arc', calib_ID=self.calib_ID)
lamps = self.spectrograph.get_lamps(self.fitstbl[is_arc]) \
if self.par['wavelengths']['lamps'] == ['use_header'] else self.par['wavelengths']['lamps']
# Instantiate
self.waveCalib = wavecalib.BuildWaveCalib(
self.msarc,
self.slits,
self.spectrograph,
self.par['wavelengths'],
lamps,
binspectral=binspec,
det=self.det,
master_key=self.master_key_dict['arc'],
qa_path=self.qa_path) #, msbpm=self.msbpm)
self.wv_calib = self.waveCalib.run(skip_QA=(not self.write_qa))
# Save to Masters
self.wv_calib.to_master_file(masterframe_name)
# Return
return self.wv_calib
def main(args):
import time
import os
import numpy as np
from pypeit.spectrographs.util import load_spectrograph
from pypeit import edgetrace
from pypeit import slittrace
from pypeit.pypeit import PypeIt
from pypeit.images import buildimage
from pypeit import masterframe
from IPython import embed
if args.pypeit_file is not None:
pypeit_file = args.pypeit_file
if not os.path.isfile(pypeit_file):
raise FileNotFoundError('File does not exist: {0}'.format(pypeit_file))
pypeit_file = os.path.abspath(pypeit_file)
redux_path = os.path.abspath(os.path.split(pypeit_file)[0]
if args.redux_path is None else args.redux_path)
rdx = PypeIt(pypeit_file, redux_path=redux_path)
detectors = rdx.par['rdx']['detnum'] if args.detector is None else args.detectors
# Save the spectrograph
spec = rdx.spectrograph
# Get the calibration group to use
group = np.unique(rdx.fitstbl['calib'])[0] if args.group is None else args.group
if group not in np.unique(rdx.fitstbl['calib']):
raise ValueError('Not a valid calibration group: {0}'.format(group))
# Find the rows in the metadata table with trace frames in the
# specified calibration group
tbl_rows = rdx.fitstbl.find_frames('trace', calib_ID=int(group), index=True)
# Master keyword
master_key_base = '_'.join(rdx.fitstbl.master_key(tbl_rows[0]).split('_')[:2])
# Save the binning
binning = rdx.fitstbl['binning'][tbl_rows[0]]
# Save the full file paths
files = rdx.fitstbl.frame_paths(tbl_rows)
# Trace image processing parameters
proc_par = rdx.par['calibrations']['traceframe']
# Slit tracing parameters
trace_par = rdx.par['calibrations']['slitedges']
# Get the bias files, if requested
bias_rows = rdx.fitstbl.find_frames('bias', calib_ID=int(group), index=True)
bias_files = rdx.fitstbl.frame_paths(bias_rows)
bias_par = rdx.par['calibrations']['biasframe']
if len(bias_files) == 0:
bias_files = None
# Set the QA path
qa_path = rdx.qa_path
else:
detectors = args.detector
spec = load_spectrograph(args.spectrograph)
master_key_base = 'A_1'
binning = '1,1' if args.binning is None else args.binning
if not os.path.isfile(args.trace_file):
raise FileNotFoundError('File does not exist: {0}'.format(args.trace_file))
files = [os.path.abspath(args.trace_file)]
redux_path = os.path.abspath(os.path.split(files[0])[0]
if args.redux_path is None else args.redux_path)
par = spec.default_pypeit_par()
proc_par = par['calibrations']['traceframe']
trace_par = par['calibrations']['slitedges']
bias_files = None
bias_par = None
# Set the QA path
qa_path = os.path.join(os.path.abspath(os.path.split(files[0])[0]), 'QA')
if detectors is None:
detectors = np.arange(spec.ndet)+1
if isinstance(detectors, int):
detectors = [detectors]
master_dir = os.path.join(redux_path, args.master_dir)
for det in detectors:
# Master keyword for output file name
master_key = '{0}_{1}'.format(master_key_base, str(det).zfill(2))
# Get the bias frame if requested
if bias_files is None:
proc_par['process']['bias'] = 'skip'
msbias = None
else:
#biasFrame = biasframe.BiasFrame(spec, files=bias_files, det=det, par=bias_par,
# master_key=master_key, master_dir=master_dir)
#msbias = biasFrame.build_image()
msbias = buildimage.buildimage_fromlist(spec, det, bias_par, bias_files)
msbpm = spec.bpm(files[0], det)
# Build the trace image
#traceImage = traceimage.TraceImage(spec, files=files, det=det, par=proc_par, bias=msbias)
#traceImage.build_image(bias=msbias, bpm=msbpm)
traceImage = buildimage.buildimage_fromlist(spec, det, proc_par, files, bias=msbias,
bpm=msbpm)
# Trace the slit edges
t = time.perf_counter()
edges = edgetrace.EdgeTraceSet(traceImage, spec, trace_par, det=det, bpm=msbpm,
auto=True, debug=args.debug, show_stages=args.show,
qa_path=qa_path)
print('Tracing for detector {0} finished in {1} s.'.format(det, time.perf_counter()-t))
# Write the MasterEdges file
edge_masterframe_name = masterframe.construct_file_name(edgetrace.EdgeTraceSet,
master_key,
master_dir=master_dir)
edges.save(edge_masterframe_name, master_dir=master_dir, master_key=master_key)
# Write the MasterSlits file
slit_masterframe_name = masterframe.construct_file_name(slittrace.SlitTraceSet,
master_key, master_dir=master_dir)
edges.get_slits().to_master_file(slit_masterframe_name) #master_dir, master_key, # Naming
#spec.spectrograph) # Header
return 0
def get_flats(self):
"""
Load or generate a normalized pixel flat and slit illumination
flat.
Requires :attr:`slits`, :attr:`wavetilts`, :attr:`det`,
:attr:`par`.
Returns:
:class:`pypeit.flatfield.FlatImages`:
"""
# Check for existing data
if not self._chk_objs(['msarc', 'msbpm', 'slits', 'wv_calib']):
msgs.warn(
'Must have the arc, bpm, slits, and wv_calib defined to make flats! Skipping and may crash down the line'
)
self.flatimages = flatfield.FlatImages(None, None, None, None)
return
# Slit and tilt traces are required to flat-field the data
if not self._chk_objs(['slits', 'wavetilts']):
# TODO: Why doesn't this fault?
msgs.warn(
'Flats were requested, but there are quantities missing necessary to '
'create flats. Proceeding without flat fielding....')
self.flatimages = flatfield.FlatImages(None, None, None, None)
return
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
illum_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('illumflat')
pixflat_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('pixelflat')
masterframe_filename = masterframe.construct_file_name(
flatfield.FlatImages,
self.master_key_dict['flat'],
master_dir=self.master_dir)
# The following if-elif-else does:
# 1. Try to load a MasterFrame (if reuse_masters is True). If successful, pass it back
# 2. Build from scratch
# 3. Load any user-supplied images to over-ride any built
# Load MasterFrame?
if os.path.isfile(masterframe_filename) and self.reuse_masters:
self.flatimages = flatfield.FlatImages.from_file(
masterframe_filename)
self.flatimages.is_synced(self.slits)
self.slits.mask_flats(self.flatimages)
return self.flatimages
# TODO -- Allow for separate pixelflat and illumflat images
# Generate the image
stacked_flat = None
if len(illum_image_files) > 0:
stacked_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['illumflatframe'],
illum_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
if stacked_flat is None and len(pixflat_image_files) > 0:
stacked_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['pixelflatframe'],
pixflat_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
if stacked_flat is not None:
# Create pixelflat and illumination flat from illumination flat stack
flatField = flatfield.FlatField(stacked_flat, self.spectrograph,
self.par['flatfield'], self.slits,
self.wavetilts)
# Run
self.flatimages = flatField.run(show=self.show)
# Save to Masters
self.flatimages.to_master_file(masterframe_filename)
# Save slits too, in case they were tweaked
self.slits.to_master_file()
else:
self.flatimages = flatfield.FlatImages(None, None, None, None)
# 3) Load user-supplied images
# NOTE: This is the *final* images, not just a stack
# And it will over-ride what is generated below (if generated)
if self.par['flatfield']['pixelflat_file'] is not None:
# - Name is explicitly correct?
# THIS IS DONE IN PYPEITPAR
#if os.path.isfile(self.par['flatfield']['pixelflat_file']):
# flat_file = self.par['flatfield']['pixelflat_file']
#else:
# msgs.error('Could not find user-defined flatfield file: {0}'.format(
# self.par['flatfield']['pixelflat_file']))
# Load
msgs.info('Using user-defined file: {0}'.format('pixelflat_file'))
with fits.open(self.par['flatfield']['pixelflat_file']) as hdu:
self.flatimages.pixelflat = hdu[self.det].data
# Return
return self.flatimages
def get_slits(mkey, mdir):
slit_masterframe_name = masterframe.construct_file_name(
slittrace.SlitTraceSet, master_key=mkey, master_dir=mdir)
return slittrace.SlitTraceSet.from_file(slit_masterframe_name)
def main(args):
import os
import sys
from pypeit import masterframe
from pypeit.spectrographs.util import load_spectrograph
from pypeit.core.gui.identify import Identify
from pypeit.core.wavecal import waveio
from pypeit.wavecalib import WaveCalib
from pypeit import slittrace
from pypeit.images.buildimage import ArcImage
# Load the MasterArc file
if os.path.exists(args.arc_file):
arcfil = args.arc_file
else:
try:
arcfil = "Masters/{0:s}".format(args.arc_file)
except FileNotFoundError:
print("Could not find MasterArc file.")
sys.exit()
msarc = ArcImage.from_file(arcfil)
mdir = msarc.head0['MSTRDIR']
mkey = msarc.head0['MSTRKEY']
# Load the spectrograph
specname = msarc.head0['PYP_SPEC']
spec = load_spectrograph(specname)
par = spec.default_pypeit_par()['calibrations']['wavelengths']
# Get the lamp list
if args.lamps is None:
lamplist = par['lamps']
if lamplist is None:
print("ERROR :: Cannot determine the lamps")
sys.exit()
else:
lamplist = args.lamps.split(",")
par['lamps'] = lamplist
# Load the slits
slits = slittrace.SlitTraceSet.from_file(args.slits_file)
# Reset the mask
slits.mask = slits.mask_init
# Check if a solution exists
solnname = os.path.join(mdir,
masterframe.construct_file_name(WaveCalib, mkey))
wv_calib = waveio.load_wavelength_calibration(
solnname) if os.path.exists(solnname) and args.solution else None
# Load the MasterFrame (if it exists and is desired)?
wavecal = WaveCalib(msarc,
slits,
spec,
par,
binspectral=slits.binspec,
det=args.det,
master_key=mkey,
msbpm=msarc.fullmask)
arccen, arc_maskslit = wavecal.extract_arcs(slitIDs=[args.slit])
# Launch the identify window
arcfitter = Identify.initialise(arccen,
slits,
slit=int(args.slit),
par=par,
wv_calib_all=wv_calib,
wavelim=[args.wmin, args.wmax],
nonlinear_counts=spec.nonlinear_counts(
msarc.detector))
final_fit = arcfitter.get_results()
# Ask the user if they wish to store the result in PypeIt calibrations
arcfitter.store_solution(final_fit,
mdir,
slits.binspec,
rmstol=args.rmstol,
specname=specname)
def main(args):
import os
import sys
import numpy as np
from pypeit import masterframe
from pypeit.spectrographs.util import load_spectrograph
from pypeit.core.gui.identify import Identify
from pypeit.core.wavecal import waveio
from pypeit.wavecalib import BuildWaveCalib, WaveCalib
from pypeit import slittrace
from pypeit.images.buildimage import ArcImage
# Load the MasterArc file
if os.path.exists(args.arc_file):
arcfil = args.arc_file
else:
try:
arcfil = "Masters/{0:s}".format(args.arc_file)
except FileNotFoundError:
print("Could not find MasterArc file.")
sys.exit()
msarc = ArcImage.from_file(arcfil)
mdir = msarc.head0['MSTRDIR']
mkey = msarc.head0['MSTRKEY']
# Load the spectrograph
specname = msarc.head0['PYP_SPEC']
spec = load_spectrograph(specname)
par = spec.default_pypeit_par()['calibrations']['wavelengths']
# Get the lamp list
if args.lamps is None:
lamplist = par['lamps']
if lamplist is None:
print("ERROR :: Cannot determine the lamps")
sys.exit()
else:
lamplist = args.lamps.split(",")
par['lamps'] = lamplist
# Load the slits
slits = slittrace.SlitTraceSet.from_file(args.slits_file)
# Reset the mask
slits.mask = slits.mask_init
# Check if a solution exists
solnname = os.path.join(mdir,
masterframe.construct_file_name(WaveCalib, mkey))
wv_calib = waveio.load_wavelength_calibration(
solnname) if os.path.exists(solnname) and args.solution else None
# Load the MasterFrame (if it exists and is desired)?
wavecal = BuildWaveCalib(msarc,
slits,
spec,
par,
binspectral=slits.binspec,
det=args.det,
master_key=mkey,
msbpm=msarc.fullmask)
arccen, arc_maskslit = wavecal.extract_arcs(slitIDs=[args.slit])
# Launch the identify window
arcfitter = Identify.initialise(arccen,
slits,
slit=int(args.slit),
par=par,
wv_calib_all=wv_calib,
wavelim=[args.wmin, args.wmax],
nonlinear_counts=spec.nonlinear_counts(
msarc.detector),
pxtoler=args.pixtol,
test=args.test,
fwhm=args.fwhm)
# Testing?
if args.test:
return arcfitter
final_fit = arcfitter.get_results()
# Build here to avoid circular import
# Note: This needs to be duplicated in test_scripts.py
# Wavecalib (wanted when dealing with multiple detectors, eg. GMOS)
if 'WaveFit' in arcfitter._fitdict.keys():
waveCalib = WaveCalib(
nslits=1,
wv_fits=np.atleast_1d(arcfitter._fitdict['WaveFit']),
arc_spectra=np.atleast_2d(arcfitter.specdata).T,
spat_ids=np.atleast_1d(arcfitter._slit),
PYP_SPEC=specname,
)
else:
waveCalib = None
# Ask the user if they wish to store the result in PypeIt calibrations
arcfitter.store_solution(final_fit,
mdir,
slits.binspec,
wvcalib=waveCalib,
rmstol=args.rmstol,
force_save=args.force_save)