def test_run():
# Masters
spectrograph, tslits_dict, tilts_dict, datasec_img \
= load_kast_blue_masters(get_spectrograph=True, tslits=True, tilts=True,
datasec=True)
# Instantiate
frametype = 'pixelflat'
par = pypeitpar.FrameGroupPar(frametype)
flatField = flatfield.FlatField(spectrograph, par, det=1, tilts_dict=tilts_dict,
tslits_dict=tslits_dict.copy())
# Use mstrace
flatField.rawflatimg = tslits_dict['mstrace'].copy()
mspixelflatnrm, msillumflat = flatField.run()
assert np.isclose(np.median(mspixelflatnrm), 1.0)
def test_run():
# Masters
spectrograph = load_spectrograph('shane_kast_blue')
edges, tilts_dict = load_kast_blue_masters(edges=True, tilts=True)
# Instantiate
frametype = 'pixelflat'
par = pypeitpar.FrameGroupPar(frametype)
flatField = flatfield.FlatField(spectrograph,
par,
det=1,
tilts_dict=tilts_dict,
tslits_dict=edges.convert_to_tslits_dict())
# Use the trace image
flatField.rawflatimg = pypeitimage.PypeItImage(edges.img.copy())
mspixelflatnrm, msillumflat = flatField.run()
assert np.isclose(np.median(mspixelflatnrm), 1.0)
masterpath = devpath + '/REDUX_OUT/Keck_LRIS_red/multi_400_8500_d560/MF_keck_lris_red/'
# Read in the msbias for bias subtraction
biasfile = masterpath + 'MasterBias_A_' + sdet + '_aa.fits'
msbias = fits.getdata(biasfile)
# Read in and process flat field images
pixflat_image_files = np.core.defchararray.add(
rawpath,
['r170320_2057.fits', 'r170320_2058.fits', 'r170320_2059.fits'
]).tolist()
spectro_name = 'keck_lris_red'
spectrograph = load_spectrograph(spectrograph=spectro_name)
par = spectrograph.default_pypeit_par()
flatField = flatfield.FlatField(spectrograph,
file_list=pixflat_image_files,
det=det,
par=par['calibrations']['pixelflatframe'],
msbias=msbias)
flatimg = flatField.build_pixflat()
# Read in the tilts
tiltsfile = masterpath + 'MasterTilts_A_' + sdet + '_aa.fits'
mstilts = fits.getdata(tiltsfile)
# Read in the tslits_dict
traceslitsroot = masterpath + 'MasterTrace_A_' + sdet + '_aa'
Tslits = traceslits.TraceSlits.from_master_files(traceslitsroot)
tslits_dict = {}
tslits_dict['lcen'] = Tslits.lcen
tslits_dict['rcen'] = Tslits.rcen
tslits_dict['slitpix'] = pixels.slit_pixels(tslits_dict['lcen'],
tslits_dict['rcen'],
flatimg.shape,
def get_flats(self):
"""
Load or generate a normalized pixel flat and slit illumination
flat.
Requires :attr:`slits`, :attr:`wavetilts`, :attr:`det`,
:attr:`par`.
Constructs :attr:`flatimages`.
"""
# Check for existing data
if not self._chk_objs(['msarc', 'msbpm', 'slits', 'wv_calib']):
msgs.warn(
'Must have the arc, bpm, slits, and wv_calib defined to make flats! Skipping and may crash down the line'
)
self.flatimages = flatfield.FlatImages()
return
# Slit and tilt traces are required to flat-field the data
if not self._chk_objs(['slits', 'wavetilts']):
# TODO: Why doesn't this fault?
msgs.warn(
'Flats were requested, but there are quantities missing necessary to '
'create flats. Proceeding without flat fielding....')
self.flatimages = flatfield.FlatImages()
return
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
illum_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('illumflat')
pixflat_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('pixelflat')
masterframe_filename = masterframe.construct_file_name(
flatfield.FlatImages,
self.master_key_dict['flat'],
master_dir=self.master_dir)
# The following if-elif-else does:
# 1. Try to load a MasterFrame (if reuse_masters is True). If successful, pass it back
# 2. Build from scratch
# 3. Load any user-supplied images to over-ride any built
# Load MasterFrame?
if os.path.isfile(masterframe_filename) and self.reuse_masters:
flatimages = flatfield.FlatImages.from_file(masterframe_filename)
flatimages.is_synced(self.slits)
# Load user defined files
if self.par['flatfield']['pixelflat_file'] is not None:
# Load
msgs.info(
'Using user-defined file: {0}'.format('pixelflat_file'))
with fits.open(self.par['flatfield']['pixelflat_file']) as hdu:
nrm_image = flatfield.FlatImages(
pixelflat_norm=hdu[self.det].data)
flatimages = flatfield.merge(flatimages, nrm_image)
self.flatimages = flatimages
# update slits
self.slits.mask_flats(self.flatimages)
return self.flatimages
# Generate the image
pixelflatImages, illumflatImages = None, None
# Check if the image files are the same
pix_is_illum = Counter(illum_image_files) == Counter(
pixflat_image_files)
if len(pixflat_image_files) > 0:
pixel_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['pixelflatframe'],
pixflat_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
# Initialise the pixel flat
pixelFlatField = flatfield.FlatField(pixel_flat, self.spectrograph,
self.par['flatfield'],
self.slits, self.wavetilts,
self.wv_calib)
# Generate
pixelflatImages = pixelFlatField.run(show=self.show)
# Only build illum_flat if the input files are different from the pixel flat
if (not pix_is_illum) and len(illum_image_files) > 0:
illum_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['illumflatframe'],
illum_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
# Initialise the pixel flat
illumFlatField = flatfield.FlatField(illum_flat,
self.spectrograph,
self.par['flatfield'],
self.slits,
self.wavetilts,
self.wv_calib,
spat_illum_only=True)
# Generate
illumflatImages = illumFlatField.run(show=self.show)
# Merge the illum flat with the pixel flat
if pixelflatImages is not None:
# Combine the pixelflat and illumflat parameters into flatimages.
# This will merge the attributes of pixelflatImages that are not None
# with the attributes of illflatImages that are not None. Default is
# to take pixelflatImages.
flatimages = flatfield.merge(pixelflatImages, illumflatImages)
else:
# No pixel flat, but there might be an illumflat. This will mean that
# the attributes prefixed with 'pixelflat_' will all be None.
flatimages = illumflatImages
# Save flat images
if flatimages is not None:
flatimages.to_master_file(masterframe_filename)
# Save slits too, in case they were tweaked
self.slits.to_master_file()
# 3) Load user-supplied images
# NOTE: This is the *final* images, not just a stack
# And it will over-ride what is generated below (if generated)
if self.par['flatfield']['pixelflat_file'] is not None:
# Load
msgs.info('Using user-defined file: {0}'.format('pixelflat_file'))
with fits.open(self.par['flatfield']['pixelflat_file']) as hdu:
flatimages = flatfield.merge(
flatimages,
flatfield.FlatImages(pixelflat_norm=hdu[self.det].data))
self.flatimages = flatimages
# Return
return self.flatimages
def get_flats(self):
"""
Load or generate a normalized pixel flat and slit illumination
flat.
Requires :attr:`tslits_dict`, :attr:`tilts_dict`, :attr:`det`,
:attr:`par`.
Returns:
`numpy.ndarray`_: Two arrays are returned, the normalized
pixel flat image (:attr:`mspixelflat`) and the slit
illumination flat (:attr:`msillumflat`). If the user
requested the field flattening be skipped
(`FlatFieldPar['method'] == 'skip'`) or if the slit and tilt
traces are not provided, the function returns two None
objects instead.
"""
# Check for existing data
if not self._chk_objs(['msarc', 'msbpm', 'tslits_dict', 'wv_calib']):
msgs.error('dont have all the objects')
if self.par['flatfield']['method'] is 'skip':
# User does not want to flat-field
self.mspixelflat = None
self.msillumflat = None
msgs.warning(
'Parameter calibrations.flatfield.method is set to skip. You are NOT '
'flatfielding your data!!!')
# TODO: Why does this not return unity arrays, like what's
# done below?
return self.mspixelflat, self.msillumflat
# Slit and tilt traces are required to flat-field the data
if not self._chk_objs(['tslits_dict', 'tilts_dict']):
msgs.warning(
'Flats were requested, but there are quantities missing necessary to '
'create flats. Proceeding without flat fielding....')
# User cannot flat-field
self.mspixelflat = None
self.msillumflat = None
# TODO: Why does this not return unity arrays, like what's
# done below?
return self.mspixelflat, self.msillumflat
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
pixflat_rows = self.fitstbl.find_frames('pixelflat',
calib_ID=self.calib_ID,
index=True)
# TODO: Why aren't these set to self
# KBW: They're kept in self.flatField.files
pixflat_image_files = self.fitstbl.frame_paths(pixflat_rows)
# Allow for user-supplied file (e.g. LRISb)
self.master_key_dict['flat'] \
= self.fitstbl.master_key(pixflat_rows[0] if len(pixflat_rows) > 0 else self.frame,
det=self.det)
# Return already generated data
if self._cached('pixelflat', self.master_key_dict['flat']) \
and self._cached('illumflat', self.master_key_dict['flat']):
self.mspixelflat = self.calib_dict[
self.master_key_dict['flat']]['pixelflat']
self.msillumflat = self.calib_dict[
self.master_key_dict['flat']]['illumflat']
return self.mspixelflat, self.msillumflat
# Instantiate
# TODO: This should automatically attempt to load and instatiate
# from a file if it exists.
self.flatField = flatfield.FlatField(
self.spectrograph,
self.par['pixelflatframe'],
files=pixflat_image_files,
det=self.det,
master_key=self.master_key_dict['flat'],
master_dir=self.master_dir,
reuse_masters=self.reuse_masters,
flatpar=self.par['flatfield'],
msbias=self.msbias,
# TODO: msbpm is not passed?
tslits_dict=self.tslits_dict,
tilts_dict=self.tilts_dict)
# --- Pixel flats
# 1) Try to load master files from disk (MasterFrame)?
_, self.mspixelflat, self.msillumflat = self.flatField.load()
# 2) Did the user specify a flat? If so load it in (e.g. LRISb with pixel flat)?
# TODO: We need to document this format for the user!
if self.par['flatfield']['frame'] != 'pixelflat':
# - Name is explicitly correct?
if os.path.isfile(self.par['flatfield']['frame']):
flat_file = self.par['flatfield']['frame']
# - Is it in the master directory?
elif os.path.isfile(
os.path.join(self.flatField.master_dir,
self.par['flatfield']['frame'])):
flat_file = os.path.join(self.flatField.master_dir,
self.par['flatfield']['frame'])
else:
msgs.error(
'Could not find user-defined flatfield file: {0}'.format(
self.par['flatfield']['frame']))
msgs.info('Using user-defined file: {0}'.format(flat_file))
with fits.open(flat_file) as hdu:
self.mspixelflat = hdu[self.det].data
self.msillumflat = None
# 3) there is no master or no user supplied flat, generate the flat
if self.mspixelflat is None and len(pixflat_image_files) != 0:
# Run
self.mspixelflat, self.msillumflat = self.flatField.run(
show=self.show, maskslits=self.tslits_dict['maskslits'])
# If we tweaked the slits, update the tilts_dict and
# tslits_dict to reflect new slit edges
if self.par['flatfield']['tweak_slits']:
msgs.info(
'Using slit boundary tweaks from IllumFlat and updated tilts image'
)
self.tslits_dict = self.flatField.tslits_dict
self.tilts_dict = self.flatField.tilts_dict
# Save to Masters
if self.save_masters:
self.flatField.save()
# If we tweaked the slits update the master files for tilts and slits
# TODO: These should be saved separately
if self.par['flatfield']['tweak_slits']:
msgs.info(
'Updating MasterTrace and MasterTilts using tweaked slit boundaries'
)
# Add tweaked boundaries to the MasterTrace file
self.traceSlits.tslits_dict = self.flatField.tslits_dict
try:
self.traceSlits.save(traceImage=self.traceImage)
except:
self.traceSlits.save(traceImage=self.mstrace)
# Write the final_tilts using the new slit boundaries to the MasterTilts file
self.waveTilts.final_tilts = self.flatField.tilts_dict[
'tilts']
self.waveTilts.tilts_dict = self.flatField.tilts_dict
self.waveTilts.save()
# 4) If either of the two flats are still None, use unity
# everywhere and print out a warning
# TODO: These will barf if self.tilts_dict['tilts'] isn't
# defined.
if self.mspixelflat is None:
self.mspixelflat = np.ones_like(self.tilts_dict['tilts'])
msgs.warn('You are not pixel flat fielding your data!!!')
if self.msillumflat is None:
self.msillumflat = np.ones_like(self.tilts_dict['tilts'])
msgs.warn('You are not illumination flat fielding your data!')
# Save & return
self._update_cache('flat', ('pixelflat', 'illumflat'),
(self.mspixelflat, self.msillumflat))
return self.mspixelflat, self.msillumflat
def get_flats(self):
"""
Load or generate a normalized pixel flat and slit illumination
flat.
Requires :attr:`slits`, :attr:`wavetilts`, :attr:`det`,
:attr:`par`.
Returns:
:class:`pypeit.flatfield.FlatImages`:
"""
# Check for existing data
if not self._chk_objs(['msarc', 'msbpm', 'slits', 'wv_calib']):
msgs.warn(
'Must have the arc, bpm, slits, and wv_calib defined to make flats! Skipping and may crash down the line'
)
self.flatimages = flatfield.FlatImages(None, None, None, None)
return
# Slit and tilt traces are required to flat-field the data
if not self._chk_objs(['slits', 'wavetilts']):
# TODO: Why doesn't this fault?
msgs.warn(
'Flats were requested, but there are quantities missing necessary to '
'create flats. Proceeding without flat fielding....')
self.flatimages = flatfield.FlatImages(None, None, None, None)
return
# Check internals
self._chk_set(['det', 'calib_ID', 'par'])
# Prep
illum_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('illumflat')
pixflat_image_files, self.master_key_dict[
'flat'] = self._prep_calibrations('pixelflat')
masterframe_filename = masterframe.construct_file_name(
flatfield.FlatImages,
self.master_key_dict['flat'],
master_dir=self.master_dir)
# The following if-elif-else does:
# 1. Try to load a MasterFrame (if reuse_masters is True). If successful, pass it back
# 2. Build from scratch
# 3. Load any user-supplied images to over-ride any built
# Load MasterFrame?
if os.path.isfile(masterframe_filename) and self.reuse_masters:
self.flatimages = flatfield.FlatImages.from_file(
masterframe_filename)
self.flatimages.is_synced(self.slits)
self.slits.mask_flats(self.flatimages)
return self.flatimages
# TODO -- Allow for separate pixelflat and illumflat images
# Generate the image
stacked_flat = None
if len(illum_image_files) > 0:
stacked_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['illumflatframe'],
illum_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
if stacked_flat is None and len(pixflat_image_files) > 0:
stacked_flat = buildimage.buildimage_fromlist(
self.spectrograph,
self.det,
self.par['pixelflatframe'],
pixflat_image_files,
dark=self.msdark,
bias=self.msbias,
bpm=self.msbpm)
if stacked_flat is not None:
# Create pixelflat and illumination flat from illumination flat stack
flatField = flatfield.FlatField(stacked_flat, self.spectrograph,
self.par['flatfield'], self.slits,
self.wavetilts)
# Run
self.flatimages = flatField.run(show=self.show)
# Save to Masters
self.flatimages.to_master_file(masterframe_filename)
# Save slits too, in case they were tweaked
self.slits.to_master_file()
else:
self.flatimages = flatfield.FlatImages(None, None, None, None)
# 3) Load user-supplied images
# NOTE: This is the *final* images, not just a stack
# And it will over-ride what is generated below (if generated)
if self.par['flatfield']['pixelflat_file'] is not None:
# - Name is explicitly correct?
# THIS IS DONE IN PYPEITPAR
#if os.path.isfile(self.par['flatfield']['pixelflat_file']):
# flat_file = self.par['flatfield']['pixelflat_file']
#else:
# msgs.error('Could not find user-defined flatfield file: {0}'.format(
# self.par['flatfield']['pixelflat_file']))
# Load
msgs.info('Using user-defined file: {0}'.format('pixelflat_file'))
with fits.open(self.par['flatfield']['pixelflat_file']) as hdu:
self.flatimages.pixelflat = hdu[self.det].data
# Return
return self.flatimages
def load_kast_blue_masters(aimg=False,
tslits=False,
tilts=False,
wvcalib=False,
pixflat=False):
"""
Load up the set of shane_kast_blue master frames
Order is Arc, tslits_dict, tilts_dict, wv_calib, pixflat
Args:
get_spectrograph:
aimg:
tslits (bool, optional):
Load the tslits_dict
tilts:
datasec:
wvcalib:
Returns:
list: List of calibration items
"""
spectrograph = load_spectrograph('shane_kast_blue')
spectrograph.naxis = (2112, 350) # Image shape with overscan
master_dir = os.path.join(os.getenv('PYPEIT_DEV'), 'Cooked',
'Shane_Kast_blue')
# master_dir = root_path+'_'+spectrograph.spectrograph
reuse_masters = True
# Load up the Masters
ret = []
# if get_spectrograph:
# ret.append(spectrograph)
master_key = 'A_1_01'
if aimg:
AImg = arcimage.ArcImage(spectrograph,
master_key=master_key,
master_dir=master_dir,
reuse_masters=reuse_masters)
msarc = AImg.load()
ret.append(msarc)
if tslits:
trace_file = os.path.join(
master_dir, MasterFrame.construct_file_name('Trace', master_key))
tslits_dict, mstrace = traceslits.TraceSlits.load_from_file(trace_file)
ret.append(tslits_dict)
ret.append(mstrace)
if tilts:
tilts_file = os.path.join(
master_dir, MasterFrame.construct_file_name('Tilts', master_key))
tilts_dict = wavetilts.WaveTilts.load_from_file(tilts_file)
ret.append(tilts_dict)
if wvcalib:
calib_file = os.path.join(
master_dir,
MasterFrame.construct_file_name('WaveCalib',
master_key,
file_format='json'))
wv_calib = waveio.load_wavelength_calibration(calib_file)
ret.append(wv_calib)
# Pixelflat
if pixflat:
flatField = flatfield.FlatField(
spectrograph,
spectrograph.default_pypeit_par()['calibrations']
['pixelflatframe'])
calib_file = os.path.join(
master_dir, MasterFrame.construct_file_name('Flat', master_key))
pixelflat = flatField.load_from_file(calib_file, 2)
ret.append(pixelflat)
# Return
return ret