def test_array_information(self):
ad = astrodata.open(os.path.join(TESTDATAPATH, 'GMOS',
'N20110524S0358_varAdded.fits'))
ret = gt.array_information(ad)
assert ret == {'amps_per_array': {1: 1, 2: 1, 3: 1},
'amps_order': [0, 1, 2], 'array_number': [1, 2, 3],
'reference_extension': 2}
def tileArrays(self, adinputs=None, **params):
"""
This primitive combines extensions by tiling (no interpolation).
The array_section() and detector_section() descriptors are used
to derive the geometry of the tiling, so outside help (from the
instrument's geometry_conf module) is only required if there are
multiple arrays being tiled together, as the gaps need to be
specified.
Parameters
----------
suffix: str
suffix to be added to output files
tile_all: bool
tile to a single extension, rather than one per array?
(array=physical detector)
sci_only: bool
tile only the data plane?
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
suffix = params['suffix']
tile_all = params['tile_all']
attributes = ['data'] if params["sci_only"] else None
adoutputs = []
for ad in adinputs:
if len(ad) == 1:
log.warning("{} has only one extension, so there's nothing "
"to tile".format(ad.filename))
adoutputs.append(ad)
continue
# Get information to calculate the output geometry
# TODO: Think about arbitrary ROIs
array_info = gt.array_information(ad)
detshape = array_info.detector_shape
if not tile_all and set(array_info.array_shapes) == {(1, 1)}:
log.warning("{} has nothing to tile, as tile_all=False but "
"each array has only one amplifier.")
adoutputs.append(ad)
continue
blocks = [
Block(ad[arrays], shape=shape) for arrays, shape in zip(
array_info.extensions, array_info.array_shapes)
]
offsets = [
ad[exts[0]].array_section() for exts in array_info.extensions
]
if tile_all and detshape != (1, 1): # We need gaps!
geotable = import_module('.geometry_conf', self.inst_lookups)
chip_gaps = geotable.tile_gaps[ad.detector_name()]
try:
xgap, ygap = chip_gaps
except TypeError: # single number, applies to both
xgap = ygap = chip_gaps
transforms = []
for i, (origin,
offset) in enumerate(zip(array_info.origins, offsets)):
xshift = (origin[1] + offset.x1 + xgap *
(i % detshape[1])) // ad.detector_x_bin()
yshift = (origin[0] + offset.y1 + ygap *
(i // detshape[1])) // ad.detector_y_bin()
transforms.append(
Transform(models.Shift(xshift) & models.Shift(yshift)))
adg = AstroDataGroup(blocks, transforms)
adg.set_reference()
ad_out = adg.transform(attributes=attributes,
process_objcat=True)
else:
# ADG.transform() produces full AD objects so we start with
# the first one, and then append the single extensions created
# by later calls to it.
for i, block in enumerate(blocks):
# Simply create a single tiled array
adg = AstroDataGroup([block])
adg.set_reference()
if i == 0:
ad_out = adg.transform(attributes=attributes,
process_objcat=True)
else:
ad_out.append(
adg.transform(attributes=attributes,
process_objcat=True)[0])
gt.mark_history(ad_out,
primname=self.myself(),
keyword=timestamp_key)
ad_out.orig_filename = ad.filename
ad_out.update_filename(suffix=suffix, strip=True)
adoutputs.append(ad_out)
return adoutputs
def mosaicDetectors(self, adinputs=None, **params):
"""
This primitive does a full mosaic of all the arrays in an AD object.
An appropriate geometry_conf.py module containing geometric information
is required.
Parameters
----------
suffix: str
suffix to be added to output files.
sci_only: bool
mosaic only SCI image data. Default is False
order: int (1-5)
order of spline interpolation
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
suffix = params['suffix']
order = params['order']
attributes = ['data'] if params['sci_only'] else None
geotable = import_module('.geometry_conf', self.inst_lookups)
adoutputs = []
for ad in adinputs:
if ad.phu.get(timestamp_key):
log.warning("No changes will be made to {}, since it has "
"already been processed by mosaicDetectors".format(
ad.filename))
adoutputs.append(ad)
continue
if len(ad) == 1:
log.warning("{} has only one extension, so there's nothing "
"to mosaic".format(ad.filename))
adoutputs.append(ad)
continue
# If there's an overscan section, we must trim it before mosaicking
try:
overscan_kw = ad._keyword_for('overscan_section')
except AttributeError: # doesn't exist for this AD, so carry on
pass
else:
if overscan_kw in ad.hdr:
ad = gt.trim_to_data_section(ad, self.keyword_comments)
# Create the blocks (individual physical detectors)
array_info = gt.array_information(ad)
blocks = [
Block(ad[arrays], shape=shape) for arrays, shape in zip(
array_info.extensions, array_info.array_shapes)
]
offsets = [
ad[exts[0]].array_section() for exts in array_info.extensions
]
detname = ad.detector_name()
xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin()
geometry = geotable.geometry[detname]
default_shape = geometry.get('default_shape')
adg = AstroDataGroup()
for block, origin, offset in zip(blocks, array_info.origins,
offsets):
# Origins are in (x, y) order in LUT
block_geom = geometry[origin[::-1]]
nx, ny = block_geom.get('shape', default_shape)
nx /= xbin
ny /= ybin
shift = block_geom.get('shift', (0, 0))
rot = block_geom.get('rotation', 0.)
mag = block_geom.get('magnification', (1, 1))
transform = Transform()
# Shift the Block's coordinates based on its location within
# the full array, to ensure any rotation takes place around
# the true centre.
if offset.x1 != 0 or offset.y1 != 0:
transform.append(
models.Shift(float(offset.x1) / xbin)
& models.Shift(float(offset.y1) / ybin))
if rot != 0 or mag != (1, 1):
# Shift to centre, do whatever, and then shift back
transform.append(
models.Shift(-0.5 * (nx - 1)) & models.Shift(-0.5 *
(ny - 1)))
if rot != 0:
# Cope with non-square pixels by scaling in one
# direction to make them square before applying the
# rotation, and then reversing that.
if xbin != ybin:
transform.append(
models.Identity(1) & models.Scale(ybin / xbin))
transform.append(models.Rotation2D(rot))
if xbin != ybin:
transform.append(
models.Identity(1) & models.Scale(xbin / ybin))
if mag != (1, 1):
transform.append(
models.Scale(mag[0]) & models.Scale(mag[1]))
transform.append(
models.Shift(0.5 * (nx - 1)) & models.Shift(0.5 *
(ny - 1)))
transform.append(
models.Shift(float(shift[0]) / xbin)
& models.Shift(float(shift[1]) / ybin))
adg.append(block, transform)
adg.set_reference()
ad_out = adg.transform(attributes=attributes,
order=order,
process_objcat=False)
ad_out.orig_filename = ad.filename
gt.mark_history(ad_out,
primname=self.myself(),
keyword=timestamp_key)
ad_out.update_filename(suffix=suffix, strip=True)
adoutputs.append(ad_out)
return adoutputs
def normalizeFlat(self, adinputs=None, **params):
"""
This primitive normalizes a GMOS Longslit spectroscopic flatfield
in a manner similar to that performed by gsflat in Gemini-IRAF.
A cubic spline is fitted along the dispersion direction of each
row, separately for each CCD.
As this primitive is GMOS-specific, we know the dispersion direction
will be along the rows, and there will be 3 CCDs.
For Hamamatsu CCDs, the 21 unbinned columns at each CCD edge are
masked out, following the procedure in gsflat.
TODO: Should we add these in the BPM?
Parameters
----------
suffix: str
suffix to be added to output files
spectral_order: int/str
order of fit in spectral direction
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
# For flexibility, the code is going to pass whatever validated
# parameters it gets (apart from suffix and spectral_order) to
# the spline fitter
spline_kwargs = params.copy()
suffix = spline_kwargs.pop("suffix")
spectral_order = spline_kwargs.pop("spectral_order")
threshold = spline_kwargs.pop("threshold")
# Parameter validation should ensure we get an int or a list of 3 ints
try:
orders = [int(x) for x in spectral_order]
except TypeError:
orders = [spectral_order] * 3
for ad in adinputs:
xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin()
array_info = gt.array_information(ad)
is_hamamatsu = 'Hamamatsu' in ad.detector_name(pretty=True)
ad_tiled = self.tileArrays([ad], tile_all=False)[0]
ad_fitted = astrodata.create(ad.phu)
for ext, order, indices in zip(ad_tiled, orders,
array_info.extensions):
# If the entire row is unilluminated, we want to fit
# the pixels but still keep the edges masked
try:
ext.mask ^= (np.bitwise_and.reduce(ext.mask, axis=1)
& DQ.unilluminated)[:, None]
except TypeError: # ext.mask is None
pass
else:
if is_hamamatsu:
ext.mask[:, :21 // xbin] = 1
ext.mask[:, -21 // xbin:] = 1
fitted_data = np.empty_like(ext.data)
pixels = np.arange(ext.shape[1])
for i, row in enumerate(ext.nddata):
masked_data = np.ma.masked_array(row.data, mask=row.mask)
weights = np.sqrt(
np.where(row.variance > 0, 1. / row.variance, 0.))
spline = astromodels.UnivariateSplineWithOutlierRemoval(
pixels,
masked_data,
order=order,
w=weights,
**spline_kwargs)
fitted_data[i] = spline(pixels)
# Copy header so we have the _section() descriptors
ad_fitted.append(fitted_data, header=ext.hdr)
# Find the largest spline value for each row across all extensions
# and mask pixels below the requested fraction of the peak
row_max = np.array([
ext_fitted.data.max(axis=1) for ext_fitted in ad_fitted
]).max(axis=0)
# Prevent runtime error in division
row_max[row_max == 0] = np.inf
for ext_fitted in ad_fitted:
ext_fitted.mask = np.where(
(ext_fitted.data.T / row_max).T < threshold,
DQ.unilluminated, DQ.good)
for ext_fitted, indices in zip(ad_fitted, array_info.extensions):
tiled_arrsec = ext_fitted.array_section()
for i in indices:
ext = ad[i]
arrsec = ext.array_section()
slice_ = (slice((arrsec.y1 - tiled_arrsec.y1) // ybin,
(arrsec.y2 - tiled_arrsec.y1) // ybin),
slice((arrsec.x1 - tiled_arrsec.x1) // xbin,
(arrsec.x2 - tiled_arrsec.x1) // xbin))
# Suppress warnings to do with fitted_data==0
# (which create NaNs in variance)
with np.errstate(invalid='ignore', divide='ignore'):
ext.divide(ext_fitted.nddata[slice_])
np.nan_to_num(ext.data, copy=False, posinf=0, neginf=0)
np.nan_to_num(ext.variance, copy=False)
# Timestamp and update filename
gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key)
ad.update_filename(suffix=suffix, strip=True)
return adinputs
def QECorrect(self, adinputs=None, **params):
"""
This primitive applies a wavelength-dependent QE correction to
a 2D spectral image, based on the wavelength solution of an
associated processed_arc.
It is only designed to work on FLATs, and therefore unmosaicked data.
Parameters
----------
suffix: str
suffix to be added to output files
arc : {None, AstroData, str}
Arc(s) with distortion map.
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
sfx = params["suffix"]
arc = params["arc"]
# Get a suitable arc frame (with distortion map) for every science AD
if arc is None:
self.getProcessedArc(adinputs, refresh=False)
arc_list = self._get_cal(adinputs, 'processed_arc')
else:
arc_list = arc
for ad, arc in zip(*gt.make_lists(adinputs, arc_list, force_ad=True)):
if ad.phu.get(timestamp_key):
log.warning("No changes will be made to {}, since it has "
"already been processed by QECorrect".
format(ad.filename))
continue
if 'e2v' in ad.detector_name(pretty=True):
log.stdinfo(f"{ad.filename} has the e2v CCDs, so no QE "
"correction is necessary")
continue
if self.timestamp_keys['mosaicDetectors'] in ad.phu:
log.warning(f"{ad.filename} has been processed by mosaic"
"Detectors so QECorrect cannot be run")
continue
# Determines whether to multiply or divide by QE correction
is_flat = 'FLAT' in ad.tags
# If the arc's binning doesn't match, we may still be able to
# fall back to the approximate solution
xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin()
if arc is not None and (arc.detector_x_bin() != xbin or
arc.detector_y_bin() != ybin):
log.warning("Science frame {} and arc {} have different binnings,"
"so cannot use arc".format(ad.filename, arc.filename))
arc = None
# The plan here is to attach the mosaic gWCS to the science frame,
# apply an origin shift to put it in the frame of the arc, and
# then use the arc's WCS to get the wavelength. If there's no arc,
# we just use the science frame's WCS.
# Since we're going to change that WCS, store it for restoration.
original_wcs = [ext.wcs for ext in ad]
try:
transform.add_mosaic_wcs(ad, geotable)
except ValueError:
log.warning(f"{ad.filename} already has a 'mosaic' coordinate"
"frame. This is unexpected but I'll continue.")
if arc is None:
if 'sq' in self.mode:
raise OSError(f"No processed arc listed for {ad.filename}")
else:
log.warning(f"No arc supplied for {ad.filename}")
else:
# OK, we definitely want to try to do this, get a wavelength solution
if self.timestamp_keys['determineWavelengthSolution'] not in arc.phu:
msg = f"Arc {arc.filename} (for {ad.filename} has not been wavelength calibrated."
if 'sq' in self.mode:
raise IOError(msg)
else:
log.warning(msg)
# We'll be modifying this
arc_wcs = deepcopy(arc[0].wcs)
if 'distortion_corrected' not in arc_wcs.available_frames:
msg = f"Arc {arc.filename} (for {ad.filename}) has no distortion model."
if 'sq' in self.mode:
raise OSError(msg)
else:
log.warning(msg)
# NB. At this point, we could have an arc that has no good
# wavelength solution nor distortion correction. But we will
# use its WCS rather than the science frame's because it must
# have been supplied by the user.
# This is GMOS so no need to be as generic as distortionCorrect
ad_detsec = ad.detector_section()
arc_detsec = arc.detector_section()[0]
if (ad_detsec[0].x1, ad_detsec[-1].x2) != (arc_detsec.x1, arc_detsec.x2):
raise ValueError("I don't know how to process the "
f"offsets between {ad.filename} "
f"and {arc.filename}")
yoff1 = arc_detsec.y1 - ad_detsec[0].y1
yoff2 = arc_detsec.y2 - ad_detsec[0].y2
arc_ext_shapes = [(ext.shape[0] - yoff1 + yoff2,
ext.shape[1]) for ext in ad]
arc_corners = np.concatenate([transform.get_output_corners(
ext.wcs.get_transform(ext.wcs.input_frame, 'mosaic'),
input_shape=arc_shape, origin=(yoff1, 0))
for ext, arc_shape in zip(ad, arc_ext_shapes)], axis=1)
arc_origin = tuple(np.ceil(min(corners)) for corners in arc_corners)
# So this is what was applied to the ARC to get the
# mosaic frame to its pixel frame, in which the distortion
# correction model was calculated. Convert coordinates
# from python order to Model order.
origin_shift = reduce(Model.__and__, [models.Shift(-origin)
for origin in arc_origin[::-1]])
arc_wcs.insert_transform(arc_wcs.input_frame, origin_shift, after=True)
array_info = gt.array_information(ad)
if array_info.detector_shape == (1, 3):
ccd2_indices = array_info.extensions[1]
else:
raise ValueError(f"{ad.filename} does not have 3 separate detectors")
for index, ext in enumerate(ad):
if index in ccd2_indices:
continue
# Use the WCS in the extension if we don't have an arc,
# otherwise use the arc's mosaic->world transformation
if arc is None:
trans = ext.wcs.forward_transform
else:
trans = (ext.wcs.get_transform(ext.wcs.input_frame, 'mosaic') |
arc_wcs.forward_transform)
ygrid, xgrid = np.indices(ext.shape)
# TODO: want with_units
waves = trans(xgrid, ygrid)[0] * u.nm # Wavelength always axis 0
try:
qe_correction = qeModel(ext)((waves / u.nm).to(u.dimensionless_unscaled).value).astype(np.float32)
except TypeError: # qeModel() returns None
msg = "No QE correction found for {}:{}".format(ad.filename, ext.hdr['EXTVER'])
if 'sq' in self.mode:
raise ValueError(msg)
else:
log.warning(msg)
log.stdinfo("Mean relative QE of EXTVER {} is {:.5f}".
format(ext.hdr['EXTVER'], qe_correction.mean()))
if not is_flat:
qe_correction = 1. / qe_correction
qe_correction[qe_correction < 0] = 0
qe_correction[qe_correction > 10] = 0
ext.multiply(qe_correction)
for ext, orig_wcs in zip(ad, original_wcs):
ext.wcs = orig_wcs
# Timestamp and update the filename
gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key)
ad.update_filename(suffix=sfx, strip=True)
return adinputs
def tileArrays(self, adinputs=None, **params):
"""
This primitive combines extensions by tiling (no interpolation).
The array_section() and detector_section() descriptors are used
to derive the geometry of the tiling, so outside help (from the
instrument's geometry_conf module) is only required if there are
multiple arrays being tiled together, as the gaps need to be
specified.
Parameters
----------
suffix: str
suffix to be added to output files
tile_all: bool
tile to a single extension, rather than one per array?
(array=physical detector)
sci_only: bool
tile only the data plane?
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
suffix = params['suffix']
tile_all = params['tile_all']
attributes = ['data'] if params["sci_only"] else None
adoutputs = []
for ad in adinputs:
if len(ad) == 1:
log.warning("{} has only one extension, so there's nothing "
"to tile".format(ad.filename))
adoutputs.append(ad)
continue
# Get information to calculate the output geometry
# TODO: Think about arbitrary ROIs
array_info = gt.array_information(ad)
detshape = array_info.detector_shape
if not tile_all and set(array_info.array_shapes) == {(1, 1)}:
log.warning("{} has nothing to tile, as tile_all=False but "
"each array has only one amplifier.")
adoutputs.append(ad)
continue
blocks = [Block(ad[arrays], shape=shape) for arrays, shape in
zip(array_info.extensions, array_info.array_shapes)]
offsets = [ad[exts[0]].array_section()
for exts in array_info.extensions]
if tile_all and detshape != (1, 1): # We need gaps!
geotable = import_module('.geometry_conf', self.inst_lookups)
chip_gaps = geotable.tile_gaps[ad.detector_name()]
try:
xgap, ygap = chip_gaps
except TypeError: # single number, applies to both
xgap = ygap = chip_gaps
transforms = []
for i, (origin, offset) in enumerate(zip(array_info.origins, offsets)):
xshift = (origin[1] + offset.x1 + xgap * (i % detshape[1])) // ad.detector_x_bin()
yshift = (origin[0] + offset.y1 + ygap * (i // detshape[1])) // ad.detector_y_bin()
transforms.append(Transform(models.Shift(xshift) & models.Shift(yshift)))
adg = AstroDataGroup(blocks, transforms)
adg.set_reference()
ad_out = adg.transform(attributes=attributes, process_objcat=True)
else:
# ADG.transform() produces full AD objects so we start with
# the first one, and then append the single extensions created
# by later calls to it.
for i, block in enumerate(blocks):
# Simply create a single tiled array
adg = AstroDataGroup([block])
adg.set_reference()
if i == 0:
ad_out = adg.transform(attributes=attributes,
process_objcat=True)
else:
ad_out.append(adg.transform(attributes=attributes,
process_objcat=True)[0])
gt.mark_history(ad_out, primname=self.myself(), keyword=timestamp_key)
ad_out.orig_filename = ad.filename
ad_out.update_filename(suffix=suffix, strip=True)
adoutputs.append(ad_out)
return adoutputs
def QECorrect(self, adinputs=None, **params):
"""
This primitive applies a wavelength-dependent QE correction to
a 2D spectral image, based on the wavelength solution of an
associated processed_arc.
It is only designed to work on FLATs, and therefore unmosaicked data.
Parameters
----------
suffix: str
suffix to be added to output files
arc : {None, AstroData, str}
Arc(s) with distortion map.
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
sfx = params["suffix"]
arc = params["arc"]
use_iraf = params["use_iraf"]
do_cal = params["do_cal"]
if do_cal == 'skip':
log.warning("QE correction has been turned off.")
return adinputs
# Get a suitable arc frame (with distortion map) for every science AD
if arc is None:
arc_list = self.caldb.get_processed_arc(adinputs)
else:
arc_list = (arc, None)
# Provide an arc AD object for every science frame, and an origin
for ad, arc, origin in zip(
*gt.make_lists(adinputs, *arc_list, force_ad=(1, ))):
if ad.phu.get(timestamp_key):
log.warning(f"{ad.filename}: already processed by QECorrect. "
"Continuing.")
continue
if 'e2v' in ad.detector_name(pretty=True):
log.stdinfo(f"{ad.filename} has the e2v CCDs, so no QE "
"correction is necessary")
continue
if self.timestamp_keys['mosaicDetectors'] in ad.phu:
log.warning(f"{ad.filename} has been processed by mosaic"
"Detectors so QECorrect cannot be run")
continue
# Determines whether to multiply or divide by QE correction
is_flat = 'FLAT' in ad.tags
# If the arc's binning doesn't match, we may still be able to
# fall back to the approximate solution
xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin()
if arc is not None and (arc.detector_x_bin() != xbin
or arc.detector_y_bin() != ybin):
log.warning("Science frame and arc have different binnings.")
arc = None
# The plan here is to attach the mosaic gWCS to the science frame,
# apply an origin shift to put it in the frame of the arc, and
# then use the arc's WCS to get the wavelength. If there's no arc,
# we just use the science frame's WCS.
# Since we're going to change that WCS, store it for restoration.
original_wcs = [ext.wcs for ext in ad]
try:
transform.add_mosaic_wcs(ad, geotable)
except ValueError:
log.warning(f"{ad.filename} already has a 'mosaic' coordinate"
"frame. This is unexpected but I'll continue.")
if arc is None:
if 'sq' in self.mode or do_cal == 'force':
raise OSError(f"No processed arc listed for {ad.filename}")
else:
log.warning(f"{ad.filename}: no arc was specified. Using "
"wavelength solution in science frame.")
else:
# OK, we definitely want to try to do this, get a wavelength solution
origin_str = f" (obtained from {origin})" if origin else ""
log.stdinfo(f"{ad.filename}: using the arc {arc.filename}"
f"{origin_str}")
if self.timestamp_keys[
'determineWavelengthSolution'] not in arc.phu:
msg = f"Arc {arc.filename} (for {ad.filename} has not been wavelength calibrated."
if 'sq' in self.mode or do_cal == 'force':
raise IOError(msg)
else:
log.warning(msg)
# We'll be modifying this
arc_wcs = deepcopy(arc[0].wcs)
if 'distortion_corrected' not in arc_wcs.available_frames:
msg = f"Arc {arc.filename} (for {ad.filename}) has no distortion model."
if 'sq' in self.mode or do_cal == 'force':
raise OSError(msg)
else:
log.warning(msg)
# NB. At this point, we could have an arc that has no good
# wavelength solution nor distortion correction. But we will
# use its WCS rather than the science frame's because it must
# have been supplied by the user.
# This is GMOS so no need to be as generic as distortionCorrect
ad_detsec = ad.detector_section()
arc_detsec = arc.detector_section()[0]
if (ad_detsec[0].x1, ad_detsec[-1].x2) != (arc_detsec.x1,
arc_detsec.x2):
raise ValueError("Cannot process the offsets between "
f"{ad.filename} and {arc.filename}")
yoff1 = arc_detsec.y1 - ad_detsec[0].y1
yoff2 = arc_detsec.y2 - ad_detsec[0].y2
arc_ext_shapes = [(ext.shape[0] - yoff1 + yoff2, ext.shape[1])
for ext in ad]
arc_corners = np.concatenate([
transform.get_output_corners(ext.wcs.get_transform(
ext.wcs.input_frame, 'mosaic'),
input_shape=arc_shape,
origin=(yoff1, 0))
for ext, arc_shape in zip(ad, arc_ext_shapes)
],
axis=1)
arc_origin = tuple(
np.ceil(min(corners)) for corners in arc_corners)
# So this is what was applied to the ARC to get the
# mosaic frame to its pixel frame, in which the distortion
# correction model was calculated. Convert coordinates
# from python order to Model order.
origin_shift = reduce(
Model.__and__,
[models.Shift(-origin) for origin in arc_origin[::-1]])
arc_wcs.insert_transform(arc_wcs.input_frame,
origin_shift,
after=True)
array_info = gt.array_information(ad)
if array_info.detector_shape == (1, 3):
ccd2_indices = array_info.extensions[1]
else:
raise ValueError(
f"{ad.filename} does not have 3 separate detectors")
for index, ext in enumerate(ad):
if index in ccd2_indices:
continue
# Use the WCS in the extension if we don't have an arc,
# otherwise use the arc's mosaic->world transformation
if arc is None:
trans = ext.wcs.forward_transform
else:
trans = (ext.wcs.get_transform(ext.wcs.input_frame,
'mosaic')
| arc_wcs.forward_transform)
ygrid, xgrid = np.indices(ext.shape)
# TODO: want with_units
waves = trans(xgrid,
ygrid)[0] * u.nm # Wavelength always axis 0
# Tapering required to prevent QE correction from blowing up
# at the extremes (remember, this is a ratio, not the actual QE)
# We use half-Gaussians to taper
taper = np.ones_like(ext.data)
taper_locut, taper_losig = 350 * u.nm, 25 * u.nm
taper_hicut, taper_hisig = 1200 * u.nm, 200 * u.nm
taper[waves < taper_locut] = np.exp(-(
(waves[waves < taper_locut] - taper_locut) /
taper_losig)**2)
taper[waves > taper_hicut] = np.exp(-(
(waves[waves > taper_hicut] - taper_hicut) /
taper_hisig)**2)
try:
qe_correction = (qeModel(ext, use_iraf=use_iraf)(
(waves / u.nm).to(u.dimensionless_unscaled).value).
astype(np.float32) - 1) * taper + 1
except TypeError: # qeModel() returns None
msg = f"No QE correction found for {ad.filename} extension {ext.id}"
if 'sq' in self.mode:
raise ValueError(msg)
else:
log.warning(msg)
continue
log.stdinfo(f"Mean relative QE of extension {ext.id} is "
f"{qe_correction.mean():.5f}")
if not is_flat:
qe_correction = 1. / qe_correction
ext.multiply(qe_correction)
for ext, orig_wcs in zip(ad, original_wcs):
ext.wcs = orig_wcs
# Timestamp and update the filename
gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key)
ad.update_filename(suffix=sfx, strip=True)
return adinputs
def mosaicDetectors(self, adinputs=None, **params):
"""
This primitive does a full mosaic of all the arrays in an AD object.
An appropriate geometry_conf.py module containing geometric information
is required.
Parameters
----------
suffix: str
suffix to be added to output files.
sci_only: bool
mosaic only SCI image data. Default is False
order: int (1-5)
order of spline interpolation
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
suffix = params['suffix']
order = params['order']
attributes = ['data'] if params['sci_only'] else None
geotable = import_module('.geometry_conf', self.inst_lookups)
adoutputs = []
for ad in adinputs:
if ad.phu.get(timestamp_key):
log.warning("No changes will be made to {}, since it has "
"already been processed by mosaicDetectors".
format(ad.filename))
adoutputs.append(ad)
continue
if len(ad) == 1:
log.warning("{} has only one extension, so there's nothing "
"to mosaic".format(ad.filename))
adoutputs.append(ad)
continue
# If there's an overscan section, we must trim it before mosaicking
try:
overscan_kw = ad._keyword_for('overscan_section')
except AttributeError: # doesn't exist for this AD, so carry on
pass
else:
if overscan_kw in ad.hdr:
ad = gt.trim_to_data_section(ad, self.keyword_comments)
# Create the blocks (individual physical detectors)
array_info = gt.array_information(ad)
blocks = [Block(ad[arrays], shape=shape) for arrays, shape in
zip(array_info.extensions, array_info.array_shapes)]
offsets = [ad[exts[0]].array_section()
for exts in array_info.extensions]
detname = ad.detector_name()
xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin()
geometry = geotable.geometry[detname]
default_shape = geometry.get('default_shape')
adg = AstroDataGroup()
for block, origin, offset in zip(blocks, array_info.origins, offsets):
# Origins are in (x, y) order in LUT
block_geom = geometry[origin[::-1]]
nx, ny = block_geom.get('shape', default_shape)
nx /= xbin
ny /= ybin
shift = block_geom.get('shift', (0, 0))
rot = block_geom.get('rotation', 0.)
mag = block_geom.get('magnification', (1, 1))
transform = Transform()
# Shift the Block's coordinates based on its location within
# the full array, to ensure any rotation takes place around
# the true centre.
if offset.x1 != 0 or offset.y1 != 0:
transform.append(models.Shift(float(offset.x1) / xbin) &
models.Shift(float(offset.y1) / ybin))
if rot != 0 or mag != (1, 1):
# Shift to centre, do whatever, and then shift back
transform.append(models.Shift(-0.5*(nx-1)) &
models.Shift(-0.5*(ny-1)))
if rot != 0:
# Cope with non-square pixels by scaling in one
# direction to make them square before applying the
# rotation, and then reversing that.
if xbin != ybin:
transform.append(models.Identity(1) & models.Scale(ybin / xbin))
transform.append(models.Rotation2D(rot))
if xbin != ybin:
transform.append(models.Identity(1) & models.Scale(xbin / ybin))
if mag != (1, 1):
transform.append(models.Scale(mag[0]) &
models.Scale(mag[1]))
transform.append(models.Shift(0.5*(nx-1)) &
models.Shift(0.5*(ny-1)))
transform.append(models.Shift(float(shift[0]) / xbin) &
models.Shift(float(shift[1]) / ybin))
adg.append(block, transform)
adg.set_reference()
ad_out = adg.transform(attributes=attributes, order=order,
process_objcat=False)
ad_out.orig_filename = ad.filename
gt.mark_history(ad_out, primname=self.myself(), keyword=timestamp_key)
ad_out.update_filename(suffix=suffix, strip=True)
adoutputs.append(ad_out)
return adoutputs
def display(self, adinputs=None, **params):
"""
Displays an image on the ds9 display, using multiple frames if
there are multiple extensions. Saturated pixels can be displayed
in red, and overlays can also be shown.
Parameters
----------
extname: str
'SCI', 'VAR', or 'DQ': plane to display
frame: int
starting frame for display
ignore: bool
setting to True turns off the display
remove_bias: bool
attempt to subtract bias before displaying?
threshold: str='auto'/float
level above which to flag pixels as saturated
tile: bool
attempt to tile arrays before displaying?
zscale: bool
use zscale algorithm?
overlay: list
list of overlays for the display
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
# No-op if ignore=True
if params["ignore"]:
log.warning("display turned off per user request")
return
threshold = params['threshold']
remove_bias = params.get('remove_bias', False)
extname = params['extname']
tile = params['tile']
zscale = params['zscale']
overlays = params['overlay']
frame = params['frame'] if params['frame'] else 1
overlay_index = 0
lnd = _localNumDisplay()
if isinstance(overlays, str):
try:
overlays = _read_overlays_from_file(overlays)
except OSError:
log.warning(f"Cannot open overlays file {overlays}")
overlays = None
for ad in adinputs:
# Allows elegant break from nested loops
if frame > 16:
log.warning("Too many images; only the first 16 are displayed")
break
# Threshold and bias make sense only for SCI extension
if extname != 'SCI':
threshold = None
remove_bias = False
elif threshold == 'None':
threshold = None
elif threshold == 'auto':
mosaicked = ((ad.phu.get(
self.timestamp_keys["mosaicDetectors"]) is not None)
or (ad.phu.get(self.timestamp_keys["tileArrays"])
is not None))
has_dq = all([ext.mask is not None for ext in ad])
if not has_dq:
if mosaicked:
log.warning("Cannot add DQ to mosaicked data; no "
"threshold mask will be applied to "
"{}".format(ad.filename))
threshold = None
else:
# addDQ operates in place so deepcopy to preserve input
ad = self.addDQ([deepcopy(ad)])[0]
if remove_bias:
if (ad.phu.get('BIASIM') or ad.phu.get('DARKIM') or ad.phu.get(
self.timestamp_keys["subtractOverscan"])):
log.fullinfo("Bias level has already been removed from "
"data; no approximate correction will be "
"performed")
else:
try:
bias_level = get_bias_level(ad)
except NotImplementedError:
# For non-GMOS instruments
bias_level = None
if bias_level is not None:
ad = deepcopy(ad) # Leave original untouched!
log.stdinfo("Subtracting approximate bias level from "
"{} for display".format(ad.filename))
log.fullinfo("Bias levels used: {}".format(
str(bias_level)))
for ext, bias in zip(ad, bias_level):
ext.subtract(
np.float32(bias) if bias is not None else 0)
else:
log.warning("Bias level not found for {}; approximate "
"bias will not be removed".format(
ad.filename))
# Check whether data needs to be tiled before displaying
# Otherwise, flatten all desired extensions into a single list
num_ext = len(ad)
if tile and num_ext > 1:
log.fullinfo("Tiling extensions together before displaying")
# post-transform metadata is arranged in order of blocks, not
# slices, so we need to ensure the correct offsets are applied
# to each slice
array_info = gt.array_information(ad)
ad = self.tileArrays([ad], tile_all=True)[0]
# Logic here in case num_ext overlays sent to be applied to all ADs
if overlays and len(overlays) + overlay_index >= num_ext:
new_overlay = []
trans_data = ad.nddata[0].meta.pop("transform")
for ext_indices, corner, block in zip(
array_info.extensions, trans_data["corners"],
trans_data["block_corners"]):
xshift = int(round(corner[1][0]))
yshift = int(round(corner[0][0]))
for ext_index, b in zip(ext_indices, block):
dx, dy = xshift + b[1], yshift + b[0]
i = overlay_index + ext_index
if overlays[i]:
new_overlay.extend([(x + dx, y + dy, r)
for x, y, r in overlays[i]
])
overlays = (overlays[:overlay_index] + (new_overlay, ) +
overlays[overlay_index + num_ext:])
# Each extension is an individual display item (if the data have been
# tiled, then there'll only be one extension per AD, of course)
for ext in ad:
if frame > 16:
break
# Squeeze the data to remove any empty dimensions (eg, raw F2 data)
ext.operate(np.squeeze)
# Get the data we're going to display. TODO Replace extname with attr?
data = getattr(ext, {
'SCI': 'data',
'DQ': 'mask',
'VAR': 'variance'
}[extname], None)
dqdata = ext.mask
if data is None:
log.warning("No data to display in {}[{}]".format(
ext.filename, extname))
continue
# One-dimensional data (ie, extracted spectra)
if len(data.shape) == 1:
continue
# Make threshold mask if desired
masks = []
mask_colors = []
if threshold is not None:
if threshold != 'auto':
satmask = data > threshold
else:
if dqdata is None:
log.warning("No DQ plane found; cannot make "
"threshold mask")
satmask = None
else:
satmask = (dqdata &
(DQ.non_linear | DQ.saturated)) > 0
if satmask is not None:
masks.append(satmask)
mask_colors.append(204)
if overlays:
# Could be single overlay, or list. Replicate behaviour of
# gt.make_lists (which we can't use because we haven't
# made a complete list of displayed extensions at the start
# in order to avoid memory bloat)
try:
overlay = overlays[overlay_index]
except TypeError:
overlay = overlays
except IndexError:
if len(overlays) == 1:
overlay = overlays[0]
try:
masks.append(make_overlay_mask(overlay, ext.shape))
except Exception:
pass
else:
mask_colors.append(206)
overlay_index += 1
# Define the display name
if tile and extname == 'SCI':
name = ext.filename
elif tile:
name = f'{ext.filename}({extname})'
else:
name = f'{ext.filename}({extname}, extension {ext.id})'
try:
lnd.display(data,
name=name,
frame=frame,
zscale=zscale,
bpm=None if extname == 'DQ' else dqdata,
quiet=True,
masks=masks,
mask_colors=mask_colors)
except OSError:
log.warning("ds9 not found; cannot display input")
frame += 1
# Print from statistics for flats
if extname == 'SCI' and {'GMOS', 'IMAGE', 'FLAT'}.issubset(
ext.tags):
good_data = data[dqdata ==
0] if dqdata is not None else data
mean = np.mean(good_data)
median = np.median(good_data)
log.stdinfo("Twilight flat counts for {}:".format(
ext.filename))
log.stdinfo(" Mean value: {:.0f}".format(mean))
log.stdinfo(" Median value: {:.0f}".format(median))
return adinputs
def tileArrays(self, adinputs=None, **params):
"""
This primitive combines extensions by tiling (no interpolation).
The array_section() and detector_section() descriptors are used
to derive the geometry of the tiling, so outside help (from the
instrument's geometry_conf module) is only required if there are
multiple arrays being tiled together, as the gaps need to be
specified.
If the input AstroData objects still have non-data regions, these
will not be trimmed. However, the WCS of the final image will
only be correct for some of the image since extra space has been
introduced into the image.
Parameters
----------
suffix: str
suffix to be added to output files
tile_all: bool
tile to a single extension, rather than one per array?
(array=physical detector)
sci_only: bool
tile only the data plane?
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
suffix = params['suffix']
tile_all = params['tile_all']
attributes = ['data'] if params["sci_only"] else None
adoutputs = []
for ad in adinputs:
if len(ad) == 1:
log.warning("{} has only one extension, so there's nothing "
"to tile".format(ad.filename))
adoutputs.append(ad)
continue
array_info = gt.array_information(ad)
detshape = array_info.detector_shape
if not tile_all and set(array_info.array_shapes) == {(1, 1)}:
log.warning("{} has nothing to tile, as tile_all=False but "
"each array has only one amplifier.")
adoutputs.append(ad)
continue
if tile_all and detshape != (1, 1): # We need gaps!
geotable = import_module('.geometry_conf', self.inst_lookups)
chip_gaps = geotable.tile_gaps[ad.detector_name()]
try:
xgap, ygap = chip_gaps
except TypeError: # single number, applies to both
xgap = ygap = chip_gaps
kw = ad._keyword_for('data_section')
xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin()
# Work out additional shifts required to cope with posisble overscan
# regions, including those in already-tiled CCDs
if tile_all:
yorigins, xorigins = np.rollaxis(
np.array(array_info.origins),
1).reshape((2, ) + array_info.detector_shape)
xorigins //= xbin
yorigins //= ybin
else:
yorigins, xorigins = np.zeros((2, ) +
array_info.detector_shape)
it_ccd = np.nditer(xorigins, flags=['multi_index'])
i = 0
while not it_ccd.finished:
ccdy, ccdx = it_ccd.multi_index
shp = array_info.array_shapes[i]
exts = array_info.extensions[i]
xshifts = np.zeros(shp, dtype=np.int32)
yshifts = np.zeros(shp, dtype=np.int32)
it = np.nditer(np.array(exts).reshape(shp),
flags=['multi_index'])
while not it.finished:
iy, ix = it.multi_index
ext = ad[int(it[0])]
datsec = ext.data_section()
if datsec.x1 > 0:
xshifts[iy, ix:] += datsec.x1
if datsec.x2 < ext.shape[1]:
xshifts[iy, ix + 1:] += ext.shape[1] - datsec.x2
if datsec.y1 > 0:
yshifts[iy:, ix] += datsec.y1
if datsec.y2 < ext.shape[0]:
xshifts[iy + 1:, ix] += ext.shape[0] - datsec.y2
arrsec = ext.array_section()
ext_shift = (models.Shift(
(arrsec.x1 // xbin - datsec.x1)) & models.Shift(
(arrsec.y1 // ybin - datsec.y1)))
# We need to have a "tile" Frame to resample to.
# We also need to perform the inverse, after the "tile"
# frame, of any change we make beforehand.
if ext.wcs is None:
ext.wcs = gWCS([(Frame2D(name="pixels"), ext_shift),
(Frame2D(name="tile"), None)])
elif 'tile' not in ext.wcs.available_frames:
#ext.wcs.insert_frame(ext.wcs.input_frame, ext_shift,
# Frame2D(name="tile"))
ext.wcs = gWCS([(ext.wcs.input_frame, ext_shift),
(Frame2D(name="tile"),
ext.wcs.pipeline[0].transform)] +
ext.wcs.pipeline[1:])
ext.wcs.insert_transform('tile',
ext_shift.inverse,
after=True)
dx, dy = xshifts[iy, ix], yshifts[iy, ix]
if tile_all:
dx += xorigins[ccdy, ccdx]
dy += yorigins[ccdy, ccdx]
if dx or dy: # Don't bother if they're both zero
shift_model = models.Shift(dx) & models.Shift(dy)
ext.wcs.insert_transform('tile',
shift_model,
after=False)
if ext.wcs.output_frame.name != 'tile':
ext.wcs.insert_transform('tile',
shift_model.inverse,
after=True)
# Reset data_section since we're not trimming overscans
ext.hdr[kw] = '[1:{},1:{}]'.format(*reversed(ext.shape))
it.iternext()
if tile_all:
# We need to shift other arrays if this one is larger than
# its expected size due to overscan regions. We've kept
# track of shifts we've introduced, but it might also be
# the case that we've been sent a previous tile_all=False output
if ccdx < detshape[1] - 1:
max_xshift = max(
xshifts.max(), ext.shape[1] -
(xorigins[ccdy, ccdx + 1] - xorigins[ccdy, ccdx]))
xorigins[ccdy, ccdx + 1:] += max_xshift + xgap // xbin
if ccdy < detshape[0] - 1:
max_yshift = max(
yshifts.max(), ext.shape[0] -
(yorigins[ccdy + 1, ccdx] - yorigins[ccdy, ccdx]))
yorigins[ccdy + 1:, ccdx] += max_yshift + ygap // ybin
elif i == 0:
ad_out = transform.resample_from_wcs(ad[exts],
"tile",
attributes=attributes,
process_objcat=True)
else:
ad_out.append(
transform.resample_from_wcs(ad[exts],
"tile",
attributes=attributes,
process_objcat=True)[0])
i += 1
it_ccd.iternext()
if tile_all:
ad_out = transform.resample_from_wcs(ad,
"tile",
attributes=attributes,
process_objcat=True)
gt.mark_history(ad_out,
primname=self.myself(),
keyword=timestamp_key)
ad_out.orig_filename = ad.filename
ad_out.update_filename(suffix=suffix, strip=True)
adoutputs.append(ad_out)
return adoutputs
def normalizeFlat(self, adinputs=None, **params):
"""
This primitive normalizes a GMOS Longslit spectroscopic flatfield
in a manner similar to that performed by gsflat in Gemini-IRAF.
A cubic spline is fitted along the dispersion direction of each
row, separately for each CCD.
As this primitive is GMOS-specific, we know the dispersion direction
will be along the rows, and there will be 3 CCDs.
For Hamamatsu CCDs, the 21 unbinned columns at each CCD edge are
masked out, following the procedure in gsflat.
TODO: Should we add these in the BPM?
Parameters
----------
suffix : str/None
suffix to be added to output files
center : int/None
central row/column for 1D extraction (None => use middle)
nsum : int
number of rows/columns around center to combine
function : str
type of function to fit (splineN or polynomial types)
order : int/str
Order of the spline fit to be performed
(can be 3 ints, separated by commas)
lsigma : float/None
lower rejection limit in standard deviations
hsigma : float/None
upper rejection limit in standard deviations
niter : int
maximum number of rejection iterations
grow : float/False
growth radius for rejected pixels
threshold : float
threshold (relative to peak) for flagging unilluminated pixels
interactive : bool
set to activate an interactive preview to fine tune the input parameters
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
# For flexibility, the code is going to pass whatever validated
# parameters it gets (apart from suffix and spectral_order) to
# the spline fitter
suffix = params["suffix"]
threshold = params["threshold"]
spectral_order = params["order"]
all_fp_init = [fit_1D.translate_params(params)] * 3
interactive_reduce = params["interactive"]
# Parameter validation should ensure we get an int or a list of 3 ints
try:
orders = [int(x) for x in spectral_order]
except TypeError:
orders = [spectral_order] * 3
# capture the per extension order into the fit parameters
for order, fp_init in zip(orders, all_fp_init):
fp_init["order"] = order
for ad in adinputs:
xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin()
array_info = gt.array_information(ad)
is_hamamatsu = 'Hamamatsu' in ad.detector_name(pretty=True)
ad_tiled = self.tileArrays([ad], tile_all=False)[0]
ad_fitted = astrodata.create(ad.phu)
all_fp_init = []
# If the entire row is unilluminated, we want to fit
# the pixels but still keep the edges masked
for ext in ad_tiled:
try:
ext.mask ^= (np.bitwise_and.reduce(ext.mask, axis=1)
& DQ.unilluminated)[:, None]
except TypeError: # ext.mask is None
pass
else:
if is_hamamatsu:
ext.mask[:, :21 // xbin] = 1
ext.mask[:, -21 // xbin:] = 1
all_fp_init.append(fit_1D.translate_params(params))
# Parameter validation should ensure we get an int or a list of 3 ints
try:
orders = [int(x) for x in spectral_order]
except TypeError:
orders = [spectral_order] * 3
# capture the per extension order into the fit parameters
for order, fp_init in zip(orders, all_fp_init):
fp_init["order"] = order
# Interactive or not
if interactive_reduce:
# all_X arrays are used to track appropriate inputs for each of the N extensions
all_pixels = []
all_domains = []
nrows = ad_tiled[0].shape[0]
for ext, order, indices in zip(ad_tiled, orders,
array_info.extensions):
pixels = np.arange(ext.shape[1])
all_pixels.append(pixels)
dispaxis = 2 - ext.dispersion_axis()
all_domains.append([0, ext.shape[dispaxis] - 1])
config = self.params[self.myself()]
config.update(**params)
# Create a 'row' parameter to add to the UI so the user can select the row they
# want to fit.
reinit_params = [
"row",
]
reinit_extras = {
"row":
RangeField("Row of data to operate on",
int,
int(nrows / 2),
min=1,
max=nrows)
}
# This function is used by the interactive fitter to generate the x,y,weights to use
# for each fit. We only want to fit a single row of data interactively, so that we can
# be responsive in the UI. The 'row' extra parameter defined above will create a
# slider for the user and we will have access to the selected value in the 'extras'
# dictionary passed in here.
def reconstruct_points(conf, extras):
r = min(0, extras['row'] - 1)
all_coords = []
for rppixels, rpext in zip(all_pixels, ad_tiled):
masked_data = np.ma.masked_array(
rpext.data[r],
mask=None if rpext.mask is None else rpext.mask[r])
if rpext.variance is None:
weights = None
else:
weights = np.sqrt(at.divide0(
1., rpext.variance[r]))
all_coords.append([rppixels, masked_data, weights])
return all_coords
visualizer = fit1d.Fit1DVisualizer(reconstruct_points,
all_fp_init,
config=config,
reinit_params=reinit_params,
reinit_extras=reinit_extras,
tab_name_fmt="CCD {}",
xlabel='x',
ylabel='y',
reinit_live=True,
domains=all_domains,
title="Normalize Flat",
enable_user_masking=False)
geminidr.interactive.server.interactive_fitter(visualizer)
# The fit models were done on a single row, so we need to
# get the parameters that were used in the final fit for
# each one, and then rerun it on the full data for that
# extension.
all_m_final = visualizer.results()
for m_final, ext in zip(all_m_final, ad_tiled):
masked_data = np.ma.masked_array(ext.data, mask=ext.mask)
weights = np.sqrt(at.divide0(1., ext.variance))
fit1d_params = m_final.extract_params()
fitted_data = fit_1D(masked_data,
weights=weights,
**fit1d_params,
axis=1).evaluate()
# Copy header so we have the _section() descriptors
ad_fitted.append(fitted_data, header=ext.hdr)
else:
for ext, indices, fit1d_params in zip(ad_tiled,
array_info.extensions,
all_fp_init):
masked_data = np.ma.masked_array(ext.data, mask=ext.mask)
weights = np.sqrt(at.divide0(1., ext.variance))
fitted_data = fit_1D(masked_data,
weights=weights,
**fit1d_params,
axis=1).evaluate()
# Copy header so we have the _section() descriptors
ad_fitted.append(fitted_data, header=ext.hdr)
# Find the largest spline value for each row across all extensions
# and mask pixels below the requested fraction of the peak
row_max = np.array([
ext_fitted.data.max(axis=1) for ext_fitted in ad_fitted
]).max(axis=0)
# Prevent runtime error in division
row_max[row_max == 0] = np.inf
for ext_fitted in ad_fitted:
ext_fitted.mask = np.where(
(ext_fitted.data.T / row_max).T < threshold,
DQ.unilluminated, DQ.good).astype(DQ.datatype)
for ext_fitted, indices in zip(ad_fitted, array_info.extensions):
tiled_arrsec = ext_fitted.array_section()
for i in indices:
ext = ad[i]
arrsec = ext.array_section()
slice_ = (slice((arrsec.y1 - tiled_arrsec.y1) // ybin,
(arrsec.y2 - tiled_arrsec.y1) // ybin),
slice((arrsec.x1 - tiled_arrsec.x1) // xbin,
(arrsec.x2 - tiled_arrsec.x1) // xbin))
# Suppress warnings to do with fitted_data==0
# (which create NaNs in variance)
with np.errstate(invalid='ignore', divide='ignore'):
ext.divide(ext_fitted.nddata[slice_])
np.nan_to_num(ext.data, copy=False, posinf=0, neginf=0)
np.nan_to_num(ext.variance, copy=False)
# Timestamp and update filename
gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key)
ad.update_filename(suffix=suffix, strip=True)
return adinputs
def scaleFringeToScience(self, rc):
"""
This primitive will scale the fringes to their matching science data
The fringes should be in the stream this primitive is called on,
and the reference science frames should be loaded into the RC,
as, eg. rc["science"] = adinput.
There are two ways to find the value to scale fringes by:
1. If stats_scale is set to True, the equation:
(letting science data = b (or B), and fringe = a (or A))
arrayB = where({where[SCIb < (SCIb.median+2.5*SCIb.std)]}
> [SCIb.median-3*SCIb.std])
scale = arrayB.std / SCIa.std
The section of the SCI arrays to use for calculating these statistics
is the CCD2 SCI data excluding the outer 5% pixels on all 4 sides.
Future enhancement: allow user to choose section
2. If stats_scale=False, then scale will be calculated using:
exposure time of science / exposure time of fringe
:param stats_scale: Use statistics to calculate the scale values,
rather than exposure time
:type stats_scale: Python boolean (True/False)
"""
# Instantiate the log
log = gemLog.getGeminiLog(logType=rc["logType"],
logLevel=rc["logLevel"])
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "scaleFringeToScience",
"starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["scaleFringeToScience"]
# Check for user-supplied science frames
fringe = rc.get_inputs_as_astrodata()
science_param = rc["science"]
fringe_dict = None
if science_param is not None:
# The user supplied an input to the science parameter
if not isinstance(science_param, list):
science_list = [science_param]
else:
science_list = science_param
# If there is one fringe and multiple science frames,
# the fringe must be deepcopied to allow it to be
# scaled separately for each frame
if len(fringe)==1 and len(science_list)>1:
fringe = [deepcopy(fringe[0]) for img in science_list]
# Convert filenames to AD instances if necessary
tmp_list = []
for science in science_list:
if type(science) is not AstroData:
science = AstroData(science)
tmp_list.append(science)
science_list = tmp_list
fringe_dict = gt.make_dict(key_list=science_list,
value_list=fringe)
fringe_output = []
else:
log.warning("No science frames specified; no scaling will be done")
science_list = []
fringe_output = fringe
# Loop over each AstroData object in the science list
for ad in science_list:
# Retrieve the appropriate fringe
fringe = fringe_dict[ad]
# Check the inputs have matching filters, binning and SCI shapes.
try:
gt.check_inputs_match(ad1=ad, ad2=fringe)
except Errors.ToolboxError:
# If not, try to clip the fringe frame to the size of the
# science data
# For a GMOS example, this allows a full frame fringe to
# be used for a CCD2-only science frame.
fringe = gt.clip_auxiliary_data(
adinput=ad, aux=fringe, aux_type="cal")[0]
# Check again, but allow it to fail if they still don't match
gt.check_inputs_match(ad1=ad, ad2=fringe)
# Check whether statistics should be used
stats_scale = rc["stats_scale"]
# Calculate the scale value
scale = 1.0
if not stats_scale:
# Use the exposure times to calculate the scale
log.fullinfo("Using exposure times to calculate the scaling"+
" factor")
try:
scale = ad.exposure_time() / fringe.exposure_time()
except:
raise Errors.InputError("Could not get exposure times " +
"for %s, %s. Try stats_scale=True" %
(ad.filename,fringe.filename))
else:
# Use statistics to calculate the scaling factor
log.fullinfo("Using statistics to calculate the " +
"scaling factor")
# Deepcopy the input so it can be manipulated without
# affecting the original
statsad = deepcopy(ad)
statsfringe = deepcopy(fringe)
# Trim off any overscan region still present
statsad,statsfringe = gt.trim_to_data_section([statsad,
statsfringe])
# Check the number of science extensions; if more than
# one, use CCD2 data only
nsciext = statsad.count_exts("SCI")
if nsciext>1:
# Get the CCD numbers and ordering information
# corresponding to each extension
log.fullinfo("Trimming data to data section to remove "\
"overscan region")
sci_info,frng_info = gt.array_information([statsad,
statsfringe])
# Pull out CCD2 data
scidata = []
frngdata = []
dqdata = []
for i in range(nsciext):
# Get the next extension in physical order
sciext = statsad["SCI",sci_info["amps_order"][i]]
frngext = statsfringe["SCI",frng_info["amps_order"][i]]
# Check to see if it is on CCD2; if so, keep it
if sci_info[
"array_number"][("SCI",sciext.extver())]==2:
scidata.append(sciext.data)
dqext = statsad["DQ",sci_info["amps_order"][i]]
maskext = statsad["OBJMASK",
sci_info["amps_order"][i]]
if dqext is not None and maskext is not None:
dqdata.append(dqext.data | maskext.data)
elif dqext is not None:
dqdata.append(dqext.data)
elif maskext is not None:
dqdata.append(maskext.data)
if frng_info[
"array_number"][("SCI",frngext.extver())]==2:
frngdata.append(frngext.data)
# Stack data if necessary
if len(scidata)>1:
scidata = np.hstack(scidata)
frngdata = np.hstack(frngdata)
else:
scidata = scidata[0]
frngdata = frngdata[0]
if len(dqdata)>0:
if len(dqdata)>1:
dqdata = np.hstack(dqdata)
else:
dqdata = dqdata[0]
else:
dqdata = None
else:
scidata = statsad["SCI"].data
frngdata = statsfringe["SCI"].data
dqext = statsad["DQ"]
maskext = statsad["OBJMASK"]
if dqext is not None and maskext is not None:
dqdata = dqext.data | maskext.data
elif dqext is not None:
dqdata = dqext.data
elif maskext is not None:
dqdata = maskext.data
else:
dqdata = None
if dqdata is not None:
# Replace any DQ-flagged data with the median value
smed = np.median(scidata[dqdata==0])
scidata = np.where(dqdata!=0,smed,scidata)
# Calculate the maximum and minimum in a box centered on
# each data point. The local depth of the fringe is
# max - min. The overall fringe strength is the median
# of the local fringe depths.
# Width of the box is binning and
# filter dependent, determined by experimentation
# Results don't seem to depend heavily on the box size
if ad.filter_name(pretty=True).as_pytype=="i":
size = 20
else:
size = 40
size /= ad.detector_x_bin().as_pytype()
# Use ndimage maximum_filter and minimum_filter to
# get the local maxima and minima
import scipy.ndimage as ndimage
sci_max = ndimage.filters.maximum_filter(scidata,size)
sci_min = ndimage.filters.minimum_filter(scidata,size)
# Take off 5% of the width as a border
xborder = int(0.05 * scidata.shape[1])
yborder = int(0.05 * scidata.shape[0])
if xborder<20:
xborder = 20
if yborder<20:
yborder = 20
sci_max = sci_max[yborder:-yborder,xborder:-xborder]
sci_min = sci_min[yborder:-yborder,xborder:-xborder]
# Take the median difference
sci_df = np.median(sci_max - sci_min)
# Do the same for the fringe
frn_max = ndimage.filters.maximum_filter(frngdata,size)
frn_min = ndimage.filters.minimum_filter(frngdata,size)
frn_max = frn_max[yborder:-yborder,xborder:-xborder]
frn_min = frn_min[yborder:-yborder,xborder:-xborder]
frn_df = np.median(frn_max - frn_min)
# Scale factor
# This tends to overestimate the factor, but it is
# at least in the right ballpark, unlike the estimation
# used in girmfringe (masked_sci.std/fringe.std)
scale = sci_df / frn_df
log.fullinfo("Scale factor found = "+str(scale))
# Use mult from the arith toolbox to perform the scaling of
# the fringe frame
scaled_fringe = fringe.mult(scale)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=scaled_fringe, keyword=timestamp_key)
# Change the filename
scaled_fringe.filename = gt.filename_updater(
adinput=ad, suffix=rc["suffix"], strip=True)
fringe_output.append(scaled_fringe)
# Report the list of output AstroData objects to the reduction context
rc.report_output(fringe_output)
yield rc