def _load_config_file_s3(config_file):
if not s3_utils.object_exists(config_file):
raise ValueError("Config file {0} not found.".format(config_file))
content = s3_utils.get_object(config_file)
try:
with asdf_open(content) as asdf_file:
return _config_obj_from_asdf(asdf_file)
except (AsdfValidationError, ValueError):
logger.debug(
'Config file did not parse as ASDF. Trying as ConfigObj: %s',
config_file)
content.seek(0)
return ConfigObj(content, raise_errors=True)
def test_get_object(s3_text_file):
assert s3_utils.get_object("s3://test-s3-data/test.txt").read() == b"foo\n"
def lrs_distortion(input_model, reference_files):
"""
The LRS-FIXEDSLIT and LRS-SLITLESS WCS pipeline.
Transform from subarray (x, y) to (v2, v3, lambda) using
the "specwcs" and "distortion" reference files.
"""
# subarray to full array transform
subarray2full = subarray_transform(input_model)
# full array to v2v3 transform for the ordinary imager
with DistortionModel(reference_files['distortion']) as dist:
distortion = dist.model
# Combine models to create subarray to v2v3 distortion
if subarray2full is not None:
subarray_dist = subarray2full | distortion
else:
subarray_dist = distortion
# Read in the reference table data and get the zero point (SIAF reference point)
# of the LRS in the subarray ref frame
# We'd like to open this file as a DataModel, so we can consolidate
# the S3 URI handling to one place. The S3-related code here can
# be removed once that has been changed.
if s3_utils.is_s3_uri(reference_files['specwcs']):
ref = fits.open(s3_utils.get_object(reference_files['specwcs']))
else:
ref = fits.open(reference_files['specwcs'])
with ref:
lrsdata = np.array([d for d in ref[1].data])
# Get the zero point from the reference data.
# The zero_point is X, Y (which should be COLUMN, ROW)
# These are 1-indexed in CDP-7 (i.e., SIAF convention) so must be converted to 0-indexed
if input_model.meta.exposure.type.lower() == 'mir_lrs-fixedslit':
zero_point = ref[0].header['imx'] - 1, ref[0].header['imy'] - 1
elif input_model.meta.exposure.type.lower() == 'mir_lrs-slitless':
zero_point = ref[0].header['imxsltl'] - 1, ref[0].header['imysltl'] - 1
# Transform to slitless subarray from full array
zero_point = subarray2full.inverse(zero_point[0], zero_point[1])
# In the lrsdata reference table, X_center,y_center,wavelength describe the location of the
# centroid trace along the detector in pixels relative to nominal location.
# x0,y0(ul) x1,y1 (ur) x2,y2(lr) x3,y3(ll) define corners of the box within which the distortion
# and wavelength calibration was derived
xcen = lrsdata[:, 0]
ycen = lrsdata[:, 1]
wavetab = lrsdata[:, 2]
x0 = lrsdata[:, 3]
y0 = lrsdata[:, 4]
x1 = lrsdata[:, 5]
y2 = lrsdata[:, 8]
# If in fixed slit mode, define the bounding box using the corner locations provided in
# the CDP reference file.
if input_model.meta.exposure.type.lower() == 'mir_lrs-fixedslit':
bb_sub = ((np.floor(x0.min() + zero_point[0]) - 0.5, np.ceil(x1.max() + zero_point[0]) + 0.5),
(np.floor(y2.min() + zero_point[1]) - 0.5, np.ceil(y0.max() + zero_point[1]) + 0.5))
# If in slitless mode, define the bounding box X locations using the subarray x boundaries
# and the y locations using the corner locations in the CDP reference file. Make sure to
# omit the 4 reference pixels on the left edge of slitless subarray.
if input_model.meta.exposure.type.lower() == 'mir_lrs-slitless':
bb_sub = ((input_model.meta.subarray.xstart - 1 + 4 - 0.5, input_model.meta.subarray.xsize - 1 + 0.5),
(np.floor(y2.min() + zero_point[1]) - 0.5, np.ceil(y0.max() + zero_point[1]) + 0.5))
# Find the ROW of the zero point
row_zero_point = zero_point[1]
# The inputs to the "detector_to_v2v3" transform are
# - the indices in x spanning the entire image row
# - y is the y-value of the zero point
# This is equivalent of making a vector of x, y locations for
# every pixel in the reference row
const1d = models.Const1D(row_zero_point)
const1d.inverse = models.Const1D(row_zero_point)
det_to_v2v3 = models.Identity(1) & const1d | subarray_dist
# Now deal with the fact that the spectral trace isn't perfectly up and down along detector.
# This information is contained in the xcenter/ycenter values in the CDP table, but we'll handle it
# as a simple rotation using a linear fit to this relation provided by the CDP.
z = np.polyfit(xcen, ycen, 1)
slope = 1. / z[0]
traceangle = np.arctan(slope) * 180. / np.pi # trace angle in degrees
rot = models.Rotation2D(traceangle) # Rotation model
# Now include this rotation in our overall transform
# First shift to a frame relative to the trace zeropoint, then apply the rotation
# to correct for the curved trace. End in a rotated frame relative to zero at the reference point
# and where yrot is aligned with the spectral trace)
xysubtoxyrot = models.Shift(-zero_point[0]) & models.Shift(-zero_point[1]) | rot
# Next shift back to the subarray frame, and then map to v2v3
xyrottov2v3 = models.Shift(zero_point[0]) & models.Shift(zero_point[1]) | det_to_v2v3
# The two models together
xysubtov2v3 = xysubtoxyrot | xyrottov2v3
# Work out the spectral component of the transform
# First compute the reference trace in the rotated-Y frame
xcenrot, ycenrot = rot(xcen, ycen)
# The input table of wavelengths isn't perfect, and the delta-wavelength
# steps show some unphysical behaviour
# Therefore fit with a spline for the ycenrot->wavelength transform
# Reverse vectors so that yinv is increasing (needed for spline fitting function)
yrev = ycenrot[::-1]
wrev = wavetab[::-1]
# Spline fit with enforced smoothness
spl = UnivariateSpline(yrev, wrev, s=0.002)
# Evaluate the fit at the rotated-y reference points
wavereference = spl(yrev)
# wavereference now contains the wavelengths corresponding to regularly-sampled ycenrot, create the model
wavemodel = models.Tabular1D(lookup_table=wavereference, points=yrev, name='waveref',
bounds_error=False, fill_value=np.nan)
# Now construct the inverse spectral transform.
# First we need to create a spline going from wavereference -> ycenrot
spl2 = UnivariateSpline(wavereference[::-1], ycenrot, s=0.002)
# Make a uniform grid of wavelength points from min to max, sampled according
# to the minimum delta in the input table
dw = np.amin(np.absolute(np.diff(wavereference)))
wmin = np.amin(wavereference)
wmax = np.amax(wavereference)
wgrid = np.arange(wmin, wmax, dw)
# Evaluate the rotated y locations of the grid
ygrid = spl2(wgrid)
# ygrid now contains the rotated y pixel locations corresponding to
# regularly-sampled wavelengths, create the model
wavemodel.inverse = models.Tabular1D(lookup_table=ygrid, points=wgrid, name='waverefinv',
bounds_error=False, fill_value=np.nan)
# Wavelength barycentric correction
try:
velosys = input_model.meta.wcsinfo.velosys
except AttributeError:
pass
else:
if velosys is not None:
velocity_corr = velocity_correction(input_model.meta.wcsinfo.velosys)
wavemodel = wavemodel | velocity_corr
log.info("Applied Barycentric velocity correction : {}".format(velocity_corr[1].amplitude.value))
# Construct the full distortion model (xsub,ysub -> v2,v3,wavelength)
lrs_wav_model = xysubtoxyrot | models.Mapping([1], n_inputs=2) | wavemodel
dettotel = models.Mapping((0, 1, 0, 1)) | xysubtov2v3 & lrs_wav_model
# Construct the inverse distortion model (v2,v3,wavelength -> xsub,ysub)
# Transform to get xrot from v2,v3
v2v3toxrot = subarray_dist.inverse | xysubtoxyrot | models.Mapping([0], n_inputs=2)
# wavemodel.inverse gives yrot from wavelength
# v2,v3,lambda -> xrot,yrot
xform1 = v2v3toxrot & wavemodel.inverse
dettotel.inverse = xform1 | xysubtoxyrot.inverse
# Bounding box is the subarray bounding box, because we're assuming subarray coordinates passed in
dettotel.bounding_box = bb_sub[::-1]
return dettotel
def niriss_soss(input_model, reference_files):
"""
The NIRISS SOSS WCS pipeline.
Parameters
----------
input_model : `~jwst.datamodel.DataModel`
Input datamodel for processing
reference_files : dict
The dictionary of reference file names and their associated files
{reftype: reference file name}.
Returns
-------
pipeline : list
The pipeline list that is returned is suitable for
input into gwcs.wcs.WCS to create a GWCS object.
Notes
-----
It includes tWO coordinate frames -
"detector" and "world".
It uses the "specwcs" reference file.
"""
# Get the target RA and DEC, they will be used for setting the WCS RA
# and DEC based on a conversation with Kevin Volk.
try:
target_ra = float(input_model.meta.target.ra)
target_dec = float(input_model.meta.target.dec)
except TypeError:
# There was an error getting the target RA and DEC, so we are not going to continue.
raise ValueError(
'Problem getting the TARG_RA or TARG_DEC from input model {}'.
format(input_model))
# Define the frames
detector = cf.Frame2D(name='detector',
axes_order=(0, 1),
unit=(u.pix, u.pix))
spec = cf.SpectralFrame(name='spectral',
axes_order=(2, ),
unit=(u.micron, ),
axes_names=('wavelength', ))
sky = cf.CelestialFrame(reference_frame=coord.ICRS(),
axes_names=('ra', 'dec'),
axes_order=(0, 1),
unit=(u.deg, u.deg),
name='sky')
world = cf.CompositeFrame([sky, spec], name='world')
try:
# We'd like to open this file as a DataModel, so we can consolidate
# the S3 URI handling to one place. The S3-related code here can
# be removed once that has been changed.
if s3_utils.is_s3_uri(reference_files['specwcs']):
wl = asdf.open(s3_utils.get_object(reference_files['specwcs']))
else:
wl = asdf.open(reference_files['specwcs'])
with wl:
wl1 = wl.tree[1].copy()
wl2 = wl.tree[2].copy()
wl3 = wl.tree[3].copy()
except Exception:
raise IOError('Error reading wavelength correction from {}'.format(
reference_files['specwcs']))
try:
velosys = input_model.meta.wcsinfo.velosys
except AttributeError:
pass
else:
if velosys is not None:
velocity_corr = velocity_correction(
input_model.meta.wcsinfo.velosys)
wl1 = wl1 | velocity_corr
wl2 = wl2 | velocity_corr
wl2 = wl3 | velocity_corr
log.info("Applied Barycentric velocity correction: {}".format(
velocity_corr[1].amplitude.value))
# Reverse the order of inputs passed to Tabular because it's in python order in modeling.
# Consider changing it in modelng ?
cm_order1 = (Mapping((0, 1, 1, 0)) | \
(Const1D(target_ra) & Const1D(target_dec) & wl1)
).rename('Order1')
cm_order2 = (Mapping((0, 1, 1, 0)) | \
(Const1D(target_ra) & Const1D(target_dec) & wl2)
).rename('Order2')
cm_order3 = (Mapping((0, 1, 1, 0)) | \
(Const1D(target_ra) & Const1D(target_dec) & wl3)
).rename('Order3')
subarray2full = subarray_transform(input_model)
if subarray2full is not None:
cm_order1 = subarray2full | cm_order1
cm_order2 = subarray2full | cm_order2
cm_order3 = subarray2full | cm_order3
bbox = ((-0.5, input_model.meta.subarray.ysize - 0.5),
(-0.5, input_model.meta.subarray.xsize - 0.5))
cm_order1.bounding_box = bbox
cm_order2.bounding_box = bbox
cm_order3.bounding_box = bbox
# Define the transforms, they should accept (x,y) and return (ra, dec, lambda)
soss_model = NirissSOSSModel(
[1, 2, 3],
[cm_order1, cm_order2, cm_order3]).rename('3-order SOSS Model')
# Define the pipeline based on the frames and models above.
pipeline = [(detector, soss_model), (world, None)]
return pipeline
def open(init=None, memmap=False, **kwargs):
"""
Creates a DataModel from a number of different types
Parameters
----------
init : shape tuple, file path, file object, astropy.io.fits.HDUList,
numpy array, dict, None
- None: A default data model with no shape
- shape tuple: Initialize with empty data of the given shape
- file path: Initialize from the given file (FITS , JSON or ASDF)
- readable file object: Initialize from the given file object
- astropy.io.fits.HDUList: Initialize from the given
`~astropy.io.fits.HDUList`
- A numpy array: A new model with the data array initialized
to what was passed in.
- dict: The object model tree for the data model
memmap : bool
Turn memmap of FITS file on or off. (default: False). Ignored for
ASDF files.
kwargs : dict
Additional keyword arguments passed to lower level functions. These arguments
are generally file format-specific. Arguments of note are:
- FITS
skip_fits_update - bool or None
`True` to skip updating the ASDF tree from the FITS headers, if possible.
If `None`, value will be taken from the environmental SKIP_FITS_UPDATE.
Otherwise, the default value is `True`.
Returns
-------
model : DataModel instance
"""
from . import model_base
# Initialize variables used to select model class
hdulist = {}
shape = ()
file_name = None
file_to_close = None
# Get special cases for opening a model out of the way
# all special cases return a model if they match
if init is None:
return model_base.JwstDataModel(None)
elif isinstance(init, model_base.JwstDataModel):
# Copy the object so it knows not to close here
return init.__class__(init)
elif isinstance(init, (str, bytes)) or hasattr(init, "read"):
# If given a string, presume its a file path.
# if it has a read method, assume a file descriptor
if isinstance(init, bytes):
init = init.decode(sys.getfilesystemencoding())
file_name = basename(init)
file_type = filetype.check(init)
if file_type == "fits":
if s3_utils.is_s3_uri(init):
hdulist = fits.open(s3_utils.get_object(init))
else:
hdulist = fits.open(init, memmap=memmap)
file_to_close = hdulist
elif file_type == "asn":
# Read the file as an association / model container
from . import container
return container.ModelContainer(init, **kwargs)
elif file_type == "asdf":
if s3_utils.is_s3_uri(init):
asdffile = asdf.open(s3_utils.get_object(init), **kwargs)
else:
asdffile = asdf.open(init, **kwargs)
# Detect model type, then get defined model, and call it.
new_class = _class_from_model_type(asdffile)
if new_class is None:
# No model class found, so return generic DataModel.
return model_base.JwstDataModel(asdffile, **kwargs)
return new_class(asdffile)
elif isinstance(init, tuple):
for item in init:
if not isinstance(item, int):
raise ValueError("shape must be a tuple of ints")
shape = init
elif isinstance(init, np.ndarray):
shape = init.shape
elif isinstance(init, fits.HDUList):
hdulist = init
elif is_association(init) or isinstance(init, list):
from . import container
return container.ModelContainer(init, **kwargs)
# If we have it, determine the shape from the science hdu
if hdulist:
# So we don't need to open the image twice
init = hdulist
info = init.fileinfo(0)
if info is not None:
file_name = info.get('filename')
try:
hdu = hdulist[('SCI', 1)]
except (KeyError, NameError):
shape = ()
else:
if hasattr(hdu, 'shape'):
shape = hdu.shape
else:
shape = ()
# First try to get the class name from the primary header
new_class = _class_from_model_type(hdulist)
has_model_type = new_class is not None
# Special handling for ramp files for backwards compatibility
if new_class is None:
new_class = _class_from_ramp_type(hdulist, shape)
# Or get the class from the reference file type and other header keywords
if new_class is None:
new_class = _class_from_reftype(hdulist, shape)
# Or Get the class from the shape
if new_class is None:
new_class = _class_from_shape(hdulist, shape)
# Throw an error if these attempts were unsuccessful
if new_class is None:
raise TypeError("Can't determine datamodel class from argument to open")
# Log a message about how the model was opened
if file_name:
log.debug(f'Opening {file_name} as {new_class}')
else:
log.debug(f'Opening as {new_class}')
# Actually open the model
model = new_class(init, **kwargs)
# Close the hdulist if we opened it
if file_to_close is not None:
# TODO: We need a better solution than messing with DataModel
# internals.
model._file_references.append(_FileReference(file_to_close))
if not has_model_type:
class_name = new_class.__name__.split('.')[-1]
if file_name:
warnings.warn(f"model_type not found. Opening {file_name} as a {class_name}",
NoTypeWarning)
try:
delattr(model.meta, 'model_type')
except AttributeError:
pass
return model
