def load_config_file(config_file): """ Read the file `config_file` and return the parsed configuration. """ if s3_utils.is_s3_uri(config_file): return _load_config_file_s3(config_file) else: return _load_config_file_filesystem(config_file)
def check_reference_open(refpath): """Verify that `refpath` exists and is readable for the current user. Ignore reference path values of "N/A" or "" for checking. """ if refpath != "N/A" and refpath.strip() != "": if s3_utils.is_s3_uri(refpath): if not s3_utils.object_exists(refpath): raise RuntimeError("S3 object does not exist: " + refpath) else: with open(refpath, "rb"): pass return refpath
def test_is_s3_uri(s3_text_file): assert s3_utils.is_s3_uri("s3://test-s3-data/test.fits") is True assert s3_utils.is_s3_uri("some/filesystem/path") is False
def niriss_soss(input_model, reference_files): """ The NIRISS SOSS WCS pipeline. Parameters ---------- input_model : `~jwst.datamodel.DataModel` Input datamodel for processing reference_files : dict The dictionary of reference file names and their associated files {reftype: reference file name}. Returns ------- pipeline : list The pipeline list that is returned is suitable for input into gwcs.wcs.WCS to create a GWCS object. Notes ----- It includes tWO coordinate frames - "detector" and "world". It uses the "specwcs" reference file. """ # Get the target RA and DEC, they will be used for setting the WCS RA # and DEC based on a conversation with Kevin Volk. try: target_ra = float(input_model.meta.target.ra) target_dec = float(input_model.meta.target.dec) except TypeError: # There was an error getting the target RA and DEC, so we are not going to continue. raise ValueError( 'Problem getting the TARG_RA or TARG_DEC from input model {}'. format(input_model)) # Define the frames detector = cf.Frame2D(name='detector', axes_order=(0, 1), unit=(u.pix, u.pix)) spec = cf.SpectralFrame(name='spectral', axes_order=(2, ), unit=(u.micron, ), axes_names=('wavelength', )) sky = cf.CelestialFrame(reference_frame=coord.ICRS(), axes_names=('ra', 'dec'), axes_order=(0, 1), unit=(u.deg, u.deg), name='sky') world = cf.CompositeFrame([sky, spec], name='world') try: # We'd like to open this file as a DataModel, so we can consolidate # the S3 URI handling to one place. The S3-related code here can # be removed once that has been changed. if s3_utils.is_s3_uri(reference_files['specwcs']): wl = asdf.open(s3_utils.get_object(reference_files['specwcs'])) else: wl = asdf.open(reference_files['specwcs']) with wl: wl1 = wl.tree[1].copy() wl2 = wl.tree[2].copy() wl3 = wl.tree[3].copy() except Exception: raise IOError('Error reading wavelength correction from {}'.format( reference_files['specwcs'])) try: velosys = input_model.meta.wcsinfo.velosys except AttributeError: pass else: if velosys is not None: velocity_corr = velocity_correction( input_model.meta.wcsinfo.velosys) wl1 = wl1 | velocity_corr wl2 = wl2 | velocity_corr wl2 = wl3 | velocity_corr log.info("Applied Barycentric velocity correction: {}".format( velocity_corr[1].amplitude.value)) # Reverse the order of inputs passed to Tabular because it's in python order in modeling. # Consider changing it in modelng ? cm_order1 = (Mapping((0, 1, 1, 0)) | \ (Const1D(target_ra) & Const1D(target_dec) & wl1) ).rename('Order1') cm_order2 = (Mapping((0, 1, 1, 0)) | \ (Const1D(target_ra) & Const1D(target_dec) & wl2) ).rename('Order2') cm_order3 = (Mapping((0, 1, 1, 0)) | \ (Const1D(target_ra) & Const1D(target_dec) & wl3) ).rename('Order3') subarray2full = subarray_transform(input_model) if subarray2full is not None: cm_order1 = subarray2full | cm_order1 cm_order2 = subarray2full | cm_order2 cm_order3 = subarray2full | cm_order3 bbox = ((-0.5, input_model.meta.subarray.ysize - 0.5), (-0.5, input_model.meta.subarray.xsize - 0.5)) cm_order1.bounding_box = bbox cm_order2.bounding_box = bbox cm_order3.bounding_box = bbox # Define the transforms, they should accept (x,y) and return (ra, dec, lambda) soss_model = NirissSOSSModel( [1, 2, 3], [cm_order1, cm_order2, cm_order3]).rename('3-order SOSS Model') # Define the pipeline based on the frames and models above. pipeline = [(detector, soss_model), (world, None)] return pipeline
def lrs_distortion(input_model, reference_files): """ The LRS-FIXEDSLIT and LRS-SLITLESS WCS pipeline. Transform from subarray (x, y) to (v2, v3, lambda) using the "specwcs" and "distortion" reference files. """ # subarray to full array transform subarray2full = subarray_transform(input_model) # full array to v2v3 transform for the ordinary imager with DistortionModel(reference_files['distortion']) as dist: distortion = dist.model # Combine models to create subarray to v2v3 distortion if subarray2full is not None: subarray_dist = subarray2full | distortion else: subarray_dist = distortion # Read in the reference table data and get the zero point (SIAF reference point) # of the LRS in the subarray ref frame # We'd like to open this file as a DataModel, so we can consolidate # the S3 URI handling to one place. The S3-related code here can # be removed once that has been changed. if s3_utils.is_s3_uri(reference_files['specwcs']): ref = fits.open(s3_utils.get_object(reference_files['specwcs'])) else: ref = fits.open(reference_files['specwcs']) with ref: lrsdata = np.array([d for d in ref[1].data]) # Get the zero point from the reference data. # The zero_point is X, Y (which should be COLUMN, ROW) # These are 1-indexed in CDP-7 (i.e., SIAF convention) so must be converted to 0-indexed if input_model.meta.exposure.type.lower() == 'mir_lrs-fixedslit': zero_point = ref[0].header['imx'] - 1, ref[0].header['imy'] - 1 elif input_model.meta.exposure.type.lower() == 'mir_lrs-slitless': zero_point = ref[0].header['imxsltl'] - 1, ref[0].header['imysltl'] - 1 # Transform to slitless subarray from full array zero_point = subarray2full.inverse(zero_point[0], zero_point[1]) # In the lrsdata reference table, X_center,y_center,wavelength describe the location of the # centroid trace along the detector in pixels relative to nominal location. # x0,y0(ul) x1,y1 (ur) x2,y2(lr) x3,y3(ll) define corners of the box within which the distortion # and wavelength calibration was derived xcen = lrsdata[:, 0] ycen = lrsdata[:, 1] wavetab = lrsdata[:, 2] x0 = lrsdata[:, 3] y0 = lrsdata[:, 4] x1 = lrsdata[:, 5] y2 = lrsdata[:, 8] # If in fixed slit mode, define the bounding box using the corner locations provided in # the CDP reference file. if input_model.meta.exposure.type.lower() == 'mir_lrs-fixedslit': bb_sub = ((np.floor(x0.min() + zero_point[0]) - 0.5, np.ceil(x1.max() + zero_point[0]) + 0.5), (np.floor(y2.min() + zero_point[1]) - 0.5, np.ceil(y0.max() + zero_point[1]) + 0.5)) # If in slitless mode, define the bounding box X locations using the subarray x boundaries # and the y locations using the corner locations in the CDP reference file. Make sure to # omit the 4 reference pixels on the left edge of slitless subarray. if input_model.meta.exposure.type.lower() == 'mir_lrs-slitless': bb_sub = ((input_model.meta.subarray.xstart - 1 + 4 - 0.5, input_model.meta.subarray.xsize - 1 + 0.5), (np.floor(y2.min() + zero_point[1]) - 0.5, np.ceil(y0.max() + zero_point[1]) + 0.5)) # Find the ROW of the zero point row_zero_point = zero_point[1] # The inputs to the "detector_to_v2v3" transform are # - the indices in x spanning the entire image row # - y is the y-value of the zero point # This is equivalent of making a vector of x, y locations for # every pixel in the reference row const1d = models.Const1D(row_zero_point) const1d.inverse = models.Const1D(row_zero_point) det_to_v2v3 = models.Identity(1) & const1d | subarray_dist # Now deal with the fact that the spectral trace isn't perfectly up and down along detector. # This information is contained in the xcenter/ycenter values in the CDP table, but we'll handle it # as a simple rotation using a linear fit to this relation provided by the CDP. z = np.polyfit(xcen, ycen, 1) slope = 1. / z[0] traceangle = np.arctan(slope) * 180. / np.pi # trace angle in degrees rot = models.Rotation2D(traceangle) # Rotation model # Now include this rotation in our overall transform # First shift to a frame relative to the trace zeropoint, then apply the rotation # to correct for the curved trace. End in a rotated frame relative to zero at the reference point # and where yrot is aligned with the spectral trace) xysubtoxyrot = models.Shift(-zero_point[0]) & models.Shift(-zero_point[1]) | rot # Next shift back to the subarray frame, and then map to v2v3 xyrottov2v3 = models.Shift(zero_point[0]) & models.Shift(zero_point[1]) | det_to_v2v3 # The two models together xysubtov2v3 = xysubtoxyrot | xyrottov2v3 # Work out the spectral component of the transform # First compute the reference trace in the rotated-Y frame xcenrot, ycenrot = rot(xcen, ycen) # The input table of wavelengths isn't perfect, and the delta-wavelength # steps show some unphysical behaviour # Therefore fit with a spline for the ycenrot->wavelength transform # Reverse vectors so that yinv is increasing (needed for spline fitting function) yrev = ycenrot[::-1] wrev = wavetab[::-1] # Spline fit with enforced smoothness spl = UnivariateSpline(yrev, wrev, s=0.002) # Evaluate the fit at the rotated-y reference points wavereference = spl(yrev) # wavereference now contains the wavelengths corresponding to regularly-sampled ycenrot, create the model wavemodel = models.Tabular1D(lookup_table=wavereference, points=yrev, name='waveref', bounds_error=False, fill_value=np.nan) # Now construct the inverse spectral transform. # First we need to create a spline going from wavereference -> ycenrot spl2 = UnivariateSpline(wavereference[::-1], ycenrot, s=0.002) # Make a uniform grid of wavelength points from min to max, sampled according # to the minimum delta in the input table dw = np.amin(np.absolute(np.diff(wavereference))) wmin = np.amin(wavereference) wmax = np.amax(wavereference) wgrid = np.arange(wmin, wmax, dw) # Evaluate the rotated y locations of the grid ygrid = spl2(wgrid) # ygrid now contains the rotated y pixel locations corresponding to # regularly-sampled wavelengths, create the model wavemodel.inverse = models.Tabular1D(lookup_table=ygrid, points=wgrid, name='waverefinv', bounds_error=False, fill_value=np.nan) # Wavelength barycentric correction try: velosys = input_model.meta.wcsinfo.velosys except AttributeError: pass else: if velosys is not None: velocity_corr = velocity_correction(input_model.meta.wcsinfo.velosys) wavemodel = wavemodel | velocity_corr log.info("Applied Barycentric velocity correction : {}".format(velocity_corr[1].amplitude.value)) # Construct the full distortion model (xsub,ysub -> v2,v3,wavelength) lrs_wav_model = xysubtoxyrot | models.Mapping([1], n_inputs=2) | wavemodel dettotel = models.Mapping((0, 1, 0, 1)) | xysubtov2v3 & lrs_wav_model # Construct the inverse distortion model (v2,v3,wavelength -> xsub,ysub) # Transform to get xrot from v2,v3 v2v3toxrot = subarray_dist.inverse | xysubtoxyrot | models.Mapping([0], n_inputs=2) # wavemodel.inverse gives yrot from wavelength # v2,v3,lambda -> xrot,yrot xform1 = v2v3toxrot & wavemodel.inverse dettotel.inverse = xform1 | xysubtoxyrot.inverse # Bounding box is the subarray bounding box, because we're assuming subarray coordinates passed in dettotel.bounding_box = bb_sub[::-1] return dettotel
def open(init=None, memmap=False, **kwargs): """ Creates a DataModel from a number of different types Parameters ---------- init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, dict, None - None: A default data model with no shape - shape tuple: Initialize with empty data of the given shape - file path: Initialize from the given file (FITS , JSON or ASDF) - readable file object: Initialize from the given file object - astropy.io.fits.HDUList: Initialize from the given `~astropy.io.fits.HDUList` - A numpy array: A new model with the data array initialized to what was passed in. - dict: The object model tree for the data model memmap : bool Turn memmap of FITS file on or off. (default: False). Ignored for ASDF files. kwargs : dict Additional keyword arguments passed to lower level functions. These arguments are generally file format-specific. Arguments of note are: - FITS skip_fits_update - bool or None `True` to skip updating the ASDF tree from the FITS headers, if possible. If `None`, value will be taken from the environmental SKIP_FITS_UPDATE. Otherwise, the default value is `True`. Returns ------- model : DataModel instance """ from . import model_base # Initialize variables used to select model class hdulist = {} shape = () file_name = None file_to_close = None # Get special cases for opening a model out of the way # all special cases return a model if they match if init is None: return model_base.JwstDataModel(None) elif isinstance(init, model_base.JwstDataModel): # Copy the object so it knows not to close here return init.__class__(init) elif isinstance(init, (str, bytes)) or hasattr(init, "read"): # If given a string, presume its a file path. # if it has a read method, assume a file descriptor if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) file_name = basename(init) file_type = filetype.check(init) if file_type == "fits": if s3_utils.is_s3_uri(init): hdulist = fits.open(s3_utils.get_object(init)) else: hdulist = fits.open(init, memmap=memmap) file_to_close = hdulist elif file_type == "asn": # Read the file as an association / model container from . import container return container.ModelContainer(init, **kwargs) elif file_type == "asdf": if s3_utils.is_s3_uri(init): asdffile = asdf.open(s3_utils.get_object(init), **kwargs) else: asdffile = asdf.open(init, **kwargs) # Detect model type, then get defined model, and call it. new_class = _class_from_model_type(asdffile) if new_class is None: # No model class found, so return generic DataModel. return model_base.JwstDataModel(asdffile, **kwargs) return new_class(asdffile) elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init elif isinstance(init, np.ndarray): shape = init.shape elif isinstance(init, fits.HDUList): hdulist = init elif is_association(init) or isinstance(init, list): from . import container return container.ModelContainer(init, **kwargs) # If we have it, determine the shape from the science hdu if hdulist: # So we don't need to open the image twice init = hdulist info = init.fileinfo(0) if info is not None: file_name = info.get('filename') try: hdu = hdulist[('SCI', 1)] except (KeyError, NameError): shape = () else: if hasattr(hdu, 'shape'): shape = hdu.shape else: shape = () # First try to get the class name from the primary header new_class = _class_from_model_type(hdulist) has_model_type = new_class is not None # Special handling for ramp files for backwards compatibility if new_class is None: new_class = _class_from_ramp_type(hdulist, shape) # Or get the class from the reference file type and other header keywords if new_class is None: new_class = _class_from_reftype(hdulist, shape) # Or Get the class from the shape if new_class is None: new_class = _class_from_shape(hdulist, shape) # Throw an error if these attempts were unsuccessful if new_class is None: raise TypeError("Can't determine datamodel class from argument to open") # Log a message about how the model was opened if file_name: log.debug(f'Opening {file_name} as {new_class}') else: log.debug(f'Opening as {new_class}') # Actually open the model model = new_class(init, **kwargs) # Close the hdulist if we opened it if file_to_close is not None: # TODO: We need a better solution than messing with DataModel # internals. model._file_references.append(_FileReference(file_to_close)) if not has_model_type: class_name = new_class.__name__.split('.')[-1] if file_name: warnings.warn(f"model_type not found. Opening {file_name} as a {class_name}", NoTypeWarning) try: delattr(model.meta, 'model_type') except AttributeError: pass return model