def ifu_slicer2asdf(ifuslicer, outname): """ Create an asdf reference file with the MSA description. ifu_slicer2asdf("IFU_slicer.sgd", "ifu_slicer.asdf") Parameters ---------- ifuslicer : str A fits file with the IFU slicer description outname : str Name of output ASDF file. """ ref_kw = common_reference_file_keywords("IFUSLICER", "NIRSPEC IFU SLICER description - CDP4") f = fits.open(ifuslicer) tree = ref_kw.copy() data = f[1].data header = f[1].header shiftx = models.Shift(header['XREF'], name='ifu_slicer_xref') shifty = models.Shift(header['YREF'], name='ifu_slicer_yref') rot = models.Rotation2D(header['ROT'], name='ifu_slicer_rot') model = rot | shiftx & shifty tree['model'] = model tree['data'] = f[1].data f.close() fasdf = AsdfFile() fasdf.tree = tree fasdf.write_to(outname) return fasdf
def _check_value(value, schema): """ Perform the actual validation. """ if value is None: if schema.get('fits_required'): name = schema.get("fits_keyword") or schema.get("fits_hdu") raise jsonschema.ValidationError("%s is a required value" % name) else: validator_context = AsdfFile() validator_resolver = validator_context.resolver temp_schema = { '$schema': 'http://stsci.edu/schemas/asdf-schema/0.1.0/asdf-schema'} temp_schema.update(schema) validator = asdf_schema.get_validator(temp_schema, validator_context, validator_callbacks, validator_resolver) value = yamlutil.custom_tree_to_tagged_tree(value, validator_context) validator.validate(value, _schema=temp_schema) validator_context.close()
def alpha_beta2XanYan(input_model, reference_files): """ Create the transform from detector to Xan, Yan frame. forward transform: RegionsSelector label_mapper is LabelMapperDict() {channel_wave_range (): channel_number} selector is {channel_number: ab2Xan & ab2Yan} bacward_transform RegionsSelector label_mapper is LabelMapperDict() {channel_wave_range (): channel_number} selector is {channel_number: Xan2ab & Yan2ab} """ band = input_model.meta.instrument.band channel = input_model.meta.instrument.channel # used to read the wavelength range channels = [c + band for c in channel] f = AsdfFile.open(reference_files['v2v3']) v23 = f.tree['model'] f.close() f = AsdfFile.open(reference_files['wavelengthrange']) # the following should go in the asdf reader wave_range = f.tree['wavelengthrange'].copy() wave_channels = f.tree['channels'] wr = {} for ch, r in zip(wave_channels, wave_range): wr[ch] = r f.close() dict_mapper = {} sel = {} for c in channels: ch = int(c[0]) dict_mapper[tuple(wr[c])] = models.Mapping((2,), name="mapping_lam") | \ models.Const1D(ch, name="channel #") map1 = models.Mapping((1, 0, 1, 0), name='map2poly') map1._outputs = ('alpha', 'beta', 'alpha', 'beta') map1._inputs = ('alpha', 'beta') map1.inverse = models.Mapping((0, 1)) ident1 = models.Identity(1, name='identity_lam') ident1._inputs = ('lam',) chan_v23 = v23[c] v23chan_backward = chan_v23.inverse del chan_v23.inverse v23_spatial = map1 | chan_v23 v23_spatial.inverse = map1 | v23chan_backward v23c = v23_spatial & ident1 sel[ch] = v23c wave_range_mapper = selector.LabelMapperRange(('alpha', 'beta', 'lam'), dict_mapper, inputs_mapping=models.Mapping([2,])) wave_range_mapper.inverse = wave_range_mapper.copy() ab2xyan = selector.RegionsSelector(('alpha', 'beta', 'lam'), ('v2', 'v3', 'lam'), label_mapper=wave_range_mapper, selector=sel) return ab2xyan
def detector_to_gwa(reference_files, detector, disperser): """ Transform from DETECTOR frame to GWA frame. Parameters ---------- reference_files: dict Dictionary with reference files returned by CRDS. detector : str The detector keyword. disperser : dict A corrected disperser ASDF object. Returns ------- model : `~astropy.modeling.core.Model` model. Transform from DETECTOR frame to GWA frame. """ with AsdfFile.open(reference_files['fpa']) as f: fpa = f.tree[detector].copy() with AsdfFile.open(reference_files['camera']) as f: camera = f.tree['model'].copy() angles = [disperser['theta_x'], disperser['theta_y'], disperser['theta_z'], disperser['tilt_y']] rotation = Rotation3DToGWA(angles, axes_order="xyzy", name='rotaton') u2dircos = Unitless2DirCos(name='unitless2directional_cosines') model = (models.Shift(-1) & models.Shift(-1) | fpa | camera | u2dircos | rotation) return model
def saveto_asdf(self, path, header=None, update_path=True): """ This function ... :param path: :param header: :param update_path: :return: """ # If a header is not specified, created it from the WCS if header is None: header = self.header # Import from asdf import AsdfFile # Create the tree tree = dict() tree["data"] = self._data tree["header"] = header # Create the asdf file ff = AsdfFile(tree) # Write ff.write_to(path) # Update the path if update_path: self.path = path
def test_backwards_compat_gcrs(): obsgeoloc = ( 3.0856775814671916e+16, 9.257032744401574e+16, 6.1713551629343834e+19 ) obsgeovel = (2.0, 1.0, 8.0) old_frame_yaml = """ frames: - !wcs/celestial_frame-1.0.0 axes_names: [lon, lat] name: CelestialFrame reference_frame: type: GCRS obsgeoloc: - [%f, %f, %f] - !unit/unit-1.0.0 m obsgeovel: - [%f, %f, %f] - !unit/unit-1.0.0 m s-1 obstime: !time/time-1.0.0 2010-01-01 00:00:00.000 unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] """ % (obsgeovel + obsgeoloc) new_frame_yaml = """ frames: - !wcs/celestial_frame-1.1.0 axes_names: [lon, lat] name: CelestialFrame reference_frame: type: GCRS obsgeoloc: - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} obsgeovel: - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} obstime: !time/time-1.1.0 2010-01-01 00:00:00.000 unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] """ % (obsgeovel + obsgeoloc) old_buff = helpers.yaml_to_asdf(old_frame_yaml) old_asdf = AsdfFile.open(old_buff) old_frame = old_asdf.tree['frames'][0] old_loc = old_frame.reference_frame.obsgeoloc old_vel = old_frame.reference_frame.obsgeovel new_buff = helpers.yaml_to_asdf(new_frame_yaml) new_asdf = AsdfFile.open(new_buff) new_frame = new_asdf.tree['frames'][0] new_loc = new_frame.reference_frame.obsgeoloc new_vel = new_frame.reference_frame.obsgeovel assert (old_loc.x == new_loc.x and old_loc.y == new_loc.y and old_loc.z == new_loc.z) assert (old_vel.x == new_vel.x and old_vel.y == new_vel.y and old_vel.z == new_vel.z)
def prism2asdf(prifile, tiltyfile, tiltxfile, outname): """Create a NIRSPEC prism disperser reference file in ASDF format. Combine information stored in disperser_G?.dis and disperser_G?_TiltY.gtp files delievred by the IDT. Parameters ---------- prifile : list or str File with primary information for the PRSIM tiltyfile : str File with tilt_Y data, e.g. disperser_PRISM_TiltY.gtp. tiltxfile: str File with tilt_x data, e.g. disperser_PRISM_TiltX.gtp. outname : str Name of output ASDF file. Returns ------- fasdf : asdf.AsdfFile """ params = common_reference_file_keywords("PRISM", "NIRSPEC PRISM Model") flist = [prifile, tiltyfile, tiltxfile] # translate the files for fname in flist: try: refparams = dict_from_file(fname) except: print("Disperser file was not converted.") raise pdict = {} coeffs = {} parts = fname.lower().split(".")[0] ref = str("_".join(parts.split("_")[1:])) if "pri" not in fname: try: for i, c in enumerate(refparams['CoeffsTemperature00']): coeffs['c' + str(i)] = c pdict['tilt_model'] = models.Polynomial1D(len(coeffs)-1, **coeffs) del refparams['CoeffsTemperature00'] except KeyError: print("Missing CoeffsTemperature in {0}".format(fname)) raise # store the rest of the keys for k, v in refparams.items(): pdict[k] = v print(pdict) params[ref] = pdict fasdf = AsdfFile() fasdf.tree = params fasdf.write_to(outname) return fasdf
def ifuslit_to_msa(slits, reference_files): """ The transform from slit_frame to msa_frame. Parameters ---------- slits_id : list A list of slit IDs for all open shutters/slitlets. msafile : str The name of the msa reference file. Returns ------- model : `~jwst_lib.pipeline_models.Slit2Msa` model. Transform from slit_frame to msa_frame. """ with AsdfFile.open(reference_files['ifufore']) as f: ifufore = f.tree['model'] ifuslicer = AsdfFile.open(reference_files['ifuslicer']) models = {} ifuslicer_model = (ifuslicer.tree['model']).rename('ifuslicer_model') for slit in slits: slitdata = ifuslicer.tree['data'][slit] slitdata_model = (get_slit_location_model(slitdata)).rename('slitdata_model') msa_transform = slitdata_model | ifuslicer_model models[slit] = msa_transform ifuslicer.close() return Slit2Msa(models)
def calc_cube(numb, fried_parameter = 4, time_between = 0.7): filepath = os.getcwd().split("vApp_reduction",1)[0]+"vApp_reduction/data/psf_cube_cache/" filepath += "psf_cube_"+str(float(fried_parameter))+"_"+str(float(time_between))+"_"+str(int(numb))+".asdf" #expand to only demand numb =< numb on disk if os.path.exists(filepath): tree = AsdfFile.open(filepath, copy_arrays=True).tree tree_keys = tree.keys() psf_params = sorted(list(filter(lambda key: isinstance(key, float), tree_keys))) psf_cube = list(map(lambda param: np.copy(tree[param]), psf_params)) return psf_cube, psf_params else: path = os.getcwd()+"/code/psf/generate_vAPP_cube.py" params = [str(fried_parameter), str(time_between), str(numb)] cmd = [sys.executable, path, *params] print("please run: ") print(' '.join(cmd)) input("Press Enter to continue...") tree = AsdfFile.open(filepath).tree tree_keys = tree.keys() psf_params = sorted(list(filter(lambda key: isinstance(key, float), tree_keys))) psf_cube = list(map(lambda param: tree[param], psf_params)) return psf_cube, psf_params
def ifu_msa_to_oteip(reference_files): """ Transform from the MSA frame to the OTEIP frame. Parameters ---------- reference_files: dict Dictionary with reference files returned by CRDS. Returns ------- model : `~astropy.modeling.core.Model` model. Transform from MSA to OTEIP. """ with AsdfFile.open(reference_files['fore']) as f: fore = f.tree['model'].copy() with AsdfFile.open(reference_files['ifufore']) as f: ifufore = f.tree['model'].copy() msa2fore_mapping = Mapping((0, 1, 2, 2)) msa2fore_mapping.inverse = Identity(3) ifu_fore_transform = ifufore & Identity(1) ifu_fore_transform.inverse = Mapping((0, 1, 2, 2)) | ifufore.inverse & Identity(1) fore_transform = msa2fore_mapping | fore & Identity(1) return msa2fore_mapping | ifu_fore_transform | fore_transform
def imaging_distortion(input_model, reference_files): """ Create pixe2sky and sky2pixel transformation for the MIRI imager. Parameters ---------- model : jwst_lib.models.ImagingModel input model reference_files : dict reference files from CRDS using CDP 3 Reference distortion file MIRI_FM_MIRIMAGE_F1000W_PSF_03.01.00.fits reference files/corrections needed (pixel to sky): 1. Filter dependent shift in (x,y) (!with an oposite sign to that delievred by the IT) 2. Apply MI 3. Apply Ai and BI matrices 4. Apply the TI matrix (this gives V2/V3 coordinates) 5. Apply V2/V3 to sky transformation ref_file: filter_offset.asdf - (1) ref_file: distortion.asdf -(2,3,4) """ distortion = AsdfFile.open(reference_files['distortion']).tree['model'] filter_offset = AsdfFile.open(reference_files['filteroffset']).tree[input_model.meta.instrument.filter] full_distortion = models.Shift(filter_offset['row_offset']) & models.Shift( filter_offset['column_offset']) | distortion full_distortion = full_distortion.rename('distortion') return full_distortion
def ifupost2asdf(ifupost_files, outname): """ Create a reference file of type ``ifupost`` . Combines all IDT ``IFU-POST`` reference files in one ASDF file. forward direction : MSA to Collimator backward_direction: Collimator to MSA Parameters ---------- ifupost_files : list Names of all ``IFU-POST`` IDT reference files outname : str Name of output ``ASDF`` file """ ref_kw = common_reference_file_keywords("IFUPOST", "NIRSPEC IFU-POST transforms - CDP4") fa = AsdfFile() fa.tree = ref_kw for fifu in ifupost_files: n = int((fifu.split('IFU-POST_')[1]).split('.pcf')[0]) fa.tree[n] = {} with open(fifu) as f: lines = [l.strip() for l in f.readlines()] factors = lines[lines.index('*Factor 2') + 1].split() rotation_angle = float(lines[lines.index('*Rotation') + 1]) input_rot_center = lines[lines.index('*InputRotationCentre 2') + 1].split() output_rot_center = lines[lines.index('*OutputRotationCentre 2') + 1].split() linear_sky2det = homothetic_sky2det(input_rot_center, rotation_angle, factors, output_rot_center) degree = int(lines[lines.index('*FitOrder') + 1]) xcoeff_index = lines.index('*xForwardCoefficients 21 2') xlines = lines[xcoeff_index + 1: xcoeff_index + 22] xcoeff_forward = coeffs_from_pcf(degree, xlines) x_poly_forward = models.Polynomial2D(degree, name='x_poly_forward', **xcoeff_forward) ycoeff_index = lines.index('*yForwardCoefficients 21 2') ycoeff_forward = coeffs_from_pcf(degree, lines[ycoeff_index + 1: ycoeff_index + 22]) y_poly_forward = models.Polynomial2D(degree, name='y_poly_forward', **ycoeff_forward) xcoeff_index = lines.index('*xBackwardCoefficients 21 2') xcoeff_backward = coeffs_from_pcf(degree, lines[xcoeff_index + 1: xcoeff_index + 22]) x_poly_backward = models.Polynomial2D(degree, name='x_poly_backward', **xcoeff_backward) ycoeff_index = lines.index('*yBackwardCoefficients 21 2') ycoeff_backward = coeffs_from_pcf(degree, lines[ycoeff_index + 1: ycoeff_index + 22]) y_poly_backward = models.Polynomial2D(degree, name='y_poly_backward', **ycoeff_backward) output2poly_mapping = Identity(2, name='output_mapping') output2poly_mapping.inverse = Mapping([0, 1, 0, 1]) input2poly_mapping = Mapping([0, 1, 0, 1], name='input_mapping') input2poly_mapping.inverse = Identity(2) model_poly = input2poly_mapping | (x_poly_forward & y_poly_forward) | output2poly_mapping model = linear_sky2det | model_poly fa.tree[n]['model'] = model asdffile = fa.write_to(outname) return asdffile
def pcf_forward(pcffile, outname): """ Create the **IDT** forward transform from collimator to gwa. """ with open(pcffile) as f: lines = [l.strip() for l in f.readlines()] factors = lines[lines.index('*Factor 2') + 1].split() # factor==1/factor in backward msa2ote direction and factor==factor in sky2detector direction scale = models.Scale(float(factors[0]), name="x_scale") & \ models.Scale(float(factors[1]), name="y_scale") rotation_angle = lines[lines.index('*Rotation') + 1] # The minius sign here is because astropy.modeling has the opposite direction of rotation than the idl implementation rotation = models.Rotation2D(-float(rotation_angle), name='rotation') # Here the model is called "output_shift" but in the team version it is the "input_shift". input_rot_center = lines[lines.index('*InputRotationCentre 2') + 1].split() input_rot_shift = models.Shift(-float(input_rot_center[0]), name='input_x_shift') & \ models.Shift(-float(input_rot_center[1]), name='input_y_shift') # Here the model is called "input_shift" but in the team version it is the "output_shift". output_rot_center = lines[lines.index('*OutputRotationCentre 2') + 1].split() output_rot_shift = models.Shift(float(output_rot_center[0]), name='output_x_shift') & \ models.Shift(float(output_rot_center[1]), name='output_y_shift') degree = int(lines[lines.index('*FitOrder') + 1]) xcoeff_index = lines.index('*xForwardCoefficients 21 2') xlines = lines[xcoeff_index + 1: xcoeff_index + 22] xcoeff_forward = coeffs_from_pcf(degree, xlines) x_poly_forward = models.Polynomial2D(degree, name='x_poly_forward', **xcoeff_forward) ycoeff_index = lines.index('*yForwardCoefficients 21 2') ycoeff_forward = coeffs_from_pcf(degree, lines[ycoeff_index + 1: ycoeff_index + 22]) y_poly_forward = models.Polynomial2D(degree, name='y_poly_forward', **ycoeff_forward) xcoeff_index = lines.index('*xBackwardCoefficients 21 2') xcoeff_backward = coeffs_from_pcf(degree, lines[xcoeff_index + 1: xcoeff_index + 22]) x_poly_backward = models.Polynomial2D(degree, name='x_poly_backward', **xcoeff_backward) ycoeff_index = lines.index('*yBackwardCoefficients 21 2') ycoeff_backward = coeffs_from_pcf(degree, lines[ycoeff_index + 1: ycoeff_index + 22]) y_poly_backward = models.Polynomial2D(degree, name='y_poly_backward', **ycoeff_backward) x_poly_forward.inverse = x_poly_backward y_poly_forward.inverse = y_poly_backward poly_mapping1 = Mapping((0, 1, 0, 1)) poly_mapping1.inverse = Identity(2) poly_mapping2 = Identity(2) poly_mapping2.inverse = Mapping((0, 1, 0, 1)) model = input_rot_shift | rotation | scale | output_rot_shift | \ poly_mapping1 | x_poly_forward & y_poly_forward | poly_mapping2 f = AsdfFile() f.tree = {'model': model} f.write_to(outname)
def ote2asdf(otepcf, outname, ref_kw): """ ref_kw = common_reference_file_keywords('OTE', 'NIRSPEC OTE transform - CDP4') ote2asdf('Model/Ref_Files/CoordTransform/OTE.pcf', 'jwst_nirspec_ote_0001.asdf', ref_kw) """ with open(otepcf) as f: lines = [l.strip() for l in f.readlines()] factors = lines[lines.index('*Factor 2 1') + 1].split() # this corresponds to modeling Rotation direction as is rotation_angle = float(lines[lines.index('*Rotation') + 1]) input_rot_center = lines[lines.index('*InputRotationCentre 2 1') + 1].split() output_rot_center = lines[lines.index('*OutputRotationCentre 2 1') + 1].split() mlinear = homothetic_det2sky(input_rot_center, rotation_angle, factors, output_rot_center) degree = int(lines[lines.index('*FitOrder') + 1]) xcoeff_index = lines.index('*xBackwardCoefficients 21 2') xlines = lines[xcoeff_index + 1].split('\t') xcoeff_backward = coeffs_from_pcf(degree, xlines) x_poly_forward = models.Polynomial2D(degree, name='x_poly_forward', **xcoeff_backward) xcoeff_index = lines.index('*xForwardCoefficients 21 2') xlines = lines[xcoeff_index + 1].split('\t') xcoeff_forward = coeffs_from_pcf(degree, xlines) x_poly_backward = models.Polynomial2D(degree, name='x_poly_backward', **xcoeff_forward) ycoeff_index = lines.index('*yBackwardCoefficients 21 2') ylines = lines[ycoeff_index + 1].split('\t') ycoeff_backward = coeffs_from_pcf(degree, ylines) y_poly_forward = models.Polynomial2D(degree, name='y_poly_forward', **ycoeff_backward) ycoeff_index = lines.index('*yForwardCoefficients 21 2') ylines = lines[ycoeff_index + 1].split('\t') ycoeff_forward = coeffs_from_pcf(degree, ylines) y_poly_backward = models.Polynomial2D(degree, name='y_poly_backward', **ycoeff_forward) x_poly_forward.inverse = x_poly_backward y_poly_forward.inverse = y_poly_backward output2poly_mapping = Identity(2, name='output_mapping') output2poly_mapping.inverse = Mapping([0, 1, 0, 1]) input2poly_mapping = Mapping([0, 1, 0, 1], name='input_mapping') input2poly_mapping.inverse = Identity(2) model_poly = input2poly_mapping | (x_poly_forward & y_poly_forward) | output2poly_mapping model = model_poly | mlinear f = AsdfFile() f.tree = ref_kw.copy() f.tree['model'] = model f.write_to(outname) return model_poly, mlinear
def create_v23(reftype, detector, band, channels, data, name): """ Create the transform from MIRI Local to telescope V2/V3 system for all channels. """ channel = "".join([ch[0] for ch in channels]) tree = {"detector": detector, "instrument" : "MIRI", "band": band, "channel": channel, "exp_type": "MIR_MRS", "pedigree": "GROUND", "title": "MIRI IFU model - based on CDP-4", "reftype": reftype, "author": "N. Dencheva" } ab_v23 = data[0] v23_ab = data[1] m = {} c0_0, c0_1, c1_0, c1_1 = ab_v23[0][1:] ch1_v2 = models.Polynomial2D(2, c0_0=c0_0, c1_0=c1_0, c0_1=c0_1, c1_1=c1_1, name="ab_v23") c0_0, c0_1, c1_0, c1_1 = v23_ab[0][1:] ch1_a = models.Polynomial2D(2, c0_0=c0_0, c1_0=c1_0, c0_1=c0_1, c1_1=c1_1, name="v23_ab") c0_0, c0_1, c1_0, c1_1 = ab_v23[1][1:] ch1_v3 = models.Polynomial2D(2, c0_0=c0_0, c1_0=c1_0, c0_1=c0_1, c1_1=c1_1, name="ab_v23") c0_0, c0_1, c1_0, c1_1 = v23_ab[1][1:] ch1_b = models.Polynomial2D(2, c0_0=c0_0, c1_0=c1_0, c0_1=c0_1, c1_1=c1_1, name="v23_ab") c0_0, c0_1, c1_0, c1_1 = ab_v23[2][1:] ch2_v2 = models.Polynomial2D(2, c0_0=c0_0, c1_0=c1_0, c0_1=c0_1, c1_1=c1_1, name="ab_v23") c0_0, c0_1, c1_0, c1_1 = v23_ab[2][1:] ch2_a = models.Polynomial2D(2, c0_0=c0_0, c1_0=c1_0, c0_1=c0_1, c1_1=c1_1, name="v23_ab") c0_0, c0_1, c1_0, c1_1 = ab_v23[3][1:] ch2_v3 = models.Polynomial2D(2, c0_0=c0_0, c1_0=c1_0, c0_1=c0_1, c1_1=c1_1, name="ab_v23") c0_0, c0_1, c1_0, c1_1 = v23_ab[3][1:] ch2_b = models.Polynomial2D(2, c0_0=c0_0, c1_0=c1_0, c0_1=c0_1, c1_1=c1_1, name="v23_ab") ch1_for = ch1_v2 & ch1_v3 ch2_for = ch2_v2 & ch2_v3 ch1_for.inverse = ch1_a & ch1_b ch2_for.inverse = ch2_a & ch2_b m[channels[0]] = ch1_for m[channels[1]] = ch2_for tree['model'] = m f = AsdfFile() f.tree = tree f.write_to(name)
def gwa_to_ifuslit(slits, disperser, wrange, order, reference_files): """ GWA to SLIT transform. Parameters ---------- slits : list A list of slit IDs for all IFU slits 0-29. disperser : dict A corrected disperser ASDF object. filter : str The filter used. grating : str The grating used in the observation. reference_files: dict Dictionary with reference files returned by CRDS. Returns ------- model : `~jwst.transforms.Gwa2Slit` model. Transform from GWA frame to SLIT frame. """ ymin = -.55 ymax = .55 agreq = AngleFromGratingEquation(disperser['groove_density'], order, name='alpha_from_greq') lgreq = WavelengthFromGratingEquation(disperser['groove_density'], order, name='lambda_from_greq') collimator2gwa = collimator_to_gwa(reference_files, disperser) mask = mask_slit(ymin, ymax) ifuslicer = AsdfFile.open(reference_files['ifuslicer']) ifupost = AsdfFile.open(reference_files['ifupost']) slit_models = {} ifuslicer_model = ifuslicer.tree['model'] for slit in slits: slitdata = ifuslicer.tree['data'][slit] slitdata_model = get_slit_location_model(slitdata) ifuslicer_transform = (slitdata_model | ifuslicer_model) ifupost_transform = ifupost.tree[slit]['model'] msa2gwa = ifuslicer_transform | ifupost_transform | collimator2gwa gwa2msa = gwa_to_ymsa(msa2gwa)# TODO: Use model sets here bgwa2msa = Mapping((0, 1, 0, 1), n_inputs=3) | \ Const1D(0) * Identity(1) & Const1D(-1) * Identity(1) & Identity(2) | \ Identity(1) & gwa2msa & Identity(2) | \ Mapping((0, 1, 0, 1, 2, 3)) | Identity(2) & msa2gwa & Identity(2) | \ Mapping((0, 1, 2, 5), n_inputs=7) | Identity(2) & lgreq | mask # msa to before_gwa #msa2bgwa = Mapping((0, 1, 2, 2)) | msa2gwa & Identity(1) | Mapping((3, 0, 1, 2)) | agreq msa2bgwa = msa2gwa & Identity(1) | Mapping((3, 0, 1, 2)) | agreq bgwa2msa.inverse = msa2bgwa slit_models[slit] = bgwa2msa ifuslicer.close() ifupost.close() return Gwa2Slit(slit_models)
def imaging(input_model, reference_files): """ Imaging pipeline frames : detector, gwa, msa, sky """ # Get the corrected disperser model disperser = get_disperser(input_model, reference_files['disperser']) # DETECTOR to GWA transform det2gwa = detector_to_gwa(reference_files, input_model.meta.instrument.detector, disperser) gwa_through = Const1D(-1) * Identity(1) & Const1D(-1) * Identity(1) & Identity(1) angles = [ disperser['theta_x'], disperser['theta_y'], disperser['theta_z'], disperser['tilt_y']] rotation = Rotation3DToGWA(angles, axes_order="xyzy", name='rotaton').inverse dircos2unitless = DirCos2Unitless(name='directional_cosines2unitless') col = AsdfFile.open(reference_files['collimator']).tree['model'] # Get the default spectral order and wavelength range and record them in the model. sporder, wrange = get_spectral_order_wrange(input_model, reference_files['wavelengthrange']) input_model.meta.wcsinfo.waverange_start = wrange[0] input_model.meta.wcsinfo.waverange_end = wrange[1] input_model.meta.wcsinfo.spectral_order = sporder lam = wrange[0] + (wrange[1] - wrange[0]) * .5 lam_model = Mapping((0, 1, 1)) | Identity(2) & Const1D(lam) gwa2msa = gwa_through | rotation | dircos2unitless | col | lam_model gwa2msa.inverse = col.inverse | dircos2unitless.inverse | rotation.inverse | gwa_through # MSA to OTEIP transform msa2ote = msa_to_oteip(reference_files) msa2oteip = msa2ote | Mapping((0, 1), n_inputs=3) msa2oteip.inverse = Mapping((0, 1, 0, 1)) | msa2ote.inverse | Mapping((0, 1), n_inputs=3) # OTEIP to V2,V3 transform with AsdfFile.open(reference_files['ote']) as f: oteip2v23 = f.tree['model'].copy() # Create coordinate frames in the NIRSPEC WCS pipeline # "detector", "gwa", "msa", "oteip", "v2v3", "world" det, gwa, msa_frame, oteip, v2v3 = create_imaging_frames() imaging_pipeline = [(det, det2gwa), (gwa, gwa2msa), (msa_frame, msa2oteip), (oteip, oteip2v23), (v2v3, None)] return imaging_pipeline
def imaging_distortion(input_model, reference_files): """ Create pixe2sky and sky2pixel transformation for the MIRI imager. Parameters ---------- model : jwst.datamodels.ImagingModel input model reference_files : dict reference files from CRDS using CDP 3 Reference distortion file Old one: ~~MIRI_FM_MIRIMAGE_F1000W_PSF_03.01.00.fits~~ Current one: MIRI_FM_MIRIMAGE_DISTORTION_06.03.00.fits reference files/corrections needed (pixel to sky): 1. Filter dependent shift in (x,y) (!with an oposite sign to that delievred by the IT) 2. Apply MI 3. Apply Ai and BI matrices 4. Apply the TI matrix (this gives V2/V3 coordinates) 5. Apply V2/V3 to sky transformation ref_file: filter_offset.asdf - (1) ref_file: distortion.asdf -(2,3,4) """ # Load the distortion and filter from the reference files. # Load in the distortion file. distortion = AsdfFile.open(reference_files['distortion']).tree['model'] filter_offset = AsdfFile.open(reference_files['filteroffset']).tree[input_model.meta.instrument.filter] # Now apply each of the models. The Scale(60) converts from arc-minutes to deg. full_distortion = models.Shift(filter_offset['column_offset']) & models.Shift( filter_offset['row_offset']) | distortion | models.Scale(1/60) & models.Scale(1/60) # ToDo: This will likely have to change in the future, but the "filteroffset" file we have # ToDo: currently does not contain that key. filter_offset = None if input_model.meta.instrument.filter in AsdfFile.open(reference_files['filteroffset']).tree: filter_offset = AsdfFile.open(reference_files['filteroffset']).tree[input_model.meta.instrument.filter] full_distortion = models.Shift(filter_offset['row_offset']) & models.Shift( filter_offset['column_offset']) | distortion else: full_distortion = distortion full_distortion = full_distortion.rename('distortion') return full_distortion
def create_distortion_file(reftype, detector, band, channel, data, name): tree = create_reffile_header(reftype, detector, band, channel) adata, bdata, xdata, ydata, sdata1, sdata2 = data tree['alpha_model'] = adata tree['beta_model'] = bdata tree['x_model'] = xdata tree['y_model'] = ydata tree['slice_model'] = {str(channel[0])+band: sdata1, str(channel[1])+band: sdata2} f = AsdfFile() f.tree = tree f.write_to(name)
def runtest(self): from asdf import AsdfFile, block, util from asdf.tests import helpers from .extension import TestExtension name, version = parse_schema_filename(self.filename) if should_skip(name, version): return standard_version = self._find_standard_version(name, version) # Make sure that the examples in the schema files (and thus the # ASDF standard document) are valid. buff = helpers.yaml_to_asdf( 'example: ' + self.example.strip(), standard_version=standard_version) ff = AsdfFile( uri=util.filepath_to_url(os.path.abspath(self.filename)), extensions=TestExtension()) # Fake an external file ff2 = AsdfFile({'data': np.empty((1024*1024*8), dtype=np.uint8)}) ff._external_asdf_by_uri[ util.filepath_to_url( os.path.abspath( os.path.join( os.path.dirname(self.filename), 'external.asdf')))] = ff2 # Add some dummy blocks so that the ndarray examples work for i in range(3): b = block.Block(np.zeros((1024*1024*8), dtype=np.uint8)) b._used = True ff.blocks.add(b) b._array_storage = "streamed" try: with pytest.warns(None) as w: import warnings ff._open_impl(ff, buff, mode='rw') # Do not tolerate any warnings that occur during schema validation assert len(w) == 0, helpers.display_warnings(w) except Exception: print("From file:", self.filename) raise # Just test we can write it out. A roundtrip test # wouldn't always yield the correct result, so those have # to be covered by "real" unit tests. if b'external.asdf' not in buff.getvalue(): buff = io.BytesIO() ff.write_to(buff)
def fpa2asdf(fpafile, outname, ref_kw): """ Create an asdf reference file with the FPA description. The CDP2 delivery includes a fits file - "FPA.fpa" which is the input to this function. This file is converted to asdf and is a reference file of type "FPA". nirspec_fs_ref_tools.fpa2asdf('Ref_Files/CoordTransform/Description/FPA.fpa', 'fpa.asdf') Parameters ---------- fpafile : str A fits file with FPA description (FPA.fpa) outname : str Name of output ASDF file. """ with open(fpafile) as f: lines = [l.strip() for l in f.readlines()] # NRS1 ind = lines.index("*SCA491_PitchX") scalex_nrs1 = models.Scale(1/float(lines[ind+1]), name='fpa_scale_x') ind = lines.index("*SCA491_PitchY") scaley_nrs1 = models.Scale(1/float(lines[ind+1]), name='fpa_scale_y') ind = lines.index("*SCA491_RotAngle") rot_nrs1 = models.Rotation2D(np.rad2deg(-float(lines[ind+1])), name='fpa_rotation') ind = lines.index("*SCA491_PosX") shiftx_nrs1 = models.Shift(-float(lines[ind+1]), name='fpa_shift_x') ind = lines.index("*SCA491_PosY") shifty_nrs1 = models.Shift(-float(lines[ind+1]), name='fpa_shift_y') # NRS2 ind = lines.index("*SCA492_PitchX") scalex_nrs2 = models.Scale(1/float(lines[ind+1]), name='fpa_scale_x') ind = lines.index("*SCA492_PitchY") scaley_nrs2 = models.Scale(1/float(lines[ind+1]), name='fpa_scale_y') ind = lines.index("*SCA492_RotAngle") rot_nrs2 = models.Rotation2D(np.rad2deg(float(lines[ind+1])), name='fpa_rotation') ind = lines.index("*SCA492_PosX") shiftx_nrs2 = models.Shift(-float(lines[ind+1]), name='fpa_shift_x') ind = lines.index("*SCA492_PosY") shifty_nrs2 = models.Shift(-float(lines[ind+1]), name='fpa_shift_y') tree = ref_kw.copy() tree['NRS1'] = (shiftx_nrs1 & shifty_nrs1) | rot_nrs1 | (scalex_nrs1 & scaley_nrs1) tree['NRS2'] = (shiftx_nrs2 & shifty_nrs2) | rot_nrs2 | (scalex_nrs2 & scaley_nrs2) fasdf = AsdfFile() fasdf.tree = tree fasdf.write_to(outname) return fasdf
def slit_to_msa(slits_id, msafile): """ The transform from slit_frame to msa_frame. Parameters ---------- slits_id : list A list of slit IDs for all open shutters/slitlets. msafile : str The name of the msa reference file. Returns ------- model : `~jwst.transforms.Slit2Msa` model. Transform from slit_frame to msa_frame. """ msa = AsdfFile.open(msafile) models = {} for i in range(1, 6): slit_names = slits_id[slits_id[:, 0] == i] if slit_names.any(): msa_model = msa.tree[i]['model'] for slit in slit_names: index = slit[1] - 1 slitdata = msa.tree[slit[0]]['data'][index] slitdata_model = get_slit_location_model(slitdata) msa_transform = slitdata_model | msa_model s = slitid_to_slit(np.array([slit]))[0] models[s] = msa_transform msa.close() return Slit2Msa(models)
def collimator_to_gwa(reference_files, disperser): """ Transform from COLLIMATOR to GWA frame. Parameters ---------- reference_files: dict Dictionary with reference files returned by CRDS. disperser : dict A corrected disperser ASDF object. Returns ------- model : `~astropy.modeling.core.Model` model. Transform from COLLIMATOR to GWA frame. """ with AsdfFile.open(reference_files['collimator']) as f: collimator = f.tree['model'].copy() angles = [disperser['theta_x'], disperser['theta_y'], disperser['theta_z'], disperser['tilt_y']] rotation = Rotation3DToGWA(angles, axes_order="xyzy", name='rotaton') u2dircos = Unitless2DirCos(name='unitless2directional_cosines') return collimator.inverse | u2dircos | rotation
def oteip_to_v23(reference_files): """ Transform from the OTEIP frame to the V2V3 frame. Parameters ---------- reference_files: dict Dictionary with reference files returned by CRDS. Returns ------- model : `~astropy.modeling.core.Model` model. Transform from OTEIP to V2V3. """ with AsdfFile.open(reference_files['ote']) as f: ote = f.tree['model'].copy() fore2ote_mapping = Identity(3, name='fore2ote_mapping') fore2ote_mapping.inverse = Mapping((0, 1, 2, 2)) # Create the transform to v2/v3/lambda. The wavelength units up to this point are # meters as required by the pipeline but the desired output wavelength units is microns. # So we are going to Scale the spectral units by 1e6 (meters -> microns) # The spatial units are currently in deg. Convertin to arcsec. oteip_to_xyan = fore2ote_mapping | (ote & Scale(1e6)) # Add a shift for the aperture. oteip2v23 = oteip_to_xyan | Identity(1) & (Shift(468 / 3600) | Scale(-1)) & Identity(1) return oteip2v23
def slit_to_msa(open_slits, msafile): """ The transform from slit_frame to msa_frame. Parameters ---------- open_slits : list A list of slit IDs for all open shutters/slitlets. msafile : str The name of the msa reference file. Returns ------- model : `~jwst.transforms.Slit2Msa` model. Transform from slit_frame to msa_frame. """ msa = AsdfFile.open(msafile) models = [] for quadrant in range(1, 6): slits_in_quadrant = [s for s in open_slits if s.quadrant==quadrant] if any(slits_in_quadrant): msa_data = msa.tree[quadrant]['data'] msa_model = msa.tree[quadrant]['model'] for slit in slits_in_quadrant: slit_id = slit.shutter_id slitdata = msa_data[slit_id] slitdata_model = get_slit_location_model(slitdata) msa_transform = slitdata_model | msa_model models.append(msa_transform) msa.close() return Slit2Msa(open_slits, models)
def get_spectral_order_wrange(input_model, wavelengthrange_file): """ Read the spectral order and wavelength range from the reference file. Parameters ---------- filter : str The filter used. grating : str The grating used in the observation. wavelength_range_file : str Reference file of type "wavelengthrange". """ full_range = [.6e-6, 5.3e-6] filter = input_model.meta.instrument.filter lamp = input_model.meta.instrument.lamp_state grating = input_model.meta.instrument.grating wave_range = AsdfFile.open(wavelengthrange_file) if filter == "OPAQUE": keyword = lamp + '_' + grating else: keyword = filter + '_' + grating try: order = wave_range.tree['filter_grating'][keyword]['order'] wrange = wave_range.tree['filter_grating'][keyword]['range'] except KeyError: order = -1 wrange = full_range log.warning("Combination {0} missing in wavelengthrange file, setting order to -1 and range to {1}.".format(keyword, full_range)) wave_range.close() return order, wrange
def get_wavelength_range(input_model, path=None): """ Return the wavelength range used for computing the WCS. Needs access to the reference file used to construct the WCS object. Parameters ---------- input_model : `jwst.datamodels.ImagingModel` Data model after assign_wcs has been run. path : str Directory where the reference file is. (optional) """ fname = input_model.meta.ref_file.wavelengthrange.name.split('/')[-1] if path is None and not os.path.exists(fname): raise IOError("Reference file {0} not found. Please specify a path.".format(fname)) else: fname = os.path.join(path, fname) f = AsdfFile.open(fname) wave_range = f.tree['wavelengthrange'].copy() wave_channels = f.tree['channels'] f.close() wr = dict(zip(wave_channels, wave_range)) channel = input_model.meta.instrument.channel band = input_model.meta.instrument.band return dict([(ch+band, wr[ch+band]) for ch in channel ])
def create_wavelengthrange_file(name): f = AsdfFile() #wavelengthrange = {'1SHORT': (4.88, 5.77), #'1MEDIUM': (5.64, 6.67), #'1LONG': (6.50, 7.70), #'2SHORT': (7.47, 8.83), #'2MEDIUM': (8.63, 10.19), #'2LONG': (9.96, 11.77), #'3SHORT': (11.49, 13.55), #'3MEDIUM': (13.28, 15.66), #'3LONG': (15.34, 18.09), #'4SHORT': (17.60, 21.00), #'4MEDIUM': (20.51, 24.48), #'4LONG': (23.92, 28.55) #} # Relaxing the range to match the distortion. The table above # comes from the report and is "as designed". wavelengthrange = {'1SHORT': (4.68, 5.97), '1MEDIUM': (5.24, 6.87), '1LONG': (6.2, 7.90), '2SHORT': (7.27, 9.03), '2MEDIUM': (8.43, 10.39), '2LONG': (9.76, 11.97), '3SHORT': (11.29, 13.75), '3MEDIUM': (13.08, 15.86), '3LONG': (15.14, 18.29), '4SHORT': (17.40, 21.20), '4MEDIUM': (20.31, 24.68), '4LONG': (23.72, 28.75) } channels = ['1SHORT', '1MEDIUM', '1LONG', '2SHORT', '2MEDIUM', '2LONG', '3SHORT', '3MEDIUM', '3LONG', '4SHORT', '4MEDIUM', '4LONG'] tree = { "instrument": "MIRI", "exp_type": "MIR_MRS", "pedigree": "GROUND", "title": "MIRI IFU model - based on CDP-4", "reftype": "WAVELENGTHRANGE", "author": "N. Dencheva" } tree['channels'] = channels f.tree = tree vr = np.empty((12, 2), dtype=np.float) for i, ch in enumerate(channels): vr[i] = wavelengthrange[ch] f.tree['wavelengthrange'] = vr f.write_to(name)
def create_miri_imager_filter_offset(distfile, outname): """ Create an asdf reference file with the filter offsets for the MIRI imager. Note: The IDT supplied distortion file lists sky to pixel as the forward transform. Since "forward" in the JWST pipeline is from pixel to sky, the offsets are taken with the opposite sign. Parameters ---------- distfile : str MIRI imager DISTORTION file provided by the IDT team. outname : str Name of reference file to be wriiten to disk. Returns ------- fasdf : AsdfFile AsdfFile object Examples ------- >>> create_miri_imager_filer_offset('MIRI_FM_MIRIMAGE_DISTORTION_03.02.00.fits', 'jwst_miri_filter_offset_0001.asdf') """ with fits.open(distfile) as f: data = f[9].data d = dict.fromkeys(data.field('FILTER')) for i in data: d[i[0]] = {'column_offset': -i[1], 'row_offset': -i[2]} tree = {"title": "MIRI imager filter offset - CDP4", "reftype": "FILTEROFFSET", "instrument": "MIRI", "detector": "MIRIMAGE", "pedigree": "GROUND", "author": "N. Dencheva", "exp_type": "MIR_IMAGE" } tree.update(d) f = AsdfFile() f.tree = tree f.write_to(outname)
def saveto(self, path, header=None): """ This function ... :param path: :param header: :return: """ # If a header is not specified, created it from the WCS if header is None: header = self.header # FITS format if path.endswith(".fits"): from .fits import write_frame # Import here because io imports SegmentationMap # Write to a FITS file write_frame(self._data, header, path) # ASDF format elif path.endswith(".asdf"): # Import from asdf import AsdfFile # Create the tree tree = dict() tree["data"] = self._data tree["header"] = header # Create the asdf file ff = AsdfFile(tree) # Write ff.write_to(path) # Invalid else: raise ValueError("Only the FITS or ASDF filetypes are supported") # Update the path self.path = path
def copy(self, memo=None): """ Returns a deep copy of the models in this model container. """ result = self.__class__(init=None, pass_invalid_values=self._pass_invalid_values, strict_validation=self._strict_validation) instance = copy.deepcopy(self._instance, memo=memo) result._asdf = AsdfFile(instance) result._instance = instance result._iscopy = self._iscopy result._schema = result._schema result._ctx = result for m in self._models: if isinstance(m, model_base.DataModel): result.append(m.copy()) else: result.append(m) return result
def create_wavelengthrange_file(name): f = AsdfFile() #wavelengthrange = {'1SHORT': (4.88, 5.77), #'1MEDIUM': (5.64, 6.67), #'1LONG': (6.50, 7.70), #'2SHORT': (7.47, 8.83), #'2MEDIUM': (8.63, 10.19), #'2LONG': (9.96, 11.77), #'3SHORT': (11.49, 13.55), #'3MEDIUM': (13.28, 15.66), #'3LONG': (15.34, 18.09), #'4SHORT': (17.60, 21.00), #'4MEDIUM': (20.51, 24.48), #'4LONG': (23.92, 28.55) #} # Relaxing the range to match the distortion. The table above # comes from the report and is "as designed". wavelengthrange = { '1SHORT': (4.68, 5.97), '1MEDIUM': (5.24, 6.87), '1LONG': (6.2, 7.90), '2SHORT': (7.27, 9.03), '2MEDIUM': (8.43, 10.39), '2LONG': (9.76, 11.97), '3SHORT': (11.29, 13.75), '3MEDIUM': (13.08, 15.86), '3LONG': (15.14, 18.29), '4SHORT': (17.40, 21.20), '4MEDIUM': (20.31, 24.68), '4LONG': (23.72, 28.75) } channels = [ '1SHORT', '1MEDIUM', '1LONG', '2SHORT', '2MEDIUM', '2LONG', '3SHORT', '3MEDIUM', '3LONG', '4SHORT', '4MEDIUM', '4LONG' ] tree = { "instrument": "MIRI", "exp_type": "MIR_MRS", "pedigree": "GROUND", "title": "MIRI IFU model - based on CDP-4", "reftype": "WAVELENGTHRANGE", "author": "N. Dencheva" } tree['channels'] = channels f.tree = tree vr = np.empty((12, 2), dtype=np.float) for i, ch in enumerate(channels): vr[i] = wavelengthrange[ch] f.tree['wavelengthrange'] = vr f.write_to(name)
def __init__(self, init=None, asn_exptypes=None, asn_n_members=None, **kwargs): super().__init__(init=None, asn_exptypes=None, **kwargs) self._models = [] self.asn_exptypes = asn_exptypes self.asn_n_members = asn_n_members self._memmap = kwargs.get("memmap", False) if init is None: # Don't populate the container with models pass elif isinstance(init, fits.HDUList): self._models.append([datamodel_open(init, memmap=self._memmap)]) elif isinstance(init, list): if all(isinstance(x, (str, fits.HDUList, model_base.DataModel)) for x in init): # Try opening the list as datamodels try: init = [datamodel_open(m, memmap=self._memmap) for m in init] except (FileNotFoundError, ValueError): raise else: raise TypeError("list must contain items that can be opened " "with jwst.datamodels.open()") self._models = init elif isinstance(init, self.__class__): instance = copy.deepcopy(init._instance) self._schema = init._schema self._shape = init._shape self._asdf = AsdfFile(instance) self._instance = instance self._ctx = self self.__class__ = init.__class__ self._models = init._models elif is_association(init): self.from_asn(init) elif isinstance(init, str): init_from_asn = self.read_asn(init) self.from_asn(init_from_asn, asn_file_path=init) else: raise TypeError('Input {0!r} is not a list of DataModels or ' 'an ASN file'.format(init))
def open_asdf(init=None, ignore_version_mismatch=True, ignore_unrecognized_tag=False, **kwargs): """ Open an asdf object from a filename or create a new asdf object """ if isinstance(init, str): if s3_utils.is_s3_uri(init): init = s3_utils.get_object(init) asdffile = asdf.open(init, ignore_version_mismatch=ignore_version_mismatch, ignore_unrecognized_tag=ignore_unrecognized_tag) else: asdffile = AsdfFile(init, ignore_version_mismatch=ignore_version_mismatch, ignore_unrecognized_tag=ignore_unrecognized_tag ) return asdffile
def msa_to_oteip(reference_files): """ Transform from the MSA frame to the OTEIP frame. Parameters ---------- reference_files: dict Dictionary with reference files returned by CRDS. Returns ------- model : `~astropy.modeling.core.Model` model. Transform from MSA to OTEIP. """ with AsdfFile.open(reference_files['fore']) as f: fore = f.tree['model'].copy() msa2fore_mapping = Mapping((0, 1, 2, 2), name='msa2fore_mapping') msa2fore_mapping.inverse = Identity(3) return msa2fore_mapping | (fore & Identity(1))
def test_window_orthopoly(tmpdir): model1d = astmodels.Chebyshev1D(2, c0=2, c1=3, c2=0.5, domain=[-2, 2], window=[-0.5, 0.5]) model2d = astmodels.Chebyshev2D(1, 1, c0_0=1, c0_1=2, c1_0=3, x_domain=[-2, 2], y_domain=[-2, 2], x_window=[-0.5, 0.5], y_window=[-0.1, 0.5]) fa = AsdfFile() fa.tree['model1d'] = model1d fa.tree['model2d'] = model2d file_path = str(tmpdir.join('orthopoly_window.asdf')) fa.write_to(file_path) with asdf.open(file_path) as f: assert f.tree['model1d'](1.8) == model1d(1.8) assert f.tree['model2d'](1.8, -1.5) == model2d(1.8, -1.5)
def oteip_to_v23(reference_files): """ Transform from the OTEIP frame to the V2V3 frame. Parameters ---------- reference_files: dict Dictionary with reference files returned by CRDS. Returns ------- model : `~astropy.modeling.core.Model` model. Transform from OTEIP to V2V3. """ with AsdfFile.open(reference_files['ote']) as f: ote = f.tree['model'].copy() fore2ote_mapping = Identity(3, name='fore2ote_mapping') fore2ote_mapping.inverse = Mapping((0, 1, 2, 2)) # Convert the wavelength to microns return fore2ote_mapping | (ote & Identity(1) / Const1D(1e-6))
def get_disperser(input_model, disperserfile): """ Return the disperser information corrected for the uncertainty in the GWA position. Parameters ---------- input_model : `jwst_lib.models.DataModel` The input data model - either an ImageModel or a CubeModel. disperserfile : str The name of the disperser reference file. Returns ------- disperser : dict The corrected disperser information. """ with AsdfFile.open(disperserfile) as f: disperser = f.tree xtilt = input_model.meta.instrument.gwa_xtilt ytilt = input_model.meta.instrument.gwa_ytilt disperser = correct_tilt(disperser, xtilt, ytilt) return disperser
def open_asdf(init=None, extensions=None, ignore_version_mismatch=True, ignore_unrecognized_tag=False, **kwargs): """ Open an asdf object from a filename or create a new asdf object """ if isinstance(init, str): asdffile = asdf.open( init, extensions=extensions, ignore_version_mismatch=ignore_version_mismatch, ignore_unrecognized_tag=ignore_unrecognized_tag) else: asdffile = AsdfFile( init, extensions=extensions, ignore_version_mismatch=ignore_version_mismatch, ignore_unrecognized_tag=ignore_unrecognized_tag) return asdffile
def runtest(self): standard_version = self._find_standard_version() # Make sure that the examples in the schema files (and thus the # ASDF standard document) are valid. buff = helpers.yaml_to_asdf('example: ' + self.example.strip(), standard_version=standard_version) ff = AsdfFile(uri=util.filepath_to_url(os.path.abspath(self.filename)), extensions=TestExtension()) # Fake an external file ff2 = AsdfFile({'data': np.empty((1024 * 1024 * 8), dtype=np.uint8)}) ff._external_asdf_by_uri[util.filepath_to_url( os.path.abspath( os.path.join(os.path.dirname(self.filename), 'external.asdf')))] = ff2 # Add some dummy blocks so that the ndarray examples work for i in range(3): b = block.Block(np.zeros((1024 * 1024 * 8), dtype=np.uint8)) b._used = True ff.blocks.add(b) b._array_storage = "streamed" try: with pytest.warns(None) as w: import warnings ff._open_impl(ff, buff) # Do not tolerate any warnings that occur during schema validation assert len(w) == 0, helpers.display_warnings(w) except: print("From file:", self.filename) raise # Just test we can write it out. A roundtrip test # wouldn't always yield the correct result, so those have # to be covered by "real" unit tests. if b'external.asdf' not in buff.getvalue(): buff = io.BytesIO() ff.write_to(buff)
def create_distortion_file(reftype, detector, band, channel, data, name, author, useafter, description): description = 'MIRI MRS Distortion Maps' tree = create_reffile_header(reftype, detector, band, channel, author, useafter, description) tree['filename'] = name adata, bdata, xdata, ydata, bzero, bdel = data tree['alpha_model'] = adata tree['beta_model'] = bdata tree['x_model'] = xdata tree['y_model'] = ydata tree['bzero'] = bzero tree['bdel'] = bdel f = AsdfFile() f.tree = tree f.add_history_entry( "DOCUMENT: MIRI-TN-00001-ETH; SOFTWARE: polyd2c_CDP5.pro; DATA USED: Data set of: - FM Test Campaign relevant to MRS-OPT-01, MRS-OPT-02, MRS-OPT-04, MRS-OPT-08; - CV1 Test Campaign relevant to MRS-OPT-02; - CV2 Test Campaign relevant to MRS-OPT-02; - Laboratory measurement of SPO; ============ DIFFERENCES: - New file structure: Change of Extention names and Table Column Headers.; - Replaced V2/V3 with XAN/YAN;" ) f.write_to(name)
def __init__(self, init=None, **kwargs): super(ModelContainer, self).__init__(init=None, **kwargs) self._models = [] if init is None: # Don't populate the container with models pass elif isinstance(init, fits.HDUList): self._models.append([datamodel_open(init)]) elif isinstance(init, list): if all(isinstance(x, (str, fits.HDUList)) for x in init): # Try opening the list of files as datamodels try: init = [datamodel_open(m) for m in init] except (FileNotFoundError, ValueError): raise elif not all(isinstance(x, model_base.DataModel) for x in init): raise TypeError('list must contain DataModels') self._models = init elif isinstance(init, self.__class__): instance = copy.deepcopy(init._instance) self._schema = init._schema self._shape = init._shape self._asdf = AsdfFile(instance) self._instance = instance self._ctx = self self.__class__ = init.__class__ self._models = init._models elif is_association(init): self.from_asn(init) elif isinstance(init, str): init_from_asn = self.read_asn(init) self.from_asn(init_from_asn, asn_file_path=init) else: raise TypeError('Input {0!r} is not a list of DataModels or ' 'an ASN file'.format(init))
def create_wavelengthrange_file(name, detector, author, useafter, description, outformat): f = AsdfFile() # Relaxing the range to match the distortion. The table above # comes from the report and is "as designed". wavelengthrange = { '1SHORT': (4.68, 5.97), '1MEDIUM': (5.24, 6.87), '1LONG': (6.2, 7.90), '2SHORT': (7.27, 9.03), '2MEDIUM': (8.43, 10.39), '2LONG': (9.76, 11.97), '3SHORT': (11.29, 13.75), '3MEDIUM': (13.08, 15.86), '3LONG': (15.14, 18.29), '4SHORT': (17.40, 21.20), '4MEDIUM': (20.31, 24.68), '4LONG': (23.72, 28.75) } channels = [ '1SHORT', '1MEDIUM', '1LONG', '2SHORT', '2MEDIUM', '2LONG', '3SHORT', '3MEDIUM', '3LONG', '4SHORT', '4MEDIUM', '4LONG' ] tree = create_reffile_header("WAVELENGTHRANGE", detector, band="N/A", channel="N/A", author=author, useafter=useafter, description=description) tree['filename'] = name tree['author'] = 'David Law' tree['detector'] = "N/A" tree['channels'] = channels f.tree = tree vr = np.empty((12, 2), dtype=np.float) for i, ch in enumerate(channels): vr[i] = wavelengthrange[ch] f.tree['wavelengthrange'] = vr # f.add_history_entry("DOCUMENT: MIRI-TN-00001-ETH; SOFTWARE: polyd2c_CDP5.pro; DATA USED: Data set of: - FM Test Campaign relevant to MRS-OPT-01, MRS-OPT-02, MRS-OPT-04, MRS-OPT-08; - CV1 Test Campaign relevant to MRS-OPT-02; - CV2 Test Campaign relevant to MRS-OPT-02; - Laboratory measurement of SPO; ============ DIFFERENCES: - New file structure: Change of Extention names and Table Column Headers.; - Replaced V2/V3 with XAN/YAN;") f.write_to(name) #,all_array_storage=outformat)
def msa2asdf(msafile, outname, ref_kw): """ Create an asdf reference file with the MSA description. mas2asfdf("MSA.msa", "msa.asdf") Parameters ---------- msafile : str A fits file with MSA description (MSA.msa) outname : str Name of output ASDF file. """ f = fits.open(msafile) tree = ref_kw.copy() data = f[5].data # SLITS and IFU header = f[5].header shiftx = models.Shift(header['SLITXREF'], name='slit_xref') shifty = models.Shift(header['SLITYREF'], name='slit_yref') slitrot = models.Rotation2D(header['SLITROT'], name='slit_rot') tree[5] = {} tree[5]['model'] = slitrot | shiftx & shifty tree[5]['data'] = f[5].data for i in range(1, 5): header = f[i].header shiftx = models.Shift(header['QUADXREF'], name='msa_xref') shifty = models.Shift(header['QUADYREF'], name='msa_yref') slitrot = models.Rotation2D(header['QUADROT'], name='msa_rot') tree[i] = {} tree[i]['model'] = slitrot | shiftx & shifty tree[i]['data'] = f[i].data f.close() fasdf = AsdfFile() fasdf.tree = tree fasdf.add_history_entry("Build 6") fasdf.write_to(outname) return fasdf
def oteip_to_v23(reference_files): """ Transform from the OTEIP frame to the V2V3 frame. Parameters ---------- reference_files: dict Dictionary with reference files returned by CRDS. Returns ------- model : `~astropy.modeling.core.Model` model. Transform from OTEIP to V2V3. """ with AsdfFile.open(reference_files['ote']) as f: ote = f.tree['model'].copy() fore2ote_mapping = Identity(3, name='fore2ote_mapping') fore2ote_mapping.inverse = Mapping((0, 1, 2, 2)) # Create the transform to v2/v3/lambda. The wavelength units up to this point are # meters as required by the pipeline but the desired output wavelength units is microns. # So we are going to Scale the spectral units by 1e6 (meters -> microns) return fore2ote_mapping | (ote & Scale(1e6))
class DataModel(properties.ObjectNode, ndmodel.NDModel): """ Base class of all of the data models. """ schema_url = None """ The schema URI to validate the model against. If None, only basic validation of required metadata properties (filename, date, model_type) will occur. """ def __init__(self, init=None, schema=None, memmap=False, pass_invalid_values=None, strict_validation=None, ignore_missing_extensions=True, **kwargs): """ Parameters ---------- init : str, tuple, `~astropy.io.fits.HDUList`, ndarray, dict, None - None : Create a default data model with no shape. - tuple : Shape of the data array. Initialize with empty data array with shape specified by the. - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - `~astropy.io.fits.HDUList` : Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : dict, str (optional) Tree of objects representing a JSON schema, or string naming a schema. The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. memmap : bool Turn memmap of FITS file on or off. (default: False). Ignored for ASDF files. pass_invalid_values : bool or None If `True`, values that do not validate the schema will be added to the metadata. If `False`, they will be set to `None`. If `None`, value will be taken from the environmental PASS_INVALID_VALUES. Otherwise the default value is `False`. strict_validation : bool or None If `True`, schema validation errors will generate an exception. If `False`, they will generate a warning. If `None`, value will be taken from the environmental STRICT_VALIDATION. Otherwise, the default value is `False`. ignore_missing_extensions : bool When `False`, raise warnings when a file is read that contains metadata about extensions that are not available. Defaults to `True`. kwargs : dict Additional keyword arguments passed to lower level functions. These arguments are generally file format-specific. Arguments of note are: - FITS skip_fits_update - bool or None `True` to skip updating the ASDF tree from the FITS headers, if possible. If `None`, value will be taken from the environmental SKIP_FITS_UPDATE. Otherwise, the default value is `True`. """ # Override value of validation parameters if not explicitly set. if pass_invalid_values is None: pass_invalid_values = get_envar_as_boolean("PASS_INVALID_VALUES", False) self._pass_invalid_values = pass_invalid_values if strict_validation is None: strict_validation = get_envar_as_boolean("STRICT_VALIDATION", False) self._strict_validation = strict_validation self._ignore_missing_extensions = ignore_missing_extensions kwargs.update({'ignore_missing_extensions': ignore_missing_extensions}) # Load the schema files if schema is None: if self.schema_url is None: schema = _DEFAULT_SCHEMA else: # Create an AsdfFile so we can use its resolver for loading schemas schema = asdf_schema.load_schema(self.schema_url, resolve_references=True) self._schema = mschema.merge_property_trees(schema) # Provide the object as context to other classes and functions self._ctx = self # Initialize with an empty AsdfFile instance as this is needed for # reading in FITS files where validate._check_value() gets called, and # ctx needs to have an _asdf attribute. self._asdf = AsdfFile() # Determine what kind of input we have (init) and execute the # proper code to intiailize the model self._files_to_close = [] self._iscopy = False is_array = False is_shape = False shape = None if init is None: asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, dict): asdffile = self.open_asdf(init=init, **kwargs) elif isinstance(init, np.ndarray): asdffile = self.open_asdf(init=None, **kwargs) shape = init.shape is_array = True elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init is_shape = True asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, DataModel): asdffile = None self.clone(self, init) if not isinstance(init, self.__class__): self.validate() return elif isinstance(init, AsdfFile): asdffile = init elif isinstance(init, fits.HDUList): asdffile = fits_support.from_fits(init, self._schema, self._ctx, **kwargs) elif isinstance(init, (str, bytes, PurePath)): if isinstance(init, PurePath): init = str(init) if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) file_type = filetype.check(init) if file_type == "fits": if s3_utils.is_s3_uri(init): init_fitsopen = s3_utils.get_object(init) memmap = None else: init_fitsopen = init hdulist = fits.open(init_fitsopen, memmap=memmap) asdffile = fits_support.from_fits(hdulist, self._schema, self._ctx, **kwargs) self._files_to_close.append(hdulist) elif file_type == "asdf": asdffile = self.open_asdf(init=init, **kwargs) else: # TODO handle json files as well raise IOError( "File does not appear to be a FITS or ASDF file.") else: raise ValueError("Can't initialize datamodel using {0}".format( str(type(init)))) # Initialize object fields as determined from the code above self._shape = shape self._instance = asdffile.tree self._asdf = asdffile # Initalize class dependent hidden fields self._no_asdf_extension = False # Instantiate the primary array of the image if is_array: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) # If a shape has been given, initialize the primary array. if is_shape: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Shape passed to DataModel.__init__, but model has " "no primary array in its schema") # Initialization occurs when the primary array is first # referenced. Do so now. getattr(self, primary_array_name) # if the input is from a file, set the filename attribute if isinstance(init, str): self.meta.filename = os.path.basename(init) elif isinstance(init, fits.HDUList): info = init.fileinfo(0) if info is not None: filename = info.get('filename') if filename is not None: self.meta.filename = os.path.basename(filename) # if the input model doesn't have a date set, use the current date/time if not self.meta.hasattr('date'): current_date = Time(datetime.datetime.now()) current_date.format = 'isot' self.meta.date = current_date.value # store the data model type, if not already set klass = self.__class__.__name__ if klass != 'DataModel': if not self.meta.hasattr('model_type'): self.meta.model_type = klass # initialize arrays from keyword arguments when they are present for attr, value in kwargs.items(): if value is not None: subschema = properties._get_schema_for_property( self._schema, attr) if 'datatype' in subschema: setattr(self, attr, value) @property def _model_type(self): return self.__class__.__name__ def __repr__(self): buf = ['<'] buf.append(self._model_type) if self.shape: buf.append(str(self.shape)) try: filename = self.meta.filename except AttributeError: filename = None if filename: buf.append(" from ") buf.append(filename) buf.append('>') return "".join(buf) def __del__(self): """Ensure closure of resources when deleted.""" self.close() @property def override_handle(self): """override_handle identifies in-memory models where a filepath would normally be used. """ # Arbitrary choice to look something like crds:// return "override://" + self.__class__.__name__ def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def _drop_arrays(self): def _drop_array(d): # Walk tree and delete numpy arrays if isinstance(d, dict): for val in d.values(): _drop_array(val) elif isinstance(d, list): for val in d: _drop_array(val) elif isinstance(d, np.ndarray): del d else: pass _drop_array(self._instance) def close(self): if not self._iscopy: if self._asdf is not None: self._asdf.close() self._drop_arrays() for fd in self._files_to_close: if fd is not None: fd.close() @staticmethod def clone(target, source, deepcopy=False, memo=None): if deepcopy: instance = copy.deepcopy(source._instance, memo=memo) target._asdf = AsdfFile(instance) target._instance = instance target._iscopy = source._iscopy else: target._asdf = source._asdf target._instance = source._instance target._iscopy = True target._files_to_close = [] target._shape = source._shape target._ctx = target target._no_asdf_extension = source._no_asdf_extension def copy(self, memo=None): """ Returns a deep copy of this model. """ result = self.__class__(init=None, pass_invalid_values=self._pass_invalid_values, strict_validation=self._strict_validation) self.clone(result, self, deepcopy=True, memo=memo) return result __copy__ = __deepcopy__ = copy def validate(self): """ Re-validate the model instance againsst its schema """ validate.value_change(str(self), self._instance, self._schema, self) def validate_required_fields(self): """ Walk the schema and make sure all required fields are in the model """ def callback(schema, path, combiner, ctx, recurse): if 'fits_required' not in schema: return # Get the value pointed at by the path to the node, # or None in case there is no entry for the node node = ctx for attr in path: node = getattr(node, attr) if node is None: break validate.value_change(path, node, schema, self) mschema.walk_schema(self._schema, callback, ctx=self) def info(self, *args, **kwargs): return self._asdf.info(**kwargs) def search(self, *args, **kwargs): return self._asdf.search(*args, **kwargs) try: info.__doc__ = AsdfFile.info.__doc__ search.__doc__ = AsdfFile.search.__doc__ except AttributeError: pass def get_primary_array_name(self): """ Returns the name "primary" array for this model, which controls the size of other arrays that are implicitly created. This is intended to be overridden in the subclasses if the primary array's name is not "data". """ if properties._find_property(self._schema, 'data'): primary_array_name = 'data' else: primary_array_name = '' return primary_array_name def on_save(self, path=None): """ This is a hook that is called just before saving the file. It can be used, for example, to update values in the metadata that are based on the content of the data. Override it in the subclass to make it do something, but don't forget to "chain up" to the base class, since it does things there, too. Parameters ---------- path : str The path to the file that we're about to save to. """ if isinstance(path, str): self.meta.filename = os.path.basename(path) current_date = Time(datetime.datetime.now()) current_date.format = 'isot' self.meta.date = current_date.value # Enforce model_type to be the actual type of model being saved. self.meta.model_type = self._model_type def save(self, path, dir_path=None, *args, **kwargs): """ Save to either a FITS or ASDF file, depending on the path. Parameters ---------- path : string or func File path to save to. If function, it takes one argument with is model.meta.filename and returns the full path string. dir_path: string Directory to save to. If not None, this will override any directory information in the `path` Returns ------- output_path: str The file path the model was saved in. """ if callable(path): path_head, path_tail = os.path.split(path(self.meta.filename)) else: path_head, path_tail = os.path.split(path) base, ext = os.path.splitext(path_tail) if isinstance(ext, bytes): ext = ext.decode(sys.getfilesystemencoding()) if dir_path: path_head = dir_path output_path = os.path.join(path_head, path_tail) # TODO: Support gzip-compressed fits if ext == '.fits': # TODO: remove 'clobber' check once depreciated fully in astropy if 'clobber' not in kwargs: kwargs.setdefault('overwrite', True) self.to_fits(output_path, *args, **kwargs) elif ext == '.asdf': self.to_asdf(output_path, *args, **kwargs) else: raise ValueError("unknown filetype {0}".format(ext)) return output_path @staticmethod def open_asdf(init=None, ignore_version_mismatch=True, ignore_unrecognized_tag=False, **kwargs): """ Open an asdf object from a filename or create a new asdf object """ if isinstance(init, str): if s3_utils.is_s3_uri(init): init = s3_utils.get_object(init) asdffile = asdf.open( init, ignore_version_mismatch=ignore_version_mismatch, ignore_unrecognized_tag=ignore_unrecognized_tag) else: asdffile = AsdfFile( init, ignore_version_mismatch=ignore_version_mismatch, ignore_unrecognized_tag=ignore_unrecognized_tag) return asdffile @classmethod def from_asdf(cls, init, schema=None, **kwargs): """ Load a data model from an ASDF file. Parameters ---------- init : str, file object, `~asdf.AsdfFile` - str : file path: initialize from the given file - readable file object: Initialize from the given file object - `~asdf.AsdfFile` : Initialize from the given`~asdf.AsdfFile`. schema : Same as for `__init__` kwargs : dict Aadditional arguments passed to lower level functions Returns ------- model : `~jwst.datamodels.DataModel` instance A data model. """ return cls(init, schema=schema, **kwargs) def to_asdf(self, init, *args, **kwargs): """ Write a data model to an ASDF file. Parameters ---------- init : file path or file object args : tuple, list Additional positional arguments passed to `~asdf.AsdfFile.write_to`. kwargs : dict Any additional keyword arguments are passed along to `~asdf.AsdfFile.write_to`. """ self.on_save(init) asdffile = self.open_asdf(self._instance, **kwargs) asdffile.write_to(init, *args, **kwargs) @classmethod def from_fits(cls, init, schema=None, **kwargs): """ Load a model from a FITS file. Parameters ---------- init : file path, file object, astropy.io.fits.HDUList - file path: Initialize from the given file - readable file object: Initialize from the given file object - astropy.io.fits.HDUList: Initialize from the given `~astropy.io.fits.HDUList`. schema : dict, str Same as for `__init__` kwargs : dict Aadditional arguments passed to lower level functions. Returns ------- model : `~jwst.datamodels.DataModel` A data model. """ return cls(init, schema=schema, **kwargs) def to_fits(self, init, *args, **kwargs): """ Write a data model to a FITS file. Parameters ---------- init : file path or file object args, kwargs Any additional arguments are passed along to `astropy.io.fits.writeto`. """ self.on_save(init) with fits_support.to_fits(self._instance, self._schema) as ff: with warnings.catch_warnings(): warnings.filterwarnings('ignore', message='Card is too long') if self._no_asdf_extension: ff._hdulist.writeto(init, *args, **kwargs) else: ff.write_to(init, *args, **kwargs) @property def shape(self): if self._shape is None: primary_array_name = self.get_primary_array_name() if primary_array_name and self.hasattr(primary_array_name): primary_array = getattr(self, primary_array_name) self._shape = primary_array.shape return self._shape def my_attribute(self, attr): properties = frozenset(("shape", "history", "_extra_fits", "schema")) return attr in properties def __setattr__(self, attr, value): if self.my_attribute(attr): object.__setattr__(self, attr, value) elif ndmodel.NDModel.my_attribute(self, attr): ndmodel.NDModel.__setattr__(self, attr, value) else: properties.ObjectNode.__setattr__(self, attr, value) def extend_schema(self, new_schema): """ Extend the model's schema using the given schema, by combining it in an "allOf" array. Parameters ---------- new_schema : dict Schema tree. """ schema = {'allOf': [self._schema, new_schema]} self._schema = mschema.merge_property_trees(schema) self.validate() return self def add_schema_entry(self, position, new_schema): """ Extend the model's schema by placing the given new_schema at the given dot-separated position in the tree. Parameters ---------- position : str Dot separated string indicating the position, e.g. ``meta.instrument.name``. new_schema : dict Schema tree. """ parts = position.split('.') schema = new_schema for part in parts[::-1]: schema = {'type': 'object', 'properties': {part: schema}} return self.extend_schema(schema) # return_result retained for backward compatibility def find_fits_keyword(self, keyword, return_result=True): """ Utility function to find a reference to a FITS keyword in this model's schema. This is intended for interactive use, and not for use within library code. Parameters ---------- keyword : str A FITS keyword name. Returns ------- locations : list of str If `return_result` is `True`, a list of the locations in the schema where this FITS keyword is used. Each element is a dot-separated path. """ from . import schema return schema.find_fits_keyword(self.schema, keyword) def search_schema(self, substring): """ Utility function to search the metadata schema for a particular phrase. This is intended for interactive use, and not for use within library code. The searching is case insensitive. Parameters ---------- substring : str The substring to search for. Returns ------- locations : list of tuples """ from . import schema return schema.search_schema(self.schema, substring) def __getitem__(self, key): """ Get a metadata value using a dotted name. """ assert isinstance(key, str) meta = self for part in key.split('.'): try: meta = getattr(meta, part) except AttributeError: raise KeyError(repr(key)) return meta def get_item_as_json_value(self, key): """ Equivalent to __getitem__, except returns the value as a JSON basic type, rather than an arbitrary Python type. """ assert isinstance(key, str) meta = self parts = key.split('.') for part in parts: try: meta = getattr(meta, part) except AttributeError: raise KeyError(repr(key)) return yamlutil.custom_tree_to_tagged_tree(meta, self._instance) def __setitem__(self, key, value): """ Set a metadata value using a dotted name. """ assert isinstance(key, str) meta = self parts = key.split('.') for part in parts[:-1]: try: part = int(part) except ValueError: try: meta = getattr(meta, part) except AttributeError: raise KeyError(repr(key)) else: meta = meta[part] part = parts[-1] try: part = int(part) except ValueError: setattr(meta, part, value) else: meta[part] = value def iteritems(self): """ Iterates over all of the schema items in a flat way. Each element is a pair (`key`, `value`). Each `key` is a dot-separated name. For example, the schema element `meta.observation.date` will end up in the result as:: ("meta.observation.date": "2012-04-22T03:22:05.432") """ def recurse(tree, path=[]): if isinstance(tree, dict): for key, val in tree.items(): for x in recurse(val, path + [key]): yield x elif isinstance(tree, (list, tuple)): for i, val in enumerate(tree): for x in recurse(val, path + [i]): yield x elif tree is not None: yield ('.'.join(str(x) for x in path), tree) for x in recurse(self._instance): yield x # We are just going to define the items to return the iteritems items = iteritems def iterkeys(self): """ Iterates over all of the schema keys in a flat way. Each result of the iterator is a `key`. Each `key` is a dot-separated name. For example, the schema element `meta.observation.date` will end up in the result as the string `"meta.observation.date"`. """ for key, val in self.iteritems(): yield key keys = iterkeys def itervalues(self): """ Iterates over all of the schema values in a flat way. """ for key, val in self.iteritems(): yield val values = itervalues def update(self, d, only=None, extra_fits=False): """ Updates this model with the metadata elements from another model. Note: The ``update`` method skips a WCS object, if present. Parameters ---------- d : `~jwst.datamodels.DataModel` or dictionary-like object The model to copy the metadata elements from. Can also be a dictionary or dictionary of dictionaries or lists. only: str, None Update only the named hdu, e.g. ``only='PRIMARY'``. Can either be a string or list of hdu names. Default is to update all the hdus. extra_fits : boolean Update from ``extra_fits``. Default is False. """ def hdu_keywords_from_data(d, path, hdu_keywords): # Walk tree and add paths to keywords to hdu keywords if isinstance(d, dict): for key, val in d.items(): if len(path) > 0 or key != 'extra_fits': hdu_keywords_from_data(val, path + [key], hdu_keywords) elif isinstance(d, list): for key, val in enumerate(d): hdu_keywords_from_data(val, path + [key], hdu_keywords) elif isinstance(d, np.ndarray): # skip data arrays pass else: hdu_keywords.append(path) def hdu_keywords_from_schema(subschema, path, combiner, ctx, recurse): # Add path to keyword to hdu_keywords if in list of hdu names if 'fits_keyword' in subschema: fits_hdu = subschema.get('fits_hdu', 'PRIMARY') if fits_hdu in hdu_names: ctx.append(path) def hdu_names_from_schema(subschema, path, combiner, ctx, recurse): # Build a set of hdu names from the schema hdu_name = subschema.get('fits_hdu') if hdu_name: hdu_names.add(hdu_name) def included(cursor, part): # Test if part is in the cursor if isinstance(part, int): return part >= 0 and part < len(cursor) else: return part in cursor def set_hdu_keyword(this_cursor, that_cursor, path): # Copy an element pointed to by path from that to this part = path.pop(0) if not included(that_cursor, part): return if len(path) == 0: this_cursor[part] = copy.deepcopy(that_cursor[part]) else: that_cursor = that_cursor[part] if not included(this_cursor, part): if isinstance(path[0], int): if isinstance(part, int): this_cursor.append([]) else: this_cursor[part] = [] else: if isinstance(part, int): this_cursor.append({}) elif isinstance(that_cursor, list): this_cursor[part] = [] else: this_cursor[part] = {} this_cursor = this_cursor[part] set_hdu_keyword(this_cursor, that_cursor, path) def protected_keyword(path): # Some keywords are protected and # should not be copied frpm the other image if len(path) == 2: if path[0] == 'meta': if path[1] in ('date', 'model_type'): return True return False # Get the list of hdu names from the model so that updates # are limited to those hdus if only is not None: if isinstance(only, str): hdu_names = set([only]) else: hdu_names = set(list(only)) else: hdu_names = set(['PRIMARY']) mschema.walk_schema(self._schema, hdu_names_from_schema, hdu_names) # Get the paths to all the keywords that will be updated from hdu_keywords = [] if isinstance(d, DataModel): schema = d._schema d = d._instance mschema.walk_schema(schema, hdu_keywords_from_schema, hdu_keywords) else: path = [] hdu_keywords_from_data(d, path, hdu_keywords) # Perform the updates to the keywords mentioned in the schema for path in hdu_keywords: if not protected_keyword(path): set_hdu_keyword(self._instance, d, path) # Update from extra_fits as well, if indicated if extra_fits: for hdu_name in hdu_names: path = ['extra_fits', hdu_name, 'header'] set_hdu_keyword(self._instance, d, path) self.validate() def to_flat_dict(self, include_arrays=True): """ Returns a dictionary of all of the schema items as a flat dictionary. Each dictionary key is a dot-separated name. For example, the schema element `meta.observation.date` will end up in the dictionary as:: { "meta.observation.date": "2012-04-22T03:22:05.432" } """ def convert_val(val): if isinstance(val, datetime.datetime): return val.isoformat() elif isinstance(val, Time): return str(val) return val if include_arrays: return dict( (key, convert_val(val)) for (key, val) in self.iteritems()) else: return dict((key, convert_val(val)) for (key, val) in self.iteritems() if not isinstance(val, np.ndarray)) @property def schema(self): return self._schema def get_fileext(self): return 'fits' # TODO: This is just here for backward compatibility @property def _extra_fits(self): return self.extra_fits # TODO: For backward compatibility def get_section(self, name): return getattr(self, name) @property def history(self): """ Get the history as a list of entries """ return HistoryList(self._asdf) @history.setter def history(self, values): """ Set a history entry. Parameters ---------- values : list For FITS files this should be a list of strings. For ASDF files use a list of ``HistoryEntry`` object. It can be created with `~jwst.datamodels.util.create_history_entry`. """ entries = self.history entries.clear() entries.extend(values) def get_fits_wcs(self, hdu_name='SCI', hdu_ver=1, key=' '): """ Get a `astropy.wcs.WCS` object created from the FITS WCS information in the model. Note that modifying the returned WCS object will not modify the data in this model. To update the model, use `set_fits_wcs`. Parameters ---------- hdu_name : str, optional The name of the HDU to get the WCS from. This must use named HDU's, not numerical order HDUs. To get the primary HDU, pass ``'PRIMARY'``. key : str, optional The name of a particular WCS transform to use. This may be either ``' '`` or ``'A'``-``'Z'`` and corresponds to the ``"a"`` part of the ``CTYPEia`` cards. *key* may only be provided if *header* is also provided. hdu_ver: int, optional The extension version. Used when there is more than one extension with the same name. The default value, 1, is the first. Returns ------- wcs : `astropy.wcs.WCS` or `pywcs.WCS` object The type will depend on what libraries are installed on this system. """ ff = fits_support.to_fits(self._instance, self._schema) hdu = fits_support.get_hdu(ff._hdulist, hdu_name, index=hdu_ver - 1) header = hdu.header return WCS(header, key=key, relax=True, fix=True) def set_fits_wcs(self, wcs, hdu_name='SCI'): """ Sets the FITS WCS information on the model using the given `astropy.wcs.WCS` object. Note that the "key" of the WCS is stored in the WCS object itself, so it can not be set as a parameter to this method. Parameters ---------- wcs : `astropy.wcs.WCS` or `pywcs.WCS` object The object containing FITS WCS information hdu_name : str, optional The name of the HDU to set the WCS from. This must use named HDU's, not numerical order HDUs. To set the primary HDU, pass ``'PRIMARY'``. """ header = wcs.to_header() if hdu_name == 'PRIMARY': hdu = fits.PrimaryHDU(header=header) else: hdu = fits.ImageHDU(name=hdu_name, header=header) hdulist = fits.HDUList([hdu]) ff = fits_support.from_fits( hdulist, self._schema, self._ctx, ignore_missing_extensions=self._ignore_missing_extensions) self._instance = properties.merge_tree(self._instance, ff.tree) # -------------------------------------------------------- # These two method aliases are here for astropy.registry # compatibility and should not be called directly # -------------------------------------------------------- read = __init__ def write(self, path, *args, **kwargs): self.save(path, *args, **kwargs) def getarray_noinit(self, attribute): """Retrieve array but without initilization Arrays initialize when directly referenced if they had not previously been initialized. This circumvents the initialization and instead raises `AttributeError`. Parameters ---------- attribute : str The attribute to retrieve. Returns ------- value : object The value of the attribute. Raises ------ AttributeError If the attribute does not exist. """ if attribute in self.instance: return getattr(self, attribute) raise AttributeError(f'{self} has no attribute "{attribute}"')
def niriss_soss(input_model, reference_files): """ The NIRISS SOSS WCS pipeline. It includes tWO coordinate frames - "detector" and "world". It uses the "specwcs" reference file. """ # Get the target RA and DEC, they will be used for setting the WCS RA # and DEC based on a conversation with Kevin Volk. try: target_ra = float(input_model['meta.target.ra']) target_dec = float(input_model['meta.target.dec']) except: # There was an error getting the target RA and DEC, so we are not going to continue. raise ValueError( 'Problem getting the TARG_RA or TARG_DEC from input model {}'. format(input_model)) # Define the frames detector = cf.Frame2D(name='detector', axes_order=(0, 1), unit=(u.pix, u.pix)) spec = cf.SpectralFrame(name='spectral', axes_order=(2, ), unit=(u.micron, ), axes_names=('wavelength', )) sky = cf.CelestialFrame(reference_frame=coord.ICRS(), axes_names=('ra', 'dec'), axes_order=(0, 1), unit=(u.deg, u.deg), name='sky') world = cf.CompositeFrame([sky, spec], name='world') try: with AsdfFile.open(reference_files['specwcs']) as wl: wl1 = wl.tree[1].copy() wl2 = wl.tree[2].copy() wl3 = wl.tree[3].copy() except Exception as e: raise IOError('Error reading wavelength correction from {}'.format( reference_files['specwcs'])) try: velosys = input_model.meta.wcsinfo.velosys except AttributeError: pass else: if velosys is not None: velocity_corr = velocity_correction( input_model.meta.wcsinfo.velosys) wl1 = wl1 | velocity_corr wl2 = wl2 | velocity_corr wl2 = wl3 | velocity_corr log.info("Applied Barycentric velocity correction: {}".format( velocity_corr[1].amplitude.value)) subarray2full = subarray_transform(input_model) # Reverse the order of inputs passed to Tabular because it's in python order in modeling. # Consider changing it in modelng ? cm_order1 = subarray2full | (Mapping((0, 1, 1, 0)) | \ (Const1D(target_ra) & Const1D(target_dec) & wl1) ).rename('Order1') cm_order2 = subarray2full | (Mapping((0, 1, 1, 0)) | \ (Const1D(target_ra) & Const1D(target_dec) & wl2) ).rename('Order2') cm_order3 = subarray2full | (Mapping((0, 1, 1, 0)) | \ (Const1D(target_ra) & Const1D(target_dec) & wl3) ).rename('Order3') # Define the transforms, they should accept (x,y) and return (ra, dec, lambda) soss_model = NirissSOSSModel( [1, 2, 3], [cm_order1, cm_order2, cm_order3]).rename('3-order SOSS Model') # Define the pipeline based on the frames and models above. pipeline = [(detector, soss_model), (world, None)] return pipeline
def __init__(self, init=None, schema=None, extensions=None, pass_invalid_values=False): """ Parameters ---------- init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None - None: A default data model with no shape - shape tuple: Initialize with empty data of the given shape - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - ``astropy.io.fits.HDUList``: Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : tree of objects representing a JSON schema, or string naming a schema, optional The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. extensions: classes extending the standard set of extensions, optional. If an extension is defined, the prefix used should be 'url'. pass_invalid_values: If true, values that do not validate the schema can be read and written and only a warning will be generated """ # Set the extensions if extensions is None: extensions = jwst_extensions[:] else: extensions.extend(jwst_extensions) self._extensions = extensions # Override value of pass_invalid value if environment value set if "PASS_INVALID_VALUES" in os.environ: pass_invalid_values = os.environ["PASS_INVALID_VALUES"] try: pass_invalid_values = bool(int(pass_invalid_values)) except ValueError: pass_invalid_values = False self._pass_invalid_values = pass_invalid_values # Construct the path to the schema files filename = os.path.abspath(inspect.getfile(self.__class__)) base_url = os.path.join(os.path.dirname(filename), 'schemas', '') # Load the schema files if schema is None: schema_path = os.path.join(base_url, self.schema_url) extension_list = asdf_extension.AsdfExtensionList(self._extensions) schema = asdf_schema.load_schema( schema_path, resolver=extension_list.url_mapping, resolve_references=True) self._schema = mschema.flatten_combiners(schema) # Determine what kind of input we have (init) and execute the # proper code to intiailize the model self._files_to_close = [] self._iscopy = False is_array = False is_shape = False shape = None if init is None: asdf = AsdfFile(extensions=extensions) elif isinstance(init, dict): asdf = AsdfFile(init, extensions=extensions) elif isinstance(init, np.ndarray): asdf = AsdfFile(extensions=extensions) shape = init.shape is_array = True elif isinstance(init, self.__class__): self.clone(self, init) return elif isinstance(init, DataModel): raise TypeError( "Passed in {0!r} is not of the expected subclass {1!r}".format( init.__class__.__name__, self.__class__.__name__)) elif isinstance(init, AsdfFile): asdf = init elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init asdf = AsdfFile() is_shape = True elif isinstance(init, fits.HDUList): asdf = fits_support.from_fits(init, self._schema, extensions, pass_invalid_values) elif isinstance(init, (six.string_types, bytes)): if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) file_type = filetype.check(init) if file_type == "fits": hdulist = fits.open(init) asdf = fits_support.from_fits(hdulist, self._schema, extensions, pass_invalid_values) self._files_to_close.append(hdulist) elif file_type == "asdf": asdf = AsdfFile.open(init, extensions=extensions) else: # TODO handle json files as well raise IOError( "File does not appear to be a FITS or ASDF file.") else: raise ValueError("Can't initialize datamodel using {0}".format( str(type(init)))) # Initialize object fields as determined fro the code above self._shape = shape self._instance = asdf.tree self._asdf = asdf self._ctx = self # if the input is from a file, set the filename attribute if isinstance(init, six.string_types): self.meta.filename = os.path.basename(init) elif isinstance(init, fits.HDUList): info = init.fileinfo(0) if info is not None: filename = info.get('filename') if filename is not None: self.meta.filename = os.path.basename(filename) # if the input model doesn't have a date set, use the current date/time if self.meta.date is None: self.meta.date = Time(datetime.datetime.now()) if hasattr(self.meta.date, 'value'): self.meta.date.format = 'isot' self.meta.date = str(self.meta.date.value) # store the data model type, if not already set if hasattr(self.meta, 'model_type'): if self.meta.model_type is None: self.meta.model_type = self.__class__.__name__ else: self.meta.model_type = None if is_array: primary_array_name = self.get_primary_array_name() if primary_array_name is None: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) # TODO this code looks useless if is_shape: getattr(self, self.get_primary_array_name())
def __init__(self, init=None, schema=None, extensions=None, pass_invalid_values=False): """ Parameters ---------- init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None - None: A default data model with no shape - shape tuple: Initialize with empty data of the given shape - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - ``astropy.io.fits.HDUList``: Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : tree of objects representing a JSON schema, or string naming a schema, optional The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. extensions: classes extending the standard set of extensions pass_invalid_values: If True, values that do not validate the schema can be read and written, but with a warning message """ filename = os.path.abspath(inspect.getfile(self.__class__)) base_url = os.path.join(os.path.dirname(filename), 'schemas', '') if schema is None: schema_path = os.path.join(base_url, self.schema_url) schema = asdf_schema.load_schema(schema_path, resolve_references=True) self._schema = mschema.flatten_combiners(schema) if extensions is not None: extensions.extend(jwst_extensions) else: extensions = jwst_extensions[:] self._extensions = extensions if "PASS_INVALID_VALUES" in os.environ: pass_invalid_values = os.environ["PASS_INVALID_VALUES"] try: self._pass_invalid_values = bool(int(pass_invalid_values)) except ValueError: self._pass_invalid_values = False else: self._pass_invalid_values = pass_invalid_values self._files_to_close = [] is_array = False is_shape = False shape = None if init is None: asdf = AsdfFile(extensions=extensions) elif isinstance(init, dict): asdf = AsdfFile(init, extensions=extensions) elif isinstance(init, np.ndarray): asdf = AsdfFile(extensions=extensions) shape = init.shape is_array = True elif isinstance(init, self.__class__): instance = copy.deepcopy(init._instance) self._schema = init._schema self._shape = init._shape self._asdf = AsdfFile(instance, extensions=self._extensions) self._instance = instance self._ctx = self self.__class__ = init.__class__ return elif isinstance(init, DataModel): raise TypeError( "Passed in {0!r} is not of the expected subclass {1!r}".format( init.__class__.__name__, self.__class__.__name__)) elif isinstance(init, AsdfFile): asdf = init elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init asdf = AsdfFile() is_shape = True elif isinstance(init, fits.HDUList): asdf = fits_support.from_fits( init, self._schema, extensions=self._extensions, validate=False, pass_invalid_values=self._pass_invalid_values) elif isinstance(init, six.string_types): if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) try: hdulist = fits.open(init) except IOError: try: asdf = AsdfFile.open(init, extensions=self._extensions) # TODO: Add json support except ValueError: raise IOError( "File does not appear to be a FITS or ASDF file.") else: asdf = fits_support.from_fits( hdulist, self._schema, extensions=self._extensions, validate=False, pass_invalid_values=self._pass_invalid_values) self._files_to_close.append(hdulist) self._shape = shape self._instance = asdf.tree self._asdf = asdf self._ctx = self # if the input model doesn't have a date set, use the current date/time if self.meta.date is None: self.meta.date = Time(datetime.datetime.now()) self.meta.date.format = 'isot' self.meta.date = self.meta.date.value # if the input is from a file, set the filename attribute if isinstance(init, six.string_types): self.meta.filename = os.path.basename(init) if is_array: primary_array_name = self.get_primary_array_name() if primary_array_name is None: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) if is_shape: getattr(self, self.get_primary_array_name())
def __init__(self, init=None, schema=None, extensions=None, pass_invalid_values=False, strict_validation=False): """ Parameters ---------- init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None - None: A default data model with no shape - shape tuple: Initialize with empty data of the given shape - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - ``astropy.io.fits.HDUList``: Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : tree of objects representing a JSON schema, or string naming a schema, optional The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. extensions: classes extending the standard set of extensions, optional. If an extension is defined, the prefix used should be 'url'. pass_invalid_values: If true, values that do not validate the schema will be added to the metadata. If false, they will be set to None strict_validation: if true, an schema validation errors will generate an excption. If false, they will generate a warning. """ # Set the extensions self._extensions = extensions # Override value of validation parameters # if environment value set self._pass_invalid_values = self.get_envar("PASS_INVALID_VALUES", pass_invalid_values) self._strict_validation = self.get_envar("STRICT_VALIDATION", strict_validation) # Construct the path to the schema files filename = os.path.abspath(inspect.getfile(self.__class__)) base_url = os.path.join(os.path.dirname(filename), 'schemas', '') # Load the schema files if schema is None: schema_path = os.path.join(base_url, self.schema_url) # Create an AsdfFile so we can use its resolver for loading schemas asdf_file = AsdfFile(extensions=self._extensions) if hasattr(asdf_file, 'resolver'): file_resolver = asdf_file.resolver else: file_resolver = self.get_resolver(asdf_file) schema = asdf_schema.load_schema(schema_path, resolver=file_resolver, resolve_references=True) self._schema = mschema.flatten_combiners(schema) # Provide the object as context to other classes and functions self._ctx = self # Determine what kind of input we have (init) and execute the # proper code to intiailize the model self._files_to_close = [] self._iscopy = False is_array = False is_shape = False shape = None if init is None: asdf = AsdfFile(extensions=extensions) elif isinstance(init, dict): asdf = AsdfFile(init, extensions=self._extensions) elif isinstance(init, np.ndarray): asdf = AsdfFile(extensions=self._extensions) shape = init.shape is_array = True elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init asdf = AsdfFile() is_shape = True elif isinstance(init, DataModel): self.clone(self, init) if not isinstance(init, self.__class__): self.validate() return elif isinstance(init, AsdfFile): asdf = init elif isinstance(init, fits.HDUList): asdf = fits_support.from_fits(init, self._schema, self._extensions, self._ctx) elif isinstance(init, (str, bytes)): if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) file_type = filetype.check(init) if file_type == "fits": hdulist = fits.open(init) asdf = fits_support.from_fits(hdulist, self._schema, self._extensions, self._ctx) self._files_to_close.append(hdulist) elif file_type == "asdf": asdf = AsdfFile.open(init, extensions=self._extensions) else: # TODO handle json files as well raise IOError( "File does not appear to be a FITS or ASDF file.") else: raise ValueError("Can't initialize datamodel using {0}".format( str(type(init)))) # Initialize object fields as determined from the code above self._shape = shape self._instance = asdf.tree self._asdf = asdf # Initalize class dependent hidden fields self._no_asdf_extension = False # Instantiate the primary array of the image if is_array: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) if is_shape: if not self.get_primary_array_name(): raise TypeError( "Shape passed to DataModel.__init__, but model has " "no primary array in its schema") # if the input is from a file, set the filename attribute if isinstance(init, str): self.meta.filename = os.path.basename(init) elif isinstance(init, fits.HDUList): info = init.fileinfo(0) if info is not None: filename = info.get('filename') if filename is not None: self.meta.filename = os.path.basename(filename) # if the input model doesn't have a date set, use the current date/time if not self.meta.hasattr('date'): current_date = Time(datetime.datetime.now()) current_date.format = 'isot' self.meta.date = current_date.value # store the data model type, if not already set klass = self.__class__.__name__ if klass != 'DataModel': if not self.meta.hasattr('model_type'): self.meta.model_type = klass
def test_custom_and_analytical(model): fa = AsdfFile() fa.tree['model'] = model fa.write_to('custom_and_analytical_inverse.asdf') f = asdf.open('custom_and_analytical_inverse.asdf') assert f.tree['model'].inverse is not None
def parse(*, raw_dir: str, uasts_dir: str, configs_dir: str, log_level: str) -> None: """Parse a CodRep 2019 dataset into UASTs.""" Config.from_arguments(locals(), ["raw_dir", "uasts_dir"], "configs_dir").save( Path(configs_dir) / "parse.json") logger = setup_logging(__name__, log_level) raw_dir_path = Path(raw_dir).expanduser().resolve() uasts_dir_path = Path(uasts_dir).expanduser().resolve() uasts_dir_path.mkdir(parents=True, exist_ok=True) parser = JavaParser(split_formatting=True) logger.info("Parsing %s", raw_dir_path) labels_file = raw_dir_path / "out.txt" extract_labels = labels_file.is_file() if extract_labels: error_offsets = {} for i, line in enumerate(labels_file.open("r", encoding="utf8")): error_offsets["%d.txt" % i] = int(line) - 1 for file_path in raw_dir_path.rglob("*.txt"): if file_path.samefile(labels_file): continue file_path_relative = file_path.relative_to(raw_dir_path) start = time() logger.debug("Parsing %s", file_path_relative) try: nodes = parser.parse(raw_dir_path, file_path_relative) except ParsingException: continue logger.debug( "Parsed %s into %d nodes in %.2fms", file_path_relative, len(nodes.nodes), (time() - start) * 1000, ) error_node_index = None if extract_labels: error_offset = error_offsets[file_path.name] for formatting_i, i in enumerate(nodes.formatting_indexes): node = nodes.nodes[i] if node.start == error_offset: error_node_index = formatting_i break else: logger.warning( "Could not retrieve a formatting node for the error at offset %d " "of file %s.", error_offset, file_path.with_suffix("").name, ) continue codrep_label = CodRepLabel( error_index=error_node_index, n_formatting_nodes=len(nodes.formatting_indexes), ) output_subdirectory = uasts_dir_path / file_path_relative.parent output_subdirectory.mkdir(parents=True, exist_ok=True) with (output_subdirectory / file_path.with_suffix(".asdf").name).open("wb") as fh: af = AsdfFile( dict( nodes=nodes.to_tree(file_path.read_text(encoding="utf-8")), codrep_label=codrep_label.to_tree(), filepath=str(file_path_relative), )) af.write_to(fh, all_array_compression="bzp2")
def _check_type(validator, types, instance, schema): """ Callback to check data type. Skips over null values. """ if instance is None: errors = [] else: errors = asdf_schema.validate_type(validator, types, instance, schema) return errors validator_callbacks = HashableDict(asdf_schema.YAML_VALIDATORS) validator_callbacks.update({'type': _check_type}) validator_context = AsdfFile() validator_resolver = validator_context.resolver def _check_value(value, schema): """ Perform the actual validation. """ if value is None: if schema.get('fits_required'): name = schema.get("fits_keyword") or schema.get("fits_hdu") raise jsonschema.ValidationError("%s is a required value" % name) else: temp_schema = { '$schema':
def run(self): filename = self.arguments[0] cwd = os.getcwd() os.chdir(TMPDIR) parts = [] try: ff = AsdfFile() code = AsdfFile._open_impl(ff, filename, _get_yaml_content=True) code = '{0} {1}\n'.format( ASDF_MAGIC, version_string) + code.strip().decode('utf-8') literal = nodes.literal_block(code, code) literal['language'] = 'yaml' set_source_info(self, literal) parts.append(literal) with AsdfFile.open(filename) as ff: for i, block in enumerate(ff.blocks.internal_blocks): data = codecs.encode(block.data.tostring(), 'hex') if len(data) > 40: data = data[:40] + '...'.encode() allocated = block._allocated size = block._size data_size = block._data_size flags = block._flags if flags & BLOCK_FLAG_STREAMED: allocated = size = data_size = 0 lines = [] lines.append('BLOCK {0}:'.format(i)) human_flags = [] for key, val in FLAGS.items(): if flags & key: human_flags.append(val) if len(human_flags): lines.append(' flags: {0}'.format( ' | '.join(human_flags))) if block.input_compression: lines.append(' compression: {0}'.format( block.input_compression)) lines.append(' allocated_size: {0}'.format(allocated)) lines.append(' used_size: {0}'.format(size)) lines.append(' data_size: {0}'.format(data_size)) lines.append(' data: {0}'.format(data)) code = '\n'.join(lines) literal = nodes.literal_block(code, code) literal['language'] = 'yaml' set_source_info(self, literal) parts.append(literal) internal_blocks = list(ff.blocks.internal_blocks) if (len(internal_blocks) and internal_blocks[-1].array_storage != 'streamed'): buff = io.BytesIO() ff.blocks.write_block_index(buff, ff) block_index = buff.getvalue() literal = nodes.literal_block(block_index, block_index) literal['language'] = 'yaml' set_source_info(self, literal) parts.append(literal) finally: os.chdir(cwd) result = nodes.admonition() textnodes, messages = self.state.inline_text(filename, self.lineno) title = nodes.title(filename, '', *textnodes) result += title result += parts return [result]
def dist_coeff(): """ Create distortion correction reference file. Parameters: ___________ coefficients: float coefficients The order of the input text files with coefficients is from det2sky and then sky2det. """ "Put coefficients into variables by reading in text files" coeffxr, coeffyr = np.loadtxt(sys.argv[1], skiprows=6, usecols=(1, 2), unpack=True) print(coeffxr) print(coeffyr) coeffxi, coeffyi = np.loadtxt(sys.argv[2], skiprows=6, usecols=(1, 2), unpack=True) print(coeffxi) print(coeffyi) "Transform coefficients to get undistorted, real, sky x." x0_0r = coeffxr[0] print('Coefficient for x0_0: ', x0_0r) x1_0r = coeffxr[1] print('Coefficient for x1_0: ', x1_0r) x0_1r = coeffxr[2] print('Coefficient for x0_1: ', x0_1r) x2_0r = coeffxr[3] print('Coefficient for x2_0: ', x2_0r) x1_1r = coeffxr[4] print('Coefficient for x1_1: ', x1_1r) x0_2r = coeffxr[5] print('Coefficient for x0_2: ', x0_2r) x3_0r = coeffxr[6] print('Coefficient for x3_0: ', x3_0r) x2_1r = coeffxr[7] print('Coefficient for x2_1: ', x2_1r) x1_2r = coeffxr[8] print('Coefficient for x1_2: ', x1_2r) x0_3r = coeffxr[9] print('Coefficient for x0_3: ', x0_3r) x4_0r = coeffxr[10] print('Coefficient for x4_0: ', x4_0r) x3_1r = coeffxr[11] print('Coefficient for x3_1: ', x3_1r) x2_2r = coeffxr[12] print('Coefficient for x2_2: ', x2_2r) x1_3r = coeffxr[13] print('Coefficient for x1_3: ', x1_3r) x0_4r = coeffxr[14] print('Coefficient for x0_4: ', x0_4r) "Transform coefficients to get undistorted, real, sky y." y0_0r = coeffyr[0] print('Coefficient for y0_0: ', y0_0r) y1_0r = coeffyr[1] print('Coefficient for y1_0: ', y1_0r) y0_1r = coeffyr[2] print('Coefficient for y0_1: ', y0_1r) y2_0r = coeffyr[3] print('Coefficient for y2_0: ', y2_0r) y1_1r = coeffyr[4] print('Coefficient for y1_1: ', y1_1r) y0_2r = coeffyr[5] print('Coefficient for y0_2: ', y0_2r) y3_0r = coeffyr[6] print('Coefficient for y3_0: ', y3_0r) y2_1r = coeffyr[7] print('Coefficient for y2_1: ', y2_1r) y1_2r = coeffyr[8] print('Coefficient for y1_2: ', y1_2r) y0_3r = coeffyr[9] print('Coefficient for y0_3: ', y0_3r) y4_0r = coeffyr[10] print('Coefficient for y4_0: ', y4_0r) y3_1r = coeffyr[11] print('Coefficient for y3_1: ', y3_1r) y2_2r = coeffyr[12] print('Coefficient for y2_2: ', y2_2r) y1_3r = coeffyr[13] print('Coefficient for y1_3: ', y1_3r) y0_4r = coeffyr[14] print('Coefficient for y0_4: ', y0_4r) "Transform coefficients to get distorted, ideal, detector x." x0_0i = coeffxi[0] print('Coefficient for x0_0: ', x0_0i) x1_0i = coeffxi[1] print('Coefficient for x1_0: ', x1_0i) x0_1i = coeffxi[2] print('Coefficient for x0_1: ', x0_1i) x2_0i = coeffxi[3] print('Coefficient for x2_0: ', x2_0i) x1_1i = coeffxi[4] print('Coefficient for x1_1: ', x1_1i) x0_2i = coeffxi[5] print('Coefficient for x0_2: ', x0_2i) x3_0i = coeffxi[6] print('Coefficient for x3_0: ', x3_0i) x2_1i = coeffxi[7] print('Coefficient for x2_1: ', x2_1i) x1_2i = coeffxi[8] print('Coefficient for x1_2: ', x1_2i) x0_3i = coeffxi[9] print('Coefficient for x0_3: ', x0_3i) x4_0i = coeffxi[10] print('Coefficient for x4_0: ', x4_0i) x3_1i = coeffxi[11] print('Coefficient for x3_1: ', x3_1i) x2_2i = coeffxi[12] print('Coefficient for x2_2: ', x2_2i) x1_3i = coeffxi[13] print('Coefficient for x1_3: ', x1_3i) x0_4i = coeffxi[14] print('Coefficient for x0_4: ', x0_4i) "Transform coefficients to get distorted, ideal, detector y." y0_0i = coeffyi[0] print('Coefficient for y0_0: ', y0_0i) y1_0i = coeffyi[1] print('Coefficient for y1_0: ', y1_0i) y0_1i = coeffyi[2] print('Coefficient for y0_1: ', y0_1i) y2_0i = coeffyi[3] print('Coefficient for y2_0: ', y2_0i) y1_1i = coeffyi[4] print('Coefficient for y1_1: ', y1_1i) y0_2i = coeffyi[5] print('Coefficient for y0_2: ', y0_2i) y3_0i = coeffyi[6] print('Coefficient for y3_0: ', y3_0i) y2_1i = coeffyi[7] print('Coefficient for y2_1: ', y2_1i) y1_2i = coeffyi[8] print('Coefficient for y1_2: ', y1_2i) y0_3i = coeffyi[9] print('Coefficient for y0_3: ', y0_3i) y4_0i = coeffyi[10] print('Coefficient for y4_0: ', y4_0i) y3_1i = coeffyi[11] print('Coefficient for y3_1: ', y3_1i) y2_2i = coeffyi[12] print('Coefficient for y2_2: ', y2_2i) y1_3i = coeffyi[13] print('Coefficient for y1_3: ', y1_3i) y0_4i = coeffyi[14] print('Coefficient for y0_4: ', y0_4i) "Generate ideal or detector coordinates." x_pix = np.arange(0, 2048, 1) y_pix = np.arange(0, 2048, 1) x_det_orig = [] y_det_orig = [] for x, y in itertools.product(x_pix, y_pix): x_det_orig.append(x) y_det_orig.append(y) print('Done with first for loop!!!!') print(len(x_det_orig)) print(len(y_det_orig)) "To go from ideal, distorted to real, undistorted." x_sky = [] y_sky = [] #------- for x, y in zip(x_det_orig, y_det_orig): x_real, y_real = apply_coeff(x, y, coeffxr, coeffyr) x_sky.append(x_real) y_sky.append(y_real) #------- print(' ') print('Done with second for loop!!!!') print(len(x_sky)) print(len(y_sky)) x_sky = np.array(x_sky) y_sky = np.array(y_sky) print('Above is for transformation to sky pixels.') "To go from real, undistorted to ideal, distorted." x_det = [] y_det = [] for x, y in zip(x_sky, y_sky): x_ideal, y_ideal = apply_coeff(x, y, coeffxi, coeffyi) x_det.append(x_ideal) y_det.append(y_ideal) print(' ') print('Done with third for loop!!!!') print(len(x_det)) print(len(y_det)) print('Above is for transformation to detector pixels.') "Derive residuals from the complete transformation." x_det = np.array(x_det) y_det = np.array(y_det) x_det_orig = np.array(x_det_orig) y_det_orig = np.array(y_det_orig) resid_x = x_det - x_det_orig resid_y = y_det - y_det_orig print(' ') print('Residual in x: ', resid_x) print(np.min(resid_x), np.max(resid_x)) print(' ') print('Residual in y: ', resid_y) print(np.min(resid_y), np.max(resid_y)) 'Open asdf reference file and transfrom from det to sky.' f = AsdfFile.open('niriss_ref_distortion_image.asdf') det2sky_trans = f.tree['model'] sky_x, sky_y = det2sky_trans(x_det_orig, y_det_orig) 'Transform from sky to det.' sky2det_trans = det2sky_trans.inverse inv_x, inv_y = sky2det_trans(sky_x, sky_y) 'Calculate residuals from reference file.' x_resid_ref = inv_x - x_det_orig y_resid_ref = inv_y - y_det_orig 'Calculate residuals of residuals.' x_resid_resid = np.absolute(resid_x - x_resid_ref) y_resid_resid = np.absolute(resid_y - y_resid_ref) print(' ') print(x_resid_resid) print('The minimum in x_resid_resid is: ', np.min(x_resid_resid)) print('The maximum in x_resid_resid is: ', np.max(x_resid_resid)) print('The mean in x_resid_resid is: ', np.mean(x_resid_resid)) print('The standard deviation in x_resid_resid is: ', np.std(x_resid_resid)) print('The median in x_resid_resid is: ', np.median(x_resid_resid)) print(' ') print(y_resid_resid) print('The minimum in y_resid_resid is: ', np.min(y_resid_resid)) print('The maximum in y_resid_resid is: ', np.max(y_resid_resid)) print('The mean in y_resid_resid is: ', np.mean(y_resid_resid)) print('The standard deviation in y_resid_resid is: ', np.std(y_resid_resid)) print('The median in y_resid_resid is: ', np.median(y_resid_resid)) print(' ') print('Done, Done, and Done. Git the bleep along little doggies.')
def __init__(self, init=None, schema=None, memmap=False, pass_invalid_values=None, strict_validation=None, ignore_missing_extensions=True, **kwargs): """ Parameters ---------- init : str, tuple, `~astropy.io.fits.HDUList`, ndarray, dict, None - None : Create a default data model with no shape. - tuple : Shape of the data array. Initialize with empty data array with shape specified by the. - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - `~astropy.io.fits.HDUList` : Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : dict, str (optional) Tree of objects representing a JSON schema, or string naming a schema. The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. memmap : bool Turn memmap of FITS file on or off. (default: False). Ignored for ASDF files. pass_invalid_values : bool or None If `True`, values that do not validate the schema will be added to the metadata. If `False`, they will be set to `None`. If `None`, value will be taken from the environmental PASS_INVALID_VALUES. Otherwise the default value is `False`. strict_validation : bool or None If `True`, schema validation errors will generate an exception. If `False`, they will generate a warning. If `None`, value will be taken from the environmental STRICT_VALIDATION. Otherwise, the default value is `False`. ignore_missing_extensions : bool When `False`, raise warnings when a file is read that contains metadata about extensions that are not available. Defaults to `True`. kwargs : dict Additional arguments passed to lower level functions. """ # Override value of validation parameters if not explicitly set. if not pass_invalid_values: pass_invalid_values = self._get_envar_as_boolean( "PASS_INVALID_VALUES", False) self._pass_invalid_values = pass_invalid_values if not strict_validation: strict_validation = self._get_envar_as_boolean( "STRICT_VALIDATION", False) self._strict_validation = strict_validation self._ignore_missing_extensions = ignore_missing_extensions kwargs.update({'ignore_missing_extensions': ignore_missing_extensions}) # Load the schema files if schema is None: # Create an AsdfFile so we can use its resolver for loading schemas asdf_file = AsdfFile() schema = asdf_schema.load_schema(self.schema_url, resolver=asdf_file.resolver, resolve_references=True) self._schema = mschema.merge_property_trees(schema) # Provide the object as context to other classes and functions self._ctx = self # Determine what kind of input we have (init) and execute the # proper code to intiailize the model self._files_to_close = [] self._iscopy = False is_array = False is_shape = False shape = None if init is None: asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, dict): asdffile = self.open_asdf(init=init, **kwargs) elif isinstance(init, np.ndarray): asdffile = self.open_asdf(init=None, **kwargs) shape = init.shape is_array = True elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init is_shape = True asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, DataModel): asdffile = None self.clone(self, init) if not isinstance(init, self.__class__): self.validate() return elif isinstance(init, AsdfFile): asdffile = init elif isinstance(init, fits.HDUList): asdffile = fits_support.from_fits(init, self._schema, self._ctx, **kwargs) elif isinstance(init, (str, bytes)): if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) file_type = filetype.check(init) if file_type == "fits": if s3_utils.is_s3_uri(init): init_fitsopen = s3_utils.get_object(init) memmap = None else: init_fitsopen = init with fits.open(init_fitsopen, memmap=memmap) as hdulist: asdffile = fits_support.from_fits(hdulist, self._schema, self._ctx, **kwargs) self._files_to_close.append(hdulist) elif file_type == "asdf": asdffile = self.open_asdf(init=init, **kwargs) else: # TODO handle json files as well raise IOError( "File does not appear to be a FITS or ASDF file.") else: raise ValueError("Can't initialize datamodel using {0}".format( str(type(init)))) # Initialize object fields as determined from the code above self._shape = shape self._instance = asdffile.tree self._asdf = asdffile # Initalize class dependent hidden fields self._no_asdf_extension = False # Instantiate the primary array of the image if is_array: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) # If a shape has been given, initialize the primary array. if is_shape: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Shape passed to DataModel.__init__, but model has " "no primary array in its schema") # Initialization occurs when the primary array is first # referenced. Do so now. getattr(self, primary_array_name) # if the input is from a file, set the filename attribute if isinstance(init, str): self.meta.filename = os.path.basename(init) elif isinstance(init, fits.HDUList): info = init.fileinfo(0) if info is not None: filename = info.get('filename') if filename is not None: self.meta.filename = os.path.basename(filename) # if the input model doesn't have a date set, use the current date/time if not self.meta.hasattr('date'): current_date = Time(datetime.datetime.now()) current_date.format = 'isot' self.meta.date = current_date.value # store the data model type, if not already set klass = self.__class__.__name__ if klass != 'DataModel': if not self.meta.hasattr('model_type'): self.meta.model_type = klass # initialize arrays from keyword arguments when they are present for attr, value in kwargs.items(): if value is not None: subschema = properties._get_schema_for_property( self._schema, attr) if 'datatype' in subschema: setattr(self, attr, value)
import pytest import numpy as np import asdf from asdf import AsdfFile from asdf import asdftypes from asdf import block from asdf import schema from asdf import extension from asdf import treeutil from asdf import util from asdf import versioning from . import helpers, CustomTestType _ctx = AsdfFile() _resolver = _ctx.resolver class LabelMapperTestType(CustomTestType): version = '1.0.0' name = 'transform/label_mapper' class RegionsSelectorTestType(CustomTestType): version = '1.0.0' name = 'transform/regions_selector' class TestExtension(extension.BuiltinExtension): """This class defines an extension that represents tags whose
def create_miri_imager_distortion(distfile, outname): """ Create an asdf reference file with all distortion components for the MIRI imager. The filter offsets are stored in a sepaate file. Note: The IDT supplied distortion file lists sky to pixel as the forward transform. Since "forward" in the JWST pipeline is from pixel to sky, the meaning of forward and inverse matrices and the order in which they are applied is switched. The order of operation from pixel to sky is: - Apply MI matrix - Apply Ai and BI matrices - Apply the TI matrix (this gives V2/V3 coordinates) Parameters ---------- distfile : str MIRI imager DISTORTION file provided by the IDT team. outname : str Name of reference file to be wriiten to disk. Returns ------- fasdf : AsdfFile AsdfFile object Examples -------- >>> create_miri_imager_distortion("MIRI_FM_MIRIMAGE_DISTORTION_03.02.00.fits", 'test.asdf') """ fdist = fits.open(distfile) mi_matrix = fdist[8].data mi_col = models.Polynomial1D(1, c0=mi_matrix[0, 2], c1=mi_matrix[0, 0], name="M_column_correction") mi_row = models.Polynomial1D(1, c0=mi_matrix[1, 2], c1=mi_matrix[1, 1], name="M_row_correction") m_matrix = fdist[4].data m_col = models.Polynomial1D(1, c0=m_matrix[0, 2], c1=m_matrix[0, 0]) m_row = models.Polynomial1D(1, c0=m_matrix[1, 2], c1=m_matrix[1, 1]) mi_col.inverse = m_col mi_row.inverse = m_row m_transform = mi_col & mi_row #mi_row & mi_col ai_matrix = fdist[6].data a_matrix = fdist[2].data col_poly = polynomial_from_coeffs_matrix(ai_matrix, name="A_correction") col_poly.inverse = polynomial_from_coeffs_matrix(a_matrix) bi_matrix = fdist[5].data b_matrix = fdist[1].data row_poly = polynomial_from_coeffs_matrix(bi_matrix, name="B_correction") row_poly.inverse = polynomial_from_coeffs_matrix(b_matrix) poly = col_poly & row_poly ti_matrix = fdist[7].data t_matrix = fdist[3].data ti_col = models.Polynomial2D(1, name='TI_column_correction') ti_col.parameters = ti_matrix[0][::-1] ti_row = models.Polynomial2D(1, name='TI_row_correction') ti_row.parameters = ti_matrix[1][::-1] t_col = models.Polynomial2D(1, name='T_column_correction') t_col.parameters = t_matrix[0][::-1] t_row = models.Polynomial2D(1, name='T_row_correction') t_row.parameters = t_matrix[1][::-1] ti_col.inverse = t_col ti_row.inverse = t_row t_transform = ti_row & ti_col mapping = models.Mapping([0, 1, 0, 1]) mapping.inverse = models.Identity(2) # ident is created here so that mapping can be assigned as inverse ident = models.Identity(2) ident.inverse = models.Mapping([0, 1, 0, 1]) poly2t_mapping = models.Mapping([0, 1, 0, 1]) poly2t_mapping.inverse = models.Mapping([0, 1, 0, 1]) distortion_transform = m_transform | mapping | poly | poly2t_mapping | t_transform | ident | models.Mapping( [1, 0]) fdist.close() f = AsdfFile() tree = { "title": "MIRI imager distortion - CDP4", "reftype": "DISTORTION", "instrument": "MIRI", "detector": "MIRIMAGE", "exp_type": "MIR_IMAGE", "pedigree": "GROUND", "author": "N. Dencheva", "model": distortion_transform } f.tree = tree fasdf = f.write_to(outname)
def imaging_distortion(input_model, reference_files): distortion = AsdfFile.open(reference_files['distortion']).tree['model'] return distortion
def __init__(self, init=None, schema=None, memmap=False, pass_invalid_values=None, strict_validation=None, validate_on_assignment=None, ignore_missing_extensions=True, **kwargs): """ Parameters ---------- init : str, tuple, `~astropy.io.fits.HDUList`, ndarray, dict, None - None : Create a default data model with no shape. - tuple : Shape of the data array. Initialize with empty data array with shape specified by the. - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - `~astropy.io.fits.HDUList` : Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : dict, str (optional) Tree of objects representing a JSON schema, or string naming a schema. The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. memmap : bool Turn memmap of FITS file on or off. (default: False). Ignored for ASDF files. pass_invalid_values : bool or None If `True`, values that do not validate the schema will be added to the metadata. If `False`, they will be set to `None`. If `None`, value will be taken from the environmental PASS_INVALID_VALUES. Otherwise the default value is `False`. strict_validation : bool or None If `True`, schema validation errors will generate an exception. If `False`, they will generate a warning. If `None`, value will be taken from the environmental STRICT_VALIDATION. Otherwise, the default value is `False`. validate_on_assignment : bool or None Defaults to 'None'. If `None`, value will be taken from the environmental VALIDATE_ON_ASSIGNMENT, defaulting to 'True' if no environment variable is set. If 'True', attribute assignments are validated at the time of assignment. Validation errors generate warnings and values will be set to `None`. If 'False', schema validation occurs only once at the time of write. Validation errors generate warnings. ignore_missing_extensions : bool When `False`, raise warnings when a file is read that contains metadata about extensions that are not available. Defaults to `True`. kwargs : dict Additional keyword arguments passed to lower level functions. These arguments are generally file format-specific. Arguments of note are: - FITS skip_fits_update - bool or None `True` to skip updating the ASDF tree from the FITS headers, if possible. If `None`, value will be taken from the environmental SKIP_FITS_UPDATE. Otherwise, the default value is `True`. """ # Override value of validation parameters if not explicitly set. if pass_invalid_values is None: pass_invalid_values = get_envar_as_boolean("PASS_INVALID_VALUES", False) self._pass_invalid_values = pass_invalid_values if strict_validation is None: strict_validation = get_envar_as_boolean("STRICT_VALIDATION", False) if validate_on_assignment is None: validate_on_assignment = get_envar_as_boolean( "VALIDATE_ON_ASSIGNMENT", True) self._strict_validation = strict_validation self._ignore_missing_extensions = ignore_missing_extensions self._validate_on_assignment = validate_on_assignment kwargs.update({'ignore_missing_extensions': ignore_missing_extensions}) # Load the schema files if schema is None: if self.schema_url is None: schema = _DEFAULT_SCHEMA else: # Create an AsdfFile so we can use its resolver for loading schemas schema = asdf_schema.load_schema(self.schema_url, resolve_references=True) self._schema = mschema.merge_property_trees(schema) # Provide the object as context to other classes and functions self._ctx = self # Initialize with an empty AsdfFile instance as this is needed for # reading in FITS files where validate._check_value() gets called, and # ctx needs to have an _asdf attribute. self._asdf = AsdfFile() # Determine what kind of input we have (init) and execute the # proper code to intiailize the model self._files_to_close = [] self._iscopy = False is_array = False is_shape = False shape = None if init is None: asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, dict): asdffile = self.open_asdf(init=init, **kwargs) elif isinstance(init, np.ndarray): asdffile = self.open_asdf(init=None, **kwargs) shape = init.shape is_array = True elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init is_shape = True asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, DataModel): asdffile = None self.clone(self, init) if not isinstance(init, self.__class__): self.validate() return elif isinstance(init, AsdfFile): asdffile = init elif isinstance(init, fits.HDUList): asdffile = fits_support.from_fits(init, self._schema, self._ctx, **kwargs) elif isinstance(init, (str, bytes, PurePath)): if isinstance(init, PurePath): init = str(init) if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) file_type = filetype.check(init) if file_type == "fits": if s3_utils.is_s3_uri(init): init_fitsopen = s3_utils.get_object(init) memmap = None else: init_fitsopen = init hdulist = fits.open(init_fitsopen, memmap=memmap) asdffile = fits_support.from_fits(hdulist, self._schema, self._ctx, **kwargs) self._files_to_close.append(hdulist) elif file_type == "asdf": asdffile = self.open_asdf(init=init, **kwargs) else: # TODO handle json files as well raise IOError( "File does not appear to be a FITS or ASDF file.") else: raise ValueError("Can't initialize datamodel using {0}".format( str(type(init)))) # Initialize object fields as determined from the code above self._shape = shape self._instance = asdffile.tree self._asdf = asdffile # Initalize class dependent hidden fields self._no_asdf_extension = False # Instantiate the primary array of the image if is_array: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) # If a shape has been given, initialize the primary array. if is_shape: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Shape passed to DataModel.__init__, but model has " "no primary array in its schema") # Initialization occurs when the primary array is first # referenced. Do so now. getattr(self, primary_array_name) # initialize arrays from keyword arguments when they are present for attr, value in kwargs.items(): if value is not None: subschema = properties._get_schema_for_property( self._schema, attr) if 'datatype' in subschema: setattr(self, attr, value) # Call hook that sets model properties self.on_init(init)