def _build_schema(self): """Build the schema, encorporating the core.""" # Determine the schema path filename = os.path.abspath(inspect.getfile(self.__class__)) base_url = os.path.join( os.path.dirname(filename), 'schemas', '') schema_path = os.path.join(base_url, self.schema_url) core_schema_path = os.path.join(base_url, self.core_schema_url) # Get the schemas schema = asdf_schema.load_schema( schema_path, resolve_references=True ) core_schema = asdf_schema.load_schema( core_schema_path, resolve_references=True ) # Create a new core.meta that will co-locate # with each exposure entry. This is done # by saving the meta information in a separate # FITS HDU. core_meta_schema = deepcopy(core_schema['properties']['meta']) treeutil.walk(core_meta_schema, remove_fits) exposure_schema = schema['allOf'][1]['properties']['exposures']['items'] exposure_meta_schema = exposure_schema['allOf'][1]['properties']['meta'] exposure_meta_schema.update(core_meta_schema) # That's all folks return schema
def test_schema_caching(): # Make sure that if we request the same URL, we get the *exact # same* object, to ensure the cache is working. s1 = schema.load_schema( 'http://stsci.edu/schemas/asdf/core/asdf-1.0.0') s2 = schema.load_schema( 'http://stsci.edu/schemas/asdf/core/asdf-1.0.0') assert s1 is s2
def run_schema_example_test(organization, standard, name, version, check_func=None): import asdf from asdf.tests import helpers from asdf.types import format_tag from asdf.schema import load_schema tag = format_tag(organization, standard, version, name) uri = asdf.resolver.default_tag_to_url_mapping(tag) r = asdf.AsdfFile().resolver examples = [] schema = load_schema(uri, resolver=r) for node in asdf.treeutil.iter_tree(schema): if (isinstance(node, dict) and 'examples' in node and isinstance(node['examples'], list)): for desc, example in node['examples']: examples.append(example) for example in examples: buff = helpers.yaml_to_asdf('example: ' + example.strip()) ff = asdf.AsdfFile(uri=uri) # Add some dummy blocks so that the ndarray examples work for i in range(3): b = asdf.block.Block(np.zeros((1024*1024*8), dtype=np.uint8)) b._used = True ff.blocks.add(b) ff._open_impl(ff, buff, mode='r') if check_func: check_func(ff)
def read_schema(schema_file, extensions=None): """ Read a schema file from disk in order to pass it as an argument to a new datamodel. """ def get_resolver(asdf_file): extensions = asdf_file._extensions def asdf_file_resolver(uri): return extensions._url_mapping(extensions._tag_mapping(uri)) return asdf_file_resolver default_extensions = [GWCSExtension(), JWSTExtension()] if extensions is None: extensions = default_extensions[:] else: extensions.extend(default_extensions) asdf_file = AsdfFile(extensions=extensions) if hasattr(asdf_file, 'resolver'): file_resolver = asdf_file.resolver else: file_resolver = get_resolver(asdf_file) schema = asdf_schema.load_schema(schema_file, resolver=file_resolver, resolve_references=True) schema = merge_property_trees(schema) return schema
def test_fits_without_sci(): from astropy.io import fits schema = { "allOf": [ mschema.load_schema( os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "coeffs": { 'max_ndim': 1, 'fits_hdu': 'COEFFS', 'datatype': 'float32' } } } ] } fits = fits.HDUList( [fits.PrimaryHDU(), fits.ImageHDU(name='COEFFS', data=np.array([0.0], np.float32))]) with DataModel(fits, schema=schema) as dm: assert_array_equal(dm.coeffs, [0.0])
def test_schema_resolved_via_entry_points(): """Test that entry points mappings to core schema works""" r = asdf.AsdfFile().resolver tag = types.format_tag('stsci.edu', 'asdf', '1.0.0', 'fits/fits') url = resolver.default_tag_to_url_mapping(tag) s = schema.load_schema(url, resolver=r, resolve_references=True) assert tag in repr(s)
def test_self_reference_resolution(): r = resolver.Resolver(CustomExtension().url_mapping, 'url') s = schema.load_schema( helpers.get_test_data_path('self_referencing-1.0.0.yaml'), resolver=r, resolve_references=True) assert '$ref' not in repr(s) assert s['anyOf'][1] == s['anyOf'][0]
def runtest(self): from asdf import schema from asdf.extension import default_extensions # Make sure that each schema itself is valid. schema_tree = schema.load_schema( self.schema_path, resolver=default_extensions.resolver, resolve_references=True) schema.check_schema(schema_tree)
def test_read_json_schema(): """Pytest to make sure reading JSON schemas succeeds. This was known to fail on Python 3.5 See issue #314 at https://github.com/spacetelescope/asdf/issues/314 for more details. """ json_schema = helpers.get_test_data_path('example_schema.json') schema_tree = schema.load_schema(json_schema, resolve_references=True) schema.check_schema(schema_tree)
def test_table_array(): table_schema = { "allOf": [ mschema.load_schema( os.path.join(os.path.dirname(__file__), "../schemas/image.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "table": { 'title': 'A structured table', 'fits_hdu': 'table', 'datatype': [ 'bool8', {'datatype': 'int16', 'name': 'my_int'}, {'datatype': 'float32', 'name': 'my_float', 'shape': [3, 2]}, {'datatype': ['ascii', 64], 'name': 'my_string'} ] } } } ] } with DataModel(schema=table_schema) as x: x.table = [(True, 42, 37.5, 'STRING')] assert x.table.dtype == [ (str('f0'), str('?')), (str('my_int'), str('=i2')), (str('my_float'), str('=f4'), (3, 2)), (str('my_string'), str('S64')) ] x.to_fits(TMP_FITS, overwrite=True) with DataModel(TMP_FITS, schema=table_schema) as x: table = x.table assert table.dtype == [ (str('f0'), str('?')), (str('my_int'), str('=i2')), (str('my_float'), str('=f4'), (3, 2)), (str('my_string'), str('S64')) ]
def test_property_order(): tree = {'foo': np.ndarray([1, 2, 3])} buff = io.BytesIO() ff = asdf.AsdfFile(tree) ff.write_to(buff) ndarray_schema = schema.load_schema( 'http://stsci.edu/schemas/asdf/core/ndarray-1.0.0') property_order = ndarray_schema['anyOf'][1]['propertyOrder'] last_index = 0 for prop in property_order: index = buff.getvalue().find(prop.encode('utf-8') + b':') if index != -1: assert index > last_index last_index = index
def test_time_tag(): schema = asdf_schema.load_schema( 'http://stsci.edu/schemas/asdf/time/time-1.1.0', resolve_references=True) schema = _flatten_combiners(schema) date = time.Time(datetime.datetime.now()) tree = {'date': date} asdf = AsdfFile(tree=tree) instance = yamlutil.custom_tree_to_tagged_tree(tree['date'], asdf) asdf_schema.validate(instance, schema=schema) tag = 'tag:stsci.edu:asdf/time/time-1.1.0' date = tagged.tag_object(tag, date) tree = {'date': date} asdf = AsdfFile(tree=tree) instance = yamlutil.custom_tree_to_tagged_tree(tree['date'], asdf) asdf_schema.validate(instance, schema=schema)
def test_load_schema(tmpdir): schema_def = """ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0" id: "http://stsci.edu/schemas/asdf/nugatory/nugatory-1.0.0" tag: "tag:stsci.edu:asdf/nugatory/nugatory-1.0.0" type: object properties: foobar: $ref: "../core/ndarray-1.0.0" required: [foobar] ... """ schema_path = tmpdir.join('nugatory.yaml') schema_path.write(schema_def.encode()) schema_tree = schema.load_schema(str(schema_path), resolve_references=True) schema.check_schema(schema_tree)
def test_load_schema(tmpdir): schema_def = """ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0" id: "http://stsci.edu/schemas/asdf/nugatory/nugatory-1.0.0" tag: "tag:stsci.edu:asdf/nugatory/nugatory-1.0.0" type: object properties: foobar: $ref: "../core/ndarray-1.0.0" required: [foobar] ... """ schema_path = tmpdir.join('nugatory.yaml') schema_path.write(schema_def.encode()) schema_tree = schema.load_schema(str(schema_path), resolve_references=True) schema.check_schema(schema_tree)
def test_load_schema_with_file_url(tmpdir): schema_def = """ %YAML 1.1 %TAG !asdf! tag:stsci.edu:asdf/ --- $schema: "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0" id: "http://stsci.edu/schemas/asdf/nugatory/nugatory-1.0.0" tag: "tag:stsci.edu:asdf/nugatory/nugatory-1.0.0" type: object properties: foobar: $ref: "{}" required: [foobar] ... """.format(resolver.default_resolver('tag:stsci.edu:asdf/core/ndarray-1.0.0')) schema_path = tmpdir.join('nugatory.yaml') schema_path.write(schema_def.encode()) schema_tree = schema.load_schema(str(schema_path), resolve_references=True) schema.check_schema(schema_tree)
def test_load_schema_with_file_url(tmpdir): schema_def = """ %YAML 1.1 %TAG !asdf! tag:stsci.edu:asdf/ --- $schema: "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0" id: "http://stsci.edu/schemas/asdf/nugatory/nugatory-1.0.0" tag: "tag:stsci.edu:asdf/nugatory/nugatory-1.0.0" type: object properties: foobar: $ref: "{}" required: [foobar] ... """.format(resolver.default_resolver('tag:stsci.edu:asdf/core/ndarray-1.0.0')) schema_path = tmpdir.join('nugatory.yaml') schema_path.write(schema_def.encode()) schema_tree = schema.load_schema(str(schema_path), resolve_references=True) schema.check_schema(schema_tree)
def test_load_schema_with_asdf_uri_scheme(): subschema_content = """%YAML 1.1 --- $schema: http://stsci.edu/schemas/asdf/asdf-schema-1.0.0 id: asdf://somewhere.org/schemas/bar bar: type: string ... """ content = """%YAML 1.1 --- $schema: http://stsci.edu/schemas/asdf/asdf-schema-1.0.0 id: asdf://somewhere.org/schemas/foo definitions: local_bar: type: string type: object properties: bar: $ref: asdf://somewhere.org/schemas/bar#/bar local_bar: $ref: '#/definitions/local_bar' ... """ with asdf.config_context() as config: config.add_resource_mapping( {"asdf://somewhere.org/schemas/foo": content}) config.add_resource_mapping( {"asdf://somewhere.org/schemas/bar": subschema_content}) schema_tree = schema.load_schema("asdf://somewhere.org/schemas/foo") instance = {"bar": "baz", "local_bar": "foz"} schema.validate(instance, schema=schema_tree) with pytest.raises(ValidationError): schema.validate({"bar": 12}, schema=schema_tree)
def test_nested_array_yaml(tmpdir): schema_def = """ %YAML 1.1 --- type: object properties: stuff: type: array items: type: array items: - type: integer - type: string - type: number minItems: 3 maxItems: 3 ... """ schema_path = tmpdir.join('nested.yaml') schema_path.write(schema_def.encode()) schema_tree = schema.load_schema(str(schema_path)) schema.check_schema(schema_tree) good = dict(stuff=[[1, 'hello', 2], [4, 'world', 9.7]]) schema.validate(good, schema=schema_tree) bads = [ dict(stuff=[[1, 2, 3]]), dict(stuff=[12, 'dldl']), dict(stuff=[[12, 'dldl']]), dict(stuff=[[1, 'hello', 2], [4, 5]]), dict(stuff=[[1, 'hello', 2], [4, 5, 6]]) ] for b in bads: with pytest.raises(ValidationError): schema.validate(b, schema=schema_tree)
def test_nested_array_yaml(tmpdir): schema_def = """ %YAML 1.1 --- type: object properties: stuff: type: array items: type: array items: - type: integer - type: string - type: number minItems: 3 maxItems: 3 ... """ schema_path = tmpdir.join('nested.yaml') schema_path.write(schema_def.encode()) schema_tree = schema.load_schema(str(schema_path)) schema.check_schema(schema_tree) good = dict(stuff=[[1, 'hello', 2], [4, 'world', 9.7]]) schema.validate(good, schema=schema_tree) bads = [ dict(stuff=[[1, 2, 3]]), dict(stuff=[12,'dldl']), dict(stuff=[[12, 'dldl']]), dict(stuff=[[1, 'hello', 2], [4, 5]]), dict(stuff=[[1, 'hello', 2], [4, 5, 6]]) ] for b in bads: with pytest.raises(ValidationError): schema.validate(b, schema=schema_tree)
def test_fits_without_sci(): schema = { "allOf": [ mschema.load_schema("http://stsci.edu/schemas/jwst_datamodel/core.schema", resolve_references=True), { "type": "object", "properties": { "coeffs": { 'max_ndim': 1, 'fits_hdu': 'COEFFS', 'datatype': 'float32' } } } ] } hdulist = fits.HDUList() hdulist.append(fits.PrimaryHDU()) hdulist.append(fits.ImageHDU(name='COEFFS', data=np.array([0.0], np.float32))) with JwstDataModel(hdulist, schema=schema) as dm: assert_array_equal(dm.coeffs, [0.0])
def test_schema_caching(): # Make sure that if we request the same URL, we get the *exact # same* object, to ensure the cache is working. s1 = schema.load_schema('http://stsci.edu/schemas/asdf/core/asdf-1.0.0') s2 = schema.load_schema('http://stsci.edu/schemas/asdf/core/asdf-1.0.0') assert s1 is s2
with pytest.raises(jsonschema.ValidationError): with ImageModel((50, 50)) as dm: dm.meta.date = 'Not an acceptable date' def test_date2(): from astropy import time with ImageModel((50, 50)) as dm: assert isinstance(dm.meta.date, (time.Time, datetime.datetime)) ''' transformation_schema = { "allOf": [ mschema.load_schema( os.path.join(os.path.dirname(__file__), "../schemas/image.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "meta": { "type": "object", "properties": { "transformations": { "title": "A list of transformations", "type": "array", "items": { "title": "A transformation", "type": "object", "properties": { "type": {
def test_load_schema_resolve_local_refs_deprecated(): custom_schema_path = helpers.get_test_data_path( 'custom_schema_definitions.yaml') with pytest.deprecated_call(): schema.load_schema(custom_schema_path, resolve_local_refs=True)
def __init__(self, init=None, schema=None, extensions=None, pass_invalid_values=False): """ Parameters ---------- init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None - None: A default data model with no shape - shape tuple: Initialize with empty data of the given shape - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - ``astropy.io.fits.HDUList``: Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : tree of objects representing a JSON schema, or string naming a schema, optional The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. extensions: classes extending the standard set of extensions pass_invalid_values: If True, values that do not validate the schema can be read and written, but with a warning message """ filename = os.path.abspath(inspect.getfile(self.__class__)) base_url = os.path.join( os.path.dirname(filename), 'schemas', '') if schema is None: schema_path = os.path.join(base_url, self.schema_url) schema = asdf_schema.load_schema( schema_path, resolve_references=True) self._schema = mschema.flatten_combiners(schema) if extensions is not None: extensions.extend(jwst_extensions) else: extensions = jwst_extensions[:] self._extensions = extensions if "PASS_INVALID_VALUES" in os.environ: pass_invalid_values = os.environ["PASS_INVALID_VALUES"] try: self._pass_invalid_values = bool(int(pass_invalid_values)) except ValueError: self._pass_invalid_values = False else: self._pass_invalid_values = pass_invalid_values self._files_to_close = [] is_array = False is_shape = False shape = None if init is None: asdf = AsdfFile(extensions=extensions) elif isinstance(init, dict): asdf = AsdfFile(init, extensions=extensions) elif isinstance(init, np.ndarray): asdf = AsdfFile(extensions=extensions) shape = init.shape is_array = True elif isinstance(init, self.__class__): instance = copy.deepcopy(init._instance) self._schema = init._schema self._shape = init._shape self._asdf = AsdfFile(instance, extensions=self._extensions) self._instance = instance self._ctx = self self.__class__ = init.__class__ return elif isinstance(init, DataModel): raise TypeError( "Passed in {0!r} is not of the expected subclass {1!r}".format( init.__class__.__name__, self.__class__.__name__)) elif isinstance(init, AsdfFile): asdf = init elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init asdf = AsdfFile() is_shape = True elif isinstance(init, fits.HDUList): asdf = fits_support.from_fits(init, self._schema, extensions=self._extensions, validate=False, pass_invalid_values=self._pass_invalid_values) elif isinstance(init, six.string_types): if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) try: hdulist = fits.open(init) except IOError: try: asdf = AsdfFile.open(init, extensions=self._extensions) # TODO: Add json support except ValueError: raise IOError( "File does not appear to be a FITS or ASDF file.") else: asdf = fits_support.from_fits(hdulist, self._schema, extensions=self._extensions, validate=False, pass_invalid_values=self._pass_invalid_values) self._files_to_close.append(hdulist) self._shape = shape self._instance = asdf.tree self._asdf = asdf self._ctx = self # if the input model doesn't have a date set, use the current date/time if self.meta.date is None: self.meta.date = Time(datetime.datetime.now()) self.meta.date.format='isot' self.meta.date = self.meta.date.value # if the input is from a file, set the filename attribute if isinstance(init, six.string_types): self.meta.filename = os.path.basename(init) if is_array: primary_array_name = self.get_primary_array_name() if primary_array_name is None: raise TypeError( "Array passed to model.__init__, but model has no primary " "array in its schema") setattr(self, primary_array_name, init) if is_shape: getattr(self, self.get_primary_array_name())
# Setting an invalid value should raise a ValueError dm.meta.target.ra = "FOO" def test_date2(): with ImageModel((50, 50), strict_validation=True) as dm: time_obj = time.Time(dm.meta.date) assert isinstance(time_obj, time.Time) date_obj = datetime.strptime(dm.meta.date, '%Y-%m-%dT%H:%M:%S.%f') assert isinstance(date_obj, datetime) TRANSFORMATION_SCHEMA = { "allOf": [ mschema.load_schema(os.path.join(URL_PREFIX, "image.schema"), resolver=asdf.AsdfFile().resolver, resolve_references=True), { "type": "object", "properties": { "meta": { "type": "object", "properties": { "transformations": { "title": "A list of transformations", "type": "array", "items": { "title": "A transformation", "type": "object", "properties": { "type": {
from asdf.schema import load_schema from asdf import generic_io from custom_module.custom_extension import CustomExtensions from custom_module.custom_types import CustomClass my_float = 12.45 my_list = [3, 4, 6] my_dict = {"a": 1, "b": 42} cc = CustomClass(2.45, [2, 8, 64]) tree = { "cc": cc, } s = load_schema("custom_module/schemas/CustomClass-alternative.yaml") s_string = generic_io.get_file( "custom_module/schemas/CustomClass-alternative.yaml").read().decode( 'utf-8') asdf.get_config().add_resource_mapping({ "http://CustomOrganization/schemas/CustomStandard/CustomClass-1.0.0": s_string }) print("Script --- Create AsdfFile instance") af = asdf.AsdfFile( tree, extensions=CustomExtensions(), custom_schema=f"custom_module\\schemas\\CustomSchema.yaml", )
def __init__(self, init=None, schema=None, extensions=None, pass_invalid_values=False, strict_validation=False, **kwargs): """ Gets the path to the schema using a different prefix then calls the constructor of DataModel Parameters ---------- init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None - None: A default data model with no shape - shape tuple: Initialize with empty data of the given shape - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - ``astropy.io.fits.HDUList``: Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : tree of objects representing a JSON schema, or string naming a schema, optional The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. extensions: classes extending the standard set of extensions, optional. If an extension is defined, the prefix used should be 'url'. pass_invalid_values: If true, values that do not validate the schema will be added to the metadata. If false, they will be set to None strict_validation: if true, an schema validation errors will generate an excption. If false, they will generate a warning. kwargs: Aadditional arguments passed to lower level functions """ URL_PREFIX = "http://oirlab.ucsd.edu/schemas/" # Load the schema files if schema is None: schema_path = os.path.join(URL_PREFIX, self.schema_url) # Create an AsdfFile so we can use its resolver for loading schemas asdf_file = AsdfFile() schema = asdf_schema.load_schema(schema_path, resolver=asdf_file.resolver, resolve_references=True) super().__init__(init=init, schema=schema, extensions=extensions, pass_invalid_values=pass_invalid_values, strict_validation=strict_validation, **kwargs)
def test_table_array_shape_ndim(): table_schema = { "allOf": [ mschema.load_schema(os.path.join(URL_PREFIX, "image.schema"), resolver=asdf.AsdfFile().resolver, resolve_references=True), { "type": "object", "properties": { "table": { 'title': 'A structured table', 'fits_hdu': 'table', 'datatype': [ 'bool8', {'datatype': 'int16', 'name': 'my_int'}, {'datatype': 'float32', 'name': 'my_float1', 'shape': [3, 2]}, {'datatype': 'float32', 'name': 'my_float2', 'ndim': 2}, {'datatype': 'float32', 'name': 'my_float3'}, {'datatype': 'float32', 'name': 'my_float4'}, {'datatype': 'float32', 'name': 'my_float5'}, {'datatype': ['ascii', 64], 'name': 'my_string'} ] } } } ] } with DataModel(schema=table_schema) as x: x.table = [ ( True, 42, [[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]], [[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]], [[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]], [37.5, 38.0], 37.5, 'STRING' ) ] assert x.table.dtype == [ ('f0', '?'), ('my_int', '=i2'), ('my_float1', '=f4', (3, 2)), ('my_float2', '=f4', (3, 2)), ('my_float3', '=f4', (3, 2)), ('my_float4', '=f4', (2,)), ('my_float5', '=f4'), ('my_string', 'S64') ] x.to_fits(TMP_FITS, overwrite=True) with DataModel(TMP_FITS, schema=table_schema) as x: assert x.table.dtype == [ ('f0', '?'), ('my_int', '=i2'), ('my_float1', '=f4', (3, 2)), ('my_float2', '=f4', (3, 2)), ('my_float3', '=f4', (3, 2)), ('my_float4', '=f4', (2,)), ('my_float5', '=f4'), ('my_string', 'S64') ] table_schema['allOf'][1]['properties']['table']['datatype'][3]['ndim'] = 3 with DataModel(schema=table_schema) as x: with pytest.raises(ValueError) as e: x.table = [ ( True, 42, [[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]], [[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]], [[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]], [37.5, 38.0], 37.5, 'STRING' ) ] assert str(e.value).startswith("Array has wrong number of dimensions.")
def __init__(self, init=None, schema=None, extensions=None, pass_invalid_values=False): """ Parameters ---------- init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None - None: A default data model with no shape - shape tuple: Initialize with empty data of the given shape - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - ``astropy.io.fits.HDUList``: Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : tree of objects representing a JSON schema, or string naming a schema, optional The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. extensions: classes extending the standard set of extensions pass_invalid_values: If True, values that do not validate the schema can be read and written, but with a warning message """ filename = os.path.abspath(inspect.getfile(self.__class__)) base_url = os.path.join(os.path.dirname(filename), 'schemas', '') if schema is None: schema_path = os.path.join(base_url, self.schema_url) schema = asdf_schema.load_schema(schema_path, resolve_references=True) self._schema = mschema.flatten_combiners(schema) if extensions is not None: extensions.extend(jwst_extensions) else: extensions = jwst_extensions[:] self._extensions = extensions if "PASS_INVALID_VALUES" in os.environ: pass_invalid_values = os.environ["PASS_INVALID_VALUES"] try: self._pass_invalid_values = bool(int(pass_invalid_values)) except ValueError: self._pass_invalid_values = False else: self._pass_invalid_values = pass_invalid_values self._files_to_close = [] is_array = False is_shape = False shape = None if init is None: asdf = AsdfFile(extensions=extensions) elif isinstance(init, dict): asdf = AsdfFile(init, extensions=extensions) elif isinstance(init, np.ndarray): asdf = AsdfFile(extensions=extensions) shape = init.shape is_array = True elif isinstance(init, self.__class__): instance = copy.deepcopy(init._instance) self._schema = init._schema self._shape = init._shape self._asdf = AsdfFile(instance, extensions=self._extensions) self._instance = instance self._ctx = self self.__class__ = init.__class__ return elif isinstance(init, DataModel): raise TypeError( "Passed in {0!r} is not of the expected subclass {1!r}".format( init.__class__.__name__, self.__class__.__name__)) elif isinstance(init, AsdfFile): asdf = init elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init asdf = AsdfFile() is_shape = True elif isinstance(init, fits.HDUList): asdf = fits_support.from_fits( init, self._schema, extensions=self._extensions, validate=False, pass_invalid_values=self._pass_invalid_values) elif isinstance(init, six.string_types): if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) try: hdulist = fits.open(init) except IOError: try: asdf = AsdfFile.open(init, extensions=self._extensions) # TODO: Add json support except ValueError: raise IOError( "File does not appear to be a FITS or ASDF file.") else: asdf = fits_support.from_fits( hdulist, self._schema, extensions=self._extensions, validate=False, pass_invalid_values=self._pass_invalid_values) self._files_to_close.append(hdulist) self._shape = shape self._instance = asdf.tree self._asdf = asdf self._ctx = self # if the input model doesn't have a date set, use the current date/time if self.meta.date is None: self.meta.date = Time(datetime.datetime.now()) self.meta.date.format = 'isot' self.meta.date = self.meta.date.value # if the input is from a file, set the filename attribute if isinstance(init, six.string_types): self.meta.filename = os.path.basename(init) if is_array: primary_array_name = self.get_primary_array_name() if primary_array_name is None: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) if is_shape: getattr(self, self.get_primary_array_name())
def __init__(self, init=None, schema=None, extensions=None, pass_invalid_values=False, strict_validation=False): """ Parameters ---------- init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None - None: A default data model with no shape - shape tuple: Initialize with empty data of the given shape - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - ``astropy.io.fits.HDUList``: Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : tree of objects representing a JSON schema, or string naming a schema, optional The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. extensions: classes extending the standard set of extensions, optional. If an extension is defined, the prefix used should be 'url'. pass_invalid_values: If true, values that do not validate the schema will be added to the metadata. If false, they will be set to None strict_validation: if true, an schema validation errors will generate an excption. If false, they will generate a warning. """ # Set the extensions self._extensions = extensions # Override value of validation parameters # if environment value set self._pass_invalid_values = self.get_envar("PASS_INVALID_VALUES", pass_invalid_values) self._strict_validation = self.get_envar("STRICT_VALIDATION", strict_validation) # Construct the path to the schema files filename = os.path.abspath(inspect.getfile(self.__class__)) base_url = os.path.join(os.path.dirname(filename), 'schemas', '') # Load the schema files if schema is None: schema_path = os.path.join(base_url, self.schema_url) # Create an AsdfFile so we can use its resolver for loading schemas asdf_file = AsdfFile(extensions=self._extensions) if hasattr(asdf_file, 'resolver'): file_resolver = asdf_file.resolver else: file_resolver = self.get_resolver(asdf_file) schema = asdf_schema.load_schema(schema_path, resolver=file_resolver, resolve_references=True) self._schema = mschema.flatten_combiners(schema) # Provide the object as context to other classes and functions self._ctx = self # Determine what kind of input we have (init) and execute the # proper code to intiailize the model self._files_to_close = [] self._iscopy = False is_array = False is_shape = False shape = None if init is None: asdf = AsdfFile(extensions=extensions) elif isinstance(init, dict): asdf = AsdfFile(init, extensions=self._extensions) elif isinstance(init, np.ndarray): asdf = AsdfFile(extensions=self._extensions) shape = init.shape is_array = True elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init asdf = AsdfFile() is_shape = True elif isinstance(init, DataModel): self.clone(self, init) if not isinstance(init, self.__class__): self.validate() return elif isinstance(init, AsdfFile): asdf = init elif isinstance(init, fits.HDUList): asdf = fits_support.from_fits(init, self._schema, self._extensions, self._ctx) elif isinstance(init, (str, bytes)): if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) file_type = filetype.check(init) if file_type == "fits": hdulist = fits.open(init) asdf = fits_support.from_fits(hdulist, self._schema, self._extensions, self._ctx) self._files_to_close.append(hdulist) elif file_type == "asdf": asdf = AsdfFile.open(init, extensions=self._extensions) else: # TODO handle json files as well raise IOError( "File does not appear to be a FITS or ASDF file.") else: raise ValueError("Can't initialize datamodel using {0}".format( str(type(init)))) # Initialize object fields as determined from the code above self._shape = shape self._instance = asdf.tree self._asdf = asdf # Initalize class dependent hidden fields self._no_asdf_extension = False # Instantiate the primary array of the image if is_array: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) if is_shape: if not self.get_primary_array_name(): raise TypeError( "Shape passed to DataModel.__init__, but model has " "no primary array in its schema") # if the input is from a file, set the filename attribute if isinstance(init, str): self.meta.filename = os.path.basename(init) elif isinstance(init, fits.HDUList): info = init.fileinfo(0) if info is not None: filename = info.get('filename') if filename is not None: self.meta.filename = os.path.basename(filename) # if the input model doesn't have a date set, use the current date/time if not self.meta.hasattr('date'): current_date = Time(datetime.datetime.now()) current_date.format = 'isot' self.meta.date = current_date.value # store the data model type, if not already set klass = self.__class__.__name__ if klass != 'DataModel': if not self.meta.hasattr('model_type'): self.meta.model_type = klass
def test_table_with_metadata(): schema = { "allOf": [ mschema.load_schema(os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "flux_table": { "title": "Photometric flux conversion table", "fits_hdu": "FLUX", "datatype": [{ "name": "parameter", "datatype": ['ascii', 7] }, { "name": "factor", "datatype": "float64" }, { "name": "uncertainty", "datatype": "float64" }] }, "meta": { "type": "object", "properties": { "fluxinfo": { "title": "Information about the flux conversion", "type": "object", "properties": { "exposure": { "title": "Description of exposure analyzed", "type": "string", "fits_hdu": "FLUX", "fits_keyword": "FLUXEXP" } } } } } } } ] } class FluxModel(DataModel): def __init__(self, init=None, flux_table=None, **kwargs): super(FluxModel, self).__init__(init=init, schema=schema, **kwargs) if flux_table is not None: self.flux_table = flux_table flux_im = [ ('F560W', 1.0e-5, 1.0e-7), ('F770W', 1.1e-5, 1.6e-7), ] with FluxModel(flux_table=flux_im) as datamodel: datamodel.meta.fluxinfo.exposure = 'Exposure info' datamodel.save(TMP_FITS, overwrite=True) del datamodel from astropy.io import fits with fits.open(TMP_FITS, memmap=False) as hdulist: assert len(hdulist) == 3 assert isinstance(hdulist[1], fits.BinTableHDU) assert hdulist[1].name == 'FLUX' assert hdulist[2].name == 'ASDF'
def test_table_array_convert(): """ Test that structured arrays are converted when necessary, and reused as views when not. """ from jwst.datamodels import util table_schema = { "allOf": [ mschema.load_schema( os.path.join(os.path.dirname(__file__), "../schemas/image.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "table": { 'title': 'A structured table', 'fits_hdu': 'table', 'datatype': [ 'bool8', {'datatype': 'int16', 'name': 'my_int'}, {'datatype': ['ascii', 64], 'name': 'my_string'} ] } } } ] } table = np.array( [(42, 32000, 'foo')], dtype=[ (str('f0'), str('?')), (str('my_int'), str('=i2')), (str('my_string'), str('S64')) ]) x = util.gentle_asarray(table, dtype=[ (str('f0'), str('?')), (str('my_int'), str('=i2')), (str('my_string'), str('S64')) ]) assert x is table with DataModel(schema=table_schema) as x: x.table = table assert x.table is table table = np.array( [(42, 32000, 'foo')], dtype=[ (str('f0'), str('?')), (str('my_int'), str('=i2')), (str('my_string'), str('S3')) ]) with DataModel(schema=table_schema) as x: x.table = table assert x.table is not table assert x.table['my_string'][0] == table['my_string'][0]
def test_data_array(tmp_path): """Test lots of things """ path = str(tmp_path / "data_array.fits") data_array_schema = { "allOf": [ mschema.load_schema( "http://stsci.edu/schemas/jwst_datamodel/core.schema", resolve_references=True), { "type": "object", "properties": { "arr": { 'title': 'An array of data', 'type': 'array', "fits_hdu": ["FOO", "DQ"], "items": { "title": "entry", "type": "object", "properties": { "data": { "fits_hdu": "FOO", "default": 0.0, "max_ndim": 2, "datatype": "float64" }, "dq": { "fits_hdu": "DQ", "default": 1, "datatype": "uint8" }, } } } } } ] } array1 = np.random.rand(5, 5) array2 = np.random.rand(5, 5) array3 = np.random.rand(5, 5) with JwstDataModel(schema=data_array_schema) as x: x.arr.append(x.arr.item()) x.arr[0].data = array1 assert len(x.arr) == 1 x.arr.append(x.arr.item(data=array2)) assert len(x.arr) == 2 x.arr.append({}) assert len(x.arr) == 3 x.arr[2].data = array3 del x.arr[1] assert len(x.arr) == 2 x.save(path) with JwstDataModel(path, schema=data_array_schema) as x: assert len(x.arr) == 2 assert_array_almost_equal(x.arr[0].data, array1) assert_array_almost_equal(x.arr[1].data, array3) del x.arr[0] assert len(x.arr) == 1 x.arr = [] assert len(x.arr) == 0 x.arr.append({'data': np.empty((5, 5))}) assert len(x.arr) == 1 x.arr.extend([ x.arr.item(data=np.empty((5, 5))), x.arr.item(data=np.empty((5, 5)), dq=np.empty((5, 5), dtype=np.uint8)) ]) assert len(x.arr) == 3 del x.arr[1] assert len(x.arr) == 2 x.save(path) from astropy.io import fits with fits.open(path) as hdulist: x = set() for hdu in hdulist: x.add((hdu.header.get('EXTNAME'), hdu.header.get('EXTVER'))) assert x == set([('FOO', 2), ('FOO', 1), ('ASDF', None), ('DQ', 2), (None, None)])
def test_data_array(): data_array_schema = { "allOf": [ mschema.load_schema( os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "arr": { 'title': 'An array of data', 'type': 'array', "fits_hdu": ["FOO", "DQ"], "items": { "title": "entry", "type": "object", "properties": { "data": { "fits_hdu": "FOO", "default": 0.0, "max_ndim": 2, "datatype": "float64" }, "dq": { "fits_hdu": "DQ", "default": 1, "datatype": "uint8" }, } } } } } ] } array1 = np.random.rand(5, 5) array2 = np.random.rand(5, 5) array3 = np.random.rand(5, 5) with DataModel(schema=data_array_schema) as x: x.arr.append(x.arr.item()) x.arr[0].data = array1 assert len(x.arr) == 1 x.arr.append(x.arr.item(data=array2)) assert len(x.arr) == 2 x.arr.append({}) assert len(x.arr) == 3 x.arr[2].data = array3 del x.arr[1] assert len(x.arr) == 2 x.to_fits(TMP_FITS, overwrite=True) with DataModel(TMP_FITS, schema=data_array_schema) as x: assert len(x.arr) == 2 assert_array_almost_equal(x.arr[0].data, array1) assert_array_almost_equal(x.arr[1].data, array3) del x.arr[0] assert len(x.arr) == 1 x.arr = [] assert len(x.arr) == 0 x.arr.append({'data': np.empty((5, 5))}) assert len(x.arr) == 1 x.arr.extend([ x.arr.item(data=np.empty((5, 5))), x.arr.item(data=np.empty((5, 5)), dq=np.empty((5, 5), dtype=np.uint8))]) assert len(x.arr) == 3 del x.arr[1] assert len(x.arr) == 2 x.to_fits(TMP_FITS2, overwrite=True) from astropy.io import fits with fits.open(TMP_FITS2) as hdulist: x = set() for hdu in hdulist: x.add((hdu.header.get('EXTNAME'), hdu.header.get('EXTVER'))) print(x) assert x == set( [('FOO', 2), ('FOO', 1), ('ASDF', None), ('DQ', 2), (None, None)])
def test_datamodel_schema_entry_points(): """Test that entry points for datamodels BaseExtension work with asdf""" resolver = asdf.AsdfFile().resolver mschema.load_schema('http://jwst.stsci.edu/schemas/image.schema.yaml', resolver=resolver, resolve_references=True)
with pytest.raises(jsonschema.ValidationError): with ImageModel((50, 50)) as dm: dm.meta.date = 'Not an acceptable date' def test_date2(): from astropy import time with ImageModel((50, 50)) as dm: assert isinstance(dm.meta.date, (time.Time, datetime.datetime)) ''' transformation_schema = { "allOf": [ mschema.load_schema(os.path.join(os.path.dirname(__file__), "../schemas/image.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "meta": { "type": "object", "properties": { "transformations": { "title": "A list of transformations", "type": "array", "items": { "title": "A transformation", "type": "object", "properties": { "type": {
def __init__(self, init=None, schema=None, memmap=False, pass_invalid_values=None, strict_validation=None, ignore_missing_extensions=True, **kwargs): """ Parameters ---------- init : str, tuple, `~astropy.io.fits.HDUList`, ndarray, dict, None - None : Create a default data model with no shape. - tuple : Shape of the data array. Initialize with empty data array with shape specified by the. - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - `~astropy.io.fits.HDUList` : Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : dict, str (optional) Tree of objects representing a JSON schema, or string naming a schema. The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. memmap : bool Turn memmap of FITS file on or off. (default: False). Ignored for ASDF files. pass_invalid_values : bool or None If `True`, values that do not validate the schema will be added to the metadata. If `False`, they will be set to `None`. If `None`, value will be taken from the environmental PASS_INVALID_VALUES. Otherwise the default value is `False`. strict_validation : bool or None If `True`, schema validation errors will generate an exception. If `False`, they will generate a warning. If `None`, value will be taken from the environmental STRICT_VALIDATION. Otherwise, the default value is `False`. ignore_missing_extensions : bool When `False`, raise warnings when a file is read that contains metadata about extensions that are not available. Defaults to `True`. kwargs : dict Additional arguments passed to lower level functions. """ # Override value of validation parameters if not explicitly set. if not pass_invalid_values: pass_invalid_values = self._get_envar_as_boolean( "PASS_INVALID_VALUES", False) self._pass_invalid_values = pass_invalid_values if not strict_validation: strict_validation = self._get_envar_as_boolean( "STRICT_VALIDATION", False) self._strict_validation = strict_validation self._ignore_missing_extensions = ignore_missing_extensions kwargs.update({'ignore_missing_extensions': ignore_missing_extensions}) # Load the schema files if schema is None: # Create an AsdfFile so we can use its resolver for loading schemas asdf_file = AsdfFile() schema = asdf_schema.load_schema(self.schema_url, resolver=asdf_file.resolver, resolve_references=True) self._schema = mschema.merge_property_trees(schema) # Provide the object as context to other classes and functions self._ctx = self # Determine what kind of input we have (init) and execute the # proper code to intiailize the model self._files_to_close = [] self._iscopy = False is_array = False is_shape = False shape = None if init is None: asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, dict): asdffile = self.open_asdf(init=init, **kwargs) elif isinstance(init, np.ndarray): asdffile = self.open_asdf(init=None, **kwargs) shape = init.shape is_array = True elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init is_shape = True asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, DataModel): asdffile = None self.clone(self, init) if not isinstance(init, self.__class__): self.validate() return elif isinstance(init, AsdfFile): asdffile = init elif isinstance(init, fits.HDUList): asdffile = fits_support.from_fits(init, self._schema, self._ctx, **kwargs) elif isinstance(init, (str, bytes)): if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) file_type = filetype.check(init) if file_type == "fits": if s3_utils.is_s3_uri(init): init_fitsopen = s3_utils.get_object(init) memmap = None else: init_fitsopen = init with fits.open(init_fitsopen, memmap=memmap) as hdulist: asdffile = fits_support.from_fits(hdulist, self._schema, self._ctx, **kwargs) self._files_to_close.append(hdulist) elif file_type == "asdf": asdffile = self.open_asdf(init=init, **kwargs) else: # TODO handle json files as well raise IOError( "File does not appear to be a FITS or ASDF file.") else: raise ValueError("Can't initialize datamodel using {0}".format( str(type(init)))) # Initialize object fields as determined from the code above self._shape = shape self._instance = asdffile.tree self._asdf = asdffile # Initalize class dependent hidden fields self._no_asdf_extension = False # Instantiate the primary array of the image if is_array: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) # If a shape has been given, initialize the primary array. if is_shape: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Shape passed to DataModel.__init__, but model has " "no primary array in its schema") # Initialization occurs when the primary array is first # referenced. Do so now. getattr(self, primary_array_name) # if the input is from a file, set the filename attribute if isinstance(init, str): self.meta.filename = os.path.basename(init) elif isinstance(init, fits.HDUList): info = init.fileinfo(0) if info is not None: filename = info.get('filename') if filename is not None: self.meta.filename = os.path.basename(filename) # if the input model doesn't have a date set, use the current date/time if not self.meta.hasattr('date'): current_date = Time(datetime.datetime.now()) current_date.format = 'isot' self.meta.date = current_date.value # store the data model type, if not already set klass = self.__class__.__name__ if klass != 'DataModel': if not self.meta.hasattr('model_type'): self.meta.model_type = klass # initialize arrays from keyword arguments when they are present for attr, value in kwargs.items(): if value is not None: subschema = properties._get_schema_for_property( self._schema, attr) if 'datatype' in subschema: setattr(self, attr, value)
def runtest(self): # Make sure that each schema itself is valid. schema_tree = schema.load_schema(self.schema_path, resolver=_resolver, resolve_references=True) schema.check_schema(schema_tree)
def test_table_with_metadata(): schema = { "allOf": [ mschema.load_schema( os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), {"type": "object", "properties": { "flux_table": { "title": "Photometric flux conversion table", "fits_hdu": "FLUX", "datatype": [ {"name": "parameter", "datatype": ['ascii', 7]}, {"name": "factor", "datatype": "float64"}, {"name": "uncertainty", "datatype": "float64"} ] }, "meta": { "type": "object", "properties": { "fluxinfo": { "title": "Information about the flux conversion", "type": "object", "properties": { "exposure": { "title": "Description of exposure analyzed", "type": "string", "fits_hdu": "FLUX", "fits_keyword": "FLUXEXP" } } } } } } } ] } class FluxModel(DataModel): def __init__(self, init=None, flux_table=None, **kwargs): super(FluxModel, self).__init__(init=init, schema=schema, **kwargs) if flux_table is not None: self.flux_table = flux_table flux_im = [ ('F560W', 1.0e-5, 1.0e-7), ('F770W', 1.1e-5, 1.6e-7), ] with FluxModel(flux_table=flux_im) as datamodel: datamodel.meta.fluxinfo.exposure = 'Exposure info' datamodel.save(TMP_FITS, clobber=True) del datamodel from astropy.io import fits hdulist = fits.open(TMP_FITS) assert len(hdulist) == 3 assert isinstance(hdulist[1], fits.BinTableHDU) assert hdulist[1].name == 'FLUX' assert isinstance(hdulist[2], fits.ImageHDU) assert hdulist[2].name == 'ASDF'
def test_datamodel_schema_entry_points(): """Test that entry points for DataModelExtension works as expected""" resolver = asdf.AsdfFile().resolver mschema.load_schema('http://stsci.edu/schemas/jwst_datamodel/image.schema', resolver=resolver, resolve_references=True)
def test_replace_table(): from astropy.io import fits schema_narrow = { "allOf": [ mschema.load_schema( os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "data": { "title": "relative sensitivity table", "fits_hdu": "RELSENS", "datatype": [ {"name": "TYPE", "datatype": ["ascii", 16]}, {"name": "T_OFFSET", "datatype": "float32"}, {"name": "DECAY_PEAK", "datatype": "float32"}, {"name": "DECAY_FREQ", "datatype": "float32"}, {"name": "TAU", "datatype": "float32"} ] } } } ] } schema_wide = { "allOf": [ mschema.load_schema( os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "data": { "title": "relative sensitivity table", "fits_hdu": "RELSENS", "datatype": [ {"name": "TYPE", "datatype": ["ascii", 16]}, {"name": "T_OFFSET", "datatype": "float64"}, {"name": "DECAY_PEAK", "datatype": "float64"}, {"name": "DECAY_FREQ", "datatype": "float64"}, {"name": "TAU", "datatype": "float64"} ] } } } ] } x = np.array([("string", 1., 2., 3., 4.)], dtype=[(str('TYPE'), str('S16')), (str('T_OFFSET'), np.float32), (str('DECAY_PEAK'), np.float32), (str('DECAY_FREQ'), np.float32), (str('TAU'), np.float32)]) m = DataModel(schema=schema_narrow) m.data = x m.to_fits(TMP_FITS, clobber=True) with fits.open(TMP_FITS) as hdulist: assert repr(list(x)) == repr(list(np.asarray(hdulist[1].data))) assert hdulist[1].data.dtype[1].str == '>f4' assert hdulist[1].header['TFORM2'] == 'E' with DataModel(TMP_FITS, schema=schema_wide) as m: foo = m.data m.to_fits(TMP_FITS2, clobber=True) with fits.open(TMP_FITS2) as hdulist: assert repr(list(x)) == repr(list(np.asarray(hdulist[1].data))) assert hdulist[1].data.dtype[1].str == '>f8' assert hdulist[1].header['TFORM2'] == 'D'
def test_replace_table(): from astropy.io import fits schema_narrow = { "allOf": [ mschema.load_schema(os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "data": { "title": "relative sensitivity table", "fits_hdu": "RELSENS", "datatype": [{ "name": "TYPE", "datatype": ["ascii", 16] }, { "name": "T_OFFSET", "datatype": "float32" }, { "name": "DECAY_PEAK", "datatype": "float32" }, { "name": "DECAY_FREQ", "datatype": "float32" }, { "name": "TAU", "datatype": "float32" }] } } } ] } schema_wide = { "allOf": [ mschema.load_schema(os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "data": { "title": "relative sensitivity table", "fits_hdu": "RELSENS", "datatype": [{ "name": "TYPE", "datatype": ["ascii", 16] }, { "name": "T_OFFSET", "datatype": "float64" }, { "name": "DECAY_PEAK", "datatype": "float64" }, { "name": "DECAY_FREQ", "datatype": "float64" }, { "name": "TAU", "datatype": "float64" }] } } } ] } x = np.array([("string", 1., 2., 3., 4.)], dtype=[('TYPE', 'S16'), ('T_OFFSET', np.float32), ('DECAY_PEAK', np.float32), ('DECAY_FREQ', np.float32), ('TAU', np.float32)]) m = DataModel(schema=schema_narrow) m.data = x m.to_fits(TMP_FITS, overwrite=True) with fits.open(TMP_FITS, memmap=False) as hdulist: assert records_equal(x, np.asarray(hdulist[1].data)) assert hdulist[1].data.dtype[1].str == '>f4' assert hdulist[1].header['TFORM2'] == 'E' with DataModel(TMP_FITS, schema=schema_wide) as m: m.to_fits(TMP_FITS2, overwrite=True) with fits.open(TMP_FITS2, memmap=False) as hdulist: assert records_equal(x, np.asarray(hdulist[1].data)) assert hdulist[1].data.dtype[1].str == '>f8' assert hdulist[1].header['TFORM2'] == 'D'
def __init__(self, init=None, schema=None, memmap=False, pass_invalid_values=None, strict_validation=None, validate_on_assignment=None, ignore_missing_extensions=True, **kwargs): """ Parameters ---------- init : str, tuple, `~astropy.io.fits.HDUList`, ndarray, dict, None - None : Create a default data model with no shape. - tuple : Shape of the data array. Initialize with empty data array with shape specified by the. - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - `~astropy.io.fits.HDUList` : Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : dict, str (optional) Tree of objects representing a JSON schema, or string naming a schema. The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. memmap : bool Turn memmap of FITS file on or off. (default: False). Ignored for ASDF files. pass_invalid_values : bool or None If `True`, values that do not validate the schema will be added to the metadata. If `False`, they will be set to `None`. If `None`, value will be taken from the environmental PASS_INVALID_VALUES. Otherwise the default value is `False`. strict_validation : bool or None If `True`, schema validation errors will generate an exception. If `False`, they will generate a warning. If `None`, value will be taken from the environmental STRICT_VALIDATION. Otherwise, the default value is `False`. validate_on_assignment : bool or None Defaults to 'None'. If `None`, value will be taken from the environmental VALIDATE_ON_ASSIGNMENT, defaulting to 'True' if no environment variable is set. If 'True', attribute assignments are validated at the time of assignment. Validation errors generate warnings and values will be set to `None`. If 'False', schema validation occurs only once at the time of write. Validation errors generate warnings. ignore_missing_extensions : bool When `False`, raise warnings when a file is read that contains metadata about extensions that are not available. Defaults to `True`. kwargs : dict Additional keyword arguments passed to lower level functions. These arguments are generally file format-specific. Arguments of note are: - FITS skip_fits_update - bool or None `True` to skip updating the ASDF tree from the FITS headers, if possible. If `None`, value will be taken from the environmental SKIP_FITS_UPDATE. Otherwise, the default value is `True`. """ # Override value of validation parameters if not explicitly set. if pass_invalid_values is None: pass_invalid_values = get_envar_as_boolean("PASS_INVALID_VALUES", False) self._pass_invalid_values = pass_invalid_values if strict_validation is None: strict_validation = get_envar_as_boolean("STRICT_VALIDATION", False) if validate_on_assignment is None: validate_on_assignment = get_envar_as_boolean( "VALIDATE_ON_ASSIGNMENT", True) self._strict_validation = strict_validation self._ignore_missing_extensions = ignore_missing_extensions self._validate_on_assignment = validate_on_assignment kwargs.update({'ignore_missing_extensions': ignore_missing_extensions}) # Load the schema files if schema is None: if self.schema_url is None: schema = _DEFAULT_SCHEMA else: # Create an AsdfFile so we can use its resolver for loading schemas schema = asdf_schema.load_schema(self.schema_url, resolve_references=True) self._schema = mschema.merge_property_trees(schema) # Provide the object as context to other classes and functions self._ctx = self # Initialize with an empty AsdfFile instance as this is needed for # reading in FITS files where validate._check_value() gets called, and # ctx needs to have an _asdf attribute. self._asdf = AsdfFile() # Determine what kind of input we have (init) and execute the # proper code to intiailize the model self._files_to_close = [] self._iscopy = False is_array = False is_shape = False shape = None if init is None: asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, dict): asdffile = self.open_asdf(init=init, **kwargs) elif isinstance(init, np.ndarray): asdffile = self.open_asdf(init=None, **kwargs) shape = init.shape is_array = True elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init is_shape = True asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, DataModel): asdffile = None self.clone(self, init) if not isinstance(init, self.__class__): self.validate() return elif isinstance(init, AsdfFile): asdffile = init elif isinstance(init, fits.HDUList): asdffile = fits_support.from_fits(init, self._schema, self._ctx, **kwargs) elif isinstance(init, (str, bytes, PurePath)): if isinstance(init, PurePath): init = str(init) if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) file_type = filetype.check(init) if file_type == "fits": if s3_utils.is_s3_uri(init): init_fitsopen = s3_utils.get_object(init) memmap = None else: init_fitsopen = init hdulist = fits.open(init_fitsopen, memmap=memmap) asdffile = fits_support.from_fits(hdulist, self._schema, self._ctx, **kwargs) self._files_to_close.append(hdulist) elif file_type == "asdf": asdffile = self.open_asdf(init=init, **kwargs) else: # TODO handle json files as well raise IOError( "File does not appear to be a FITS or ASDF file.") else: raise ValueError("Can't initialize datamodel using {0}".format( str(type(init)))) # Initialize object fields as determined from the code above self._shape = shape self._instance = asdffile.tree self._asdf = asdffile # Initalize class dependent hidden fields self._no_asdf_extension = False # Instantiate the primary array of the image if is_array: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) # If a shape has been given, initialize the primary array. if is_shape: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Shape passed to DataModel.__init__, but model has " "no primary array in its schema") # Initialization occurs when the primary array is first # referenced. Do so now. getattr(self, primary_array_name) # initialize arrays from keyword arguments when they are present for attr, value in kwargs.items(): if value is not None: subschema = properties._get_schema_for_property( self._schema, attr) if 'datatype' in subschema: setattr(self, attr, value) # Call hook that sets model properties self.on_init(init)
def __init__(self, init=None, schema=None, extensions=None, pass_invalid_values=False): """ Parameters ---------- init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None - None: A default data model with no shape - shape tuple: Initialize with empty data of the given shape - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - ``astropy.io.fits.HDUList``: Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : tree of objects representing a JSON schema, or string naming a schema, optional The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. extensions: classes extending the standard set of extensions, optional. If an extension is defined, the prefix used should be 'url'. pass_invalid_values: If true, values that do not validate the schema can be read and written and only a warning will be generated """ # Set the extensions if extensions is None: extensions = jwst_extensions[:] else: extensions.extend(jwst_extensions) self._extensions = extensions # Override value of pass_invalid value if environment value set if "PASS_INVALID_VALUES" in os.environ: pass_invalid_values = os.environ["PASS_INVALID_VALUES"] try: pass_invalid_values = bool(int(pass_invalid_values)) except ValueError: pass_invalid_values = False self._pass_invalid_values = pass_invalid_values # Construct the path to the schema files filename = os.path.abspath(inspect.getfile(self.__class__)) base_url = os.path.join(os.path.dirname(filename), 'schemas', '') # Load the schema files if schema is None: schema_path = os.path.join(base_url, self.schema_url) extension_list = asdf_extension.AsdfExtensionList(self._extensions) schema = asdf_schema.load_schema( schema_path, resolver=extension_list.url_mapping, resolve_references=True) self._schema = mschema.flatten_combiners(schema) # Determine what kind of input we have (init) and execute the # proper code to intiailize the model self._files_to_close = [] self._iscopy = False is_array = False is_shape = False shape = None if init is None: asdf = AsdfFile(extensions=extensions) elif isinstance(init, dict): asdf = AsdfFile(init, extensions=extensions) elif isinstance(init, np.ndarray): asdf = AsdfFile(extensions=extensions) shape = init.shape is_array = True elif isinstance(init, self.__class__): self.clone(self, init) return elif isinstance(init, DataModel): raise TypeError( "Passed in {0!r} is not of the expected subclass {1!r}".format( init.__class__.__name__, self.__class__.__name__)) elif isinstance(init, AsdfFile): asdf = init elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init asdf = AsdfFile() is_shape = True elif isinstance(init, fits.HDUList): asdf = fits_support.from_fits(init, self._schema, extensions, pass_invalid_values) elif isinstance(init, (six.string_types, bytes)): if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) file_type = filetype.check(init) if file_type == "fits": hdulist = fits.open(init) asdf = fits_support.from_fits(hdulist, self._schema, extensions, pass_invalid_values) self._files_to_close.append(hdulist) elif file_type == "asdf": asdf = AsdfFile.open(init, extensions=extensions) else: # TODO handle json files as well raise IOError( "File does not appear to be a FITS or ASDF file.") else: raise ValueError("Can't initialize datamodel using {0}".format( str(type(init)))) # Initialize object fields as determined fro the code above self._shape = shape self._instance = asdf.tree self._asdf = asdf self._ctx = self # if the input is from a file, set the filename attribute if isinstance(init, six.string_types): self.meta.filename = os.path.basename(init) elif isinstance(init, fits.HDUList): info = init.fileinfo(0) if info is not None: filename = info.get('filename') if filename is not None: self.meta.filename = os.path.basename(filename) # if the input model doesn't have a date set, use the current date/time if self.meta.date is None: self.meta.date = Time(datetime.datetime.now()) if hasattr(self.meta.date, 'value'): self.meta.date.format = 'isot' self.meta.date = str(self.meta.date.value) # store the data model type, if not already set if hasattr(self.meta, 'model_type'): if self.meta.model_type is None: self.meta.model_type = self.__class__.__name__ else: self.meta.model_type = None if is_array: primary_array_name = self.get_primary_array_name() if primary_array_name is None: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) # TODO this code looks useless if is_shape: getattr(self, self.get_primary_array_name())
def test_table_array_convert(): """ Test that structured arrays are converted when necessary, and reused as views when not. """ from jwst.datamodels import util table_schema = { "allOf": [ mschema.load_schema( os.path.join(os.path.dirname(__file__), "../schemas/image.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "table": { 'title': 'A structured table', 'fits_hdu': 'table', 'datatype': [ 'bool8', {'datatype': 'int16', 'name': 'my_int'}, {'datatype': ['ascii', 64], 'name': 'my_string'} ] } } } ] } table = np.array( [(42, 32000, 'foo')], dtype=[ (str('f0'), str('?')), (str('my_int'), str('=i2')), (str('my_string'), str('S64')) ]) x = util.gentle_asarray(table, dtype=[ (str('f0'), str('?')), (str('my_int'), str('=i2')), (str('my_string'), str('S64')) ]) assert x is table with DataModel(schema=table_schema) as x: x.table = table assert x.table is not table table = np.array( [(42, 32000, 'foo')], dtype=[ (str('f0'), str('?')), (str('my_int'), str('=i2')), (str('my_string'), str('S3')) ]) with DataModel(schema=table_schema) as x: x.table = table assert x.table is not table assert x.table['my_string'][0] != table['my_string'][0]
def test_data_array(): data_array_schema = { "allOf": [ mschema.load_schema(os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "arr": { 'title': 'An array of data', 'type': 'array', "fits_hdu": ["FOO", "DQ"], "items": { "title": "entry", "type": "object", "properties": { "data": { "fits_hdu": "FOO", "default": 0.0, "max_ndim": 2, "datatype": "float64" }, "dq": { "fits_hdu": "DQ", "default": 1, "datatype": "uint8" }, } } } } } ] } array1 = np.random.rand(5, 5) array2 = np.random.rand(5, 5) array3 = np.random.rand(5, 5) with DataModel(schema=data_array_schema) as x: x.arr.append(x.arr.item()) x.arr[0].data = array1 assert len(x.arr) == 1 x.arr.append(x.arr.item(data=array2)) assert len(x.arr) == 2 x.arr.append({}) assert len(x.arr) == 3 x.arr[2].data = array3 del x.arr[1] assert len(x.arr) == 2 x.to_fits(TMP_FITS, clobber=True) with DataModel(TMP_FITS, schema=data_array_schema) as x: assert len(x.arr) == 2 assert_array_almost_equal(x.arr[0].data, array1) assert_array_almost_equal(x.arr[1].data, array3) del x.arr[0] assert len(x.arr) == 1 x.arr = [] assert len(x.arr) == 0 x.arr.append({'data': np.empty((5, 5))}) assert len(x.arr) == 1 x.arr.extend([ x.arr.item(data=np.empty((5, 5))), x.arr.item(data=np.empty((5, 5)), dq=np.empty((5, 5), dtype=np.uint8)) ]) assert len(x.arr) == 3 del x.arr[1] assert len(x.arr) == 2 x.to_fits(TMP_FITS2, clobber=True) from astropy.io import fits with fits.open(TMP_FITS2) as hdulist: x = set() for hdu in hdulist: x.add((hdu.header.get('EXTNAME'), hdu.header.get('EXTVER'))) print(x) assert x == set([('FOO', 2), ('FOO', 1), ('ASDF', None), ('DQ', 2), (None, None)])
def __init__(self, init=None, schema=None, pass_invalid_values=False, strict_validation=False, **kwargs): """ Parameters ---------- init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None - None: A default data model with no shape - shape tuple: Initialize with empty data of the given shape - file path: Initialize from the given file (FITS or ASDF) - readable file object: Initialize from the given file object - ``astropy.io.fits.HDUList``: Initialize from the given `~astropy.io.fits.HDUList`. - A numpy array: Used to initialize the data array - dict: The object model tree for the data model schema : tree of objects representing a JSON schema, or string naming a schema, optional The schema to use to understand the elements on the model. If not provided, the schema associated with this class will be used. pass_invalid_values: If true, values that do not validate the schema will be added to the metadata. If false, they will be set to None strict_validation: if true, an schema validation errors will generate an excption. If false, they will generate a warning. kwargs: Aadditional arguments passed to lower level functions """ # Override value of validation parameters # if environment value set self._pass_invalid_values = self.get_envar("PASS_INVALID_VALUES", pass_invalid_values) self._strict_validation = self.get_envar("STRICT_VALIDATION", strict_validation) # Load the schema files if schema is None: schema_path = os.path.join(URL_PREFIX, self.schema_url) # Create an AsdfFile so we can use its resolver for loading schemas asdf_file = AsdfFile() schema = asdf_schema.load_schema(schema_path, resolver=asdf_file.resolver, resolve_references=True) self._schema = mschema.merge_property_trees(schema) # Provide the object as context to other classes and functions self._ctx = self # Determine what kind of input we have (init) and execute the # proper code to intiailize the model self._files_to_close = [] self._iscopy = False is_array = False is_shape = False shape = None if init is None: asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, dict): asdffile = self.open_asdf(init=init, **kwargs) elif isinstance(init, np.ndarray): asdffile = self.open_asdf(init=None, **kwargs) shape = init.shape is_array = True elif isinstance(init, tuple): for item in init: if not isinstance(item, int): raise ValueError("shape must be a tuple of ints") shape = init is_shape = True asdffile = self.open_asdf(init=None, **kwargs) elif isinstance(init, DataModel): asdffile = None self.clone(self, init) if not isinstance(init, self.__class__): self.validate() return elif isinstance(init, AsdfFile): asdffile = init elif isinstance(init, fits.HDUList): asdffile = fits_support.from_fits(init, self._schema, self._ctx) elif isinstance(init, (str, bytes)): if isinstance(init, bytes): init = init.decode(sys.getfilesystemencoding()) file_type = filetype.check(init) if file_type == "fits": hdulist = fits.open(init) asdffile = fits_support.from_fits(hdulist, self._schema, self._ctx, **kwargs) self._files_to_close.append(hdulist) elif file_type == "asdf": asdffile = self.open_asdf(init=init, **kwargs) else: # TODO handle json files as well raise IOError( "File does not appear to be a FITS or ASDF file.") else: raise ValueError( "Can't initialize datamodel using {0}".format(str(type(init)))) # Initialize object fields as determined from the code above self._shape = shape self._instance = asdffile.tree self._asdf = asdffile # Initalize class dependent hidden fields self._no_asdf_extension = False # Instantiate the primary array of the image if is_array: primary_array_name = self.get_primary_array_name() if not primary_array_name: raise TypeError( "Array passed to DataModel.__init__, but model has " "no primary array in its schema") setattr(self, primary_array_name, init) if is_shape: if not self.get_primary_array_name(): raise TypeError( "Shape passed to DataModel.__init__, but model has " "no primary array in its schema") # if the input is from a file, set the filename attribute if isinstance(init, str): self.meta.filename = os.path.basename(init) elif isinstance(init, fits.HDUList): info = init.fileinfo(0) if info is not None: filename = info.get('filename') if filename is not None: self.meta.filename = os.path.basename(filename) # if the input model doesn't have a date set, use the current date/time if not self.meta.hasattr('date'): current_date = Time(datetime.datetime.now()) current_date.format = 'isot' self.meta.date = current_date.value # store the data model type, if not already set klass = self.__class__.__name__ if klass != 'DataModel': if not self.meta.hasattr('model_type'): self.meta.model_type = klass # initialize arrays from keyword arguments when they are present for attr, value in kwargs.items(): if value is not None: subschema = properties._get_schema_for_property(self._schema, attr) if 'datatype' in subschema: setattr(self, attr, value)