def _build_schema(self):
"""Build the schema, encorporating the core."""
# Determine the schema path
filename = os.path.abspath(inspect.getfile(self.__class__))
base_url = os.path.join(
os.path.dirname(filename), 'schemas', '')
schema_path = os.path.join(base_url, self.schema_url)
core_schema_path = os.path.join(base_url, self.core_schema_url)
# Get the schemas
schema = asdf_schema.load_schema(
schema_path,
resolve_references=True
)
core_schema = asdf_schema.load_schema(
core_schema_path,
resolve_references=True
)
# Create a new core.meta that will co-locate
# with each exposure entry. This is done
# by saving the meta information in a separate
# FITS HDU.
core_meta_schema = deepcopy(core_schema['properties']['meta'])
treeutil.walk(core_meta_schema, remove_fits)
exposure_schema = schema['allOf'][1]['properties']['exposures']['items']
exposure_meta_schema = exposure_schema['allOf'][1]['properties']['meta']
exposure_meta_schema.update(core_meta_schema)
# That's all folks
return schema
def test_schema_caching():
# Make sure that if we request the same URL, we get the *exact
# same* object, to ensure the cache is working.
s1 = schema.load_schema(
'http://stsci.edu/schemas/asdf/core/asdf-1.0.0')
s2 = schema.load_schema(
'http://stsci.edu/schemas/asdf/core/asdf-1.0.0')
assert s1 is s2
def run_schema_example_test(organization, standard, name, version, check_func=None):
import asdf
from asdf.tests import helpers
from asdf.types import format_tag
from asdf.schema import load_schema
tag = format_tag(organization, standard, version, name)
uri = asdf.resolver.default_tag_to_url_mapping(tag)
r = asdf.AsdfFile().resolver
examples = []
schema = load_schema(uri, resolver=r)
for node in asdf.treeutil.iter_tree(schema):
if (isinstance(node, dict) and
'examples' in node and
isinstance(node['examples'], list)):
for desc, example in node['examples']:
examples.append(example)
for example in examples:
buff = helpers.yaml_to_asdf('example: ' + example.strip())
ff = asdf.AsdfFile(uri=uri)
# Add some dummy blocks so that the ndarray examples work
for i in range(3):
b = asdf.block.Block(np.zeros((1024*1024*8), dtype=np.uint8))
b._used = True
ff.blocks.add(b)
ff._open_impl(ff, buff, mode='r')
if check_func:
check_func(ff)
def read_schema(schema_file, extensions=None):
"""
Read a schema file from disk in order to pass it as an argument
to a new datamodel.
"""
def get_resolver(asdf_file):
extensions = asdf_file._extensions
def asdf_file_resolver(uri):
return extensions._url_mapping(extensions._tag_mapping(uri))
return asdf_file_resolver
default_extensions = [GWCSExtension(), JWSTExtension()]
if extensions is None:
extensions = default_extensions[:]
else:
extensions.extend(default_extensions)
asdf_file = AsdfFile(extensions=extensions)
if hasattr(asdf_file, 'resolver'):
file_resolver = asdf_file.resolver
else:
file_resolver = get_resolver(asdf_file)
schema = asdf_schema.load_schema(schema_file,
resolver=file_resolver,
resolve_references=True)
schema = merge_property_trees(schema)
return schema
def test_fits_without_sci():
from astropy.io import fits
schema = {
"allOf": [
mschema.load_schema(
os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True),
{
"type": "object",
"properties": {
"coeffs": {
'max_ndim': 1,
'fits_hdu': 'COEFFS',
'datatype': 'float32'
}
}
}
]
}
fits = fits.HDUList(
[fits.PrimaryHDU(),
fits.ImageHDU(name='COEFFS', data=np.array([0.0], np.float32))])
with DataModel(fits, schema=schema) as dm:
assert_array_equal(dm.coeffs, [0.0])
def test_schema_resolved_via_entry_points():
"""Test that entry points mappings to core schema works"""
r = asdf.AsdfFile().resolver
tag = types.format_tag('stsci.edu', 'asdf', '1.0.0', 'fits/fits')
url = resolver.default_tag_to_url_mapping(tag)
s = schema.load_schema(url, resolver=r, resolve_references=True)
assert tag in repr(s)
def test_self_reference_resolution():
r = resolver.Resolver(CustomExtension().url_mapping, 'url')
s = schema.load_schema(
helpers.get_test_data_path('self_referencing-1.0.0.yaml'),
resolver=r,
resolve_references=True)
assert '$ref' not in repr(s)
assert s['anyOf'][1] == s['anyOf'][0]
def runtest(self):
from asdf import schema
from asdf.extension import default_extensions
# Make sure that each schema itself is valid.
schema_tree = schema.load_schema(
self.schema_path, resolver=default_extensions.resolver,
resolve_references=True)
schema.check_schema(schema_tree)
def test_read_json_schema():
"""Pytest to make sure reading JSON schemas succeeds.
This was known to fail on Python 3.5 See issue #314 at
https://github.com/spacetelescope/asdf/issues/314 for more details.
"""
json_schema = helpers.get_test_data_path('example_schema.json')
schema_tree = schema.load_schema(json_schema, resolve_references=True)
schema.check_schema(schema_tree)
def test_table_array():
table_schema = {
"allOf": [
mschema.load_schema(
os.path.join(os.path.dirname(__file__),
"../schemas/image.schema.yaml"),
resolve_references=True),
{
"type": "object",
"properties": {
"table": {
'title': 'A structured table',
'fits_hdu': 'table',
'datatype': [
'bool8',
{'datatype': 'int16',
'name': 'my_int'},
{'datatype': 'float32',
'name': 'my_float',
'shape': [3, 2]},
{'datatype': ['ascii', 64],
'name': 'my_string'}
]
}
}
}
]
}
with DataModel(schema=table_schema) as x:
x.table = [(True, 42, 37.5, 'STRING')]
assert x.table.dtype == [
(str('f0'), str('?')),
(str('my_int'), str('=i2')),
(str('my_float'), str('=f4'), (3, 2)),
(str('my_string'), str('S64'))
]
x.to_fits(TMP_FITS, overwrite=True)
with DataModel(TMP_FITS, schema=table_schema) as x:
table = x.table
assert table.dtype == [
(str('f0'), str('?')),
(str('my_int'), str('=i2')),
(str('my_float'), str('=f4'), (3, 2)),
(str('my_string'), str('S64'))
]
def test_property_order():
tree = {'foo': np.ndarray([1, 2, 3])}
buff = io.BytesIO()
ff = asdf.AsdfFile(tree)
ff.write_to(buff)
ndarray_schema = schema.load_schema(
'http://stsci.edu/schemas/asdf/core/ndarray-1.0.0')
property_order = ndarray_schema['anyOf'][1]['propertyOrder']
last_index = 0
for prop in property_order:
index = buff.getvalue().find(prop.encode('utf-8') + b':')
if index != -1:
assert index > last_index
last_index = index
def test_time_tag():
schema = asdf_schema.load_schema(
'http://stsci.edu/schemas/asdf/time/time-1.1.0',
resolve_references=True)
schema = _flatten_combiners(schema)
date = time.Time(datetime.datetime.now())
tree = {'date': date}
asdf = AsdfFile(tree=tree)
instance = yamlutil.custom_tree_to_tagged_tree(tree['date'], asdf)
asdf_schema.validate(instance, schema=schema)
tag = 'tag:stsci.edu:asdf/time/time-1.1.0'
date = tagged.tag_object(tag, date)
tree = {'date': date}
asdf = AsdfFile(tree=tree)
instance = yamlutil.custom_tree_to_tagged_tree(tree['date'], asdf)
asdf_schema.validate(instance, schema=schema)
def test_load_schema(tmpdir):
schema_def = """
%YAML 1.1
---
$schema: "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0"
id: "http://stsci.edu/schemas/asdf/nugatory/nugatory-1.0.0"
tag: "tag:stsci.edu:asdf/nugatory/nugatory-1.0.0"
type: object
properties:
foobar:
$ref: "../core/ndarray-1.0.0"
required: [foobar]
...
"""
schema_path = tmpdir.join('nugatory.yaml')
schema_path.write(schema_def.encode())
schema_tree = schema.load_schema(str(schema_path), resolve_references=True)
schema.check_schema(schema_tree)
def test_load_schema(tmpdir):
schema_def = """
%YAML 1.1
---
$schema: "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0"
id: "http://stsci.edu/schemas/asdf/nugatory/nugatory-1.0.0"
tag: "tag:stsci.edu:asdf/nugatory/nugatory-1.0.0"
type: object
properties:
foobar:
$ref: "../core/ndarray-1.0.0"
required: [foobar]
...
"""
schema_path = tmpdir.join('nugatory.yaml')
schema_path.write(schema_def.encode())
schema_tree = schema.load_schema(str(schema_path), resolve_references=True)
schema.check_schema(schema_tree)
def test_load_schema_with_file_url(tmpdir):
schema_def = """
%YAML 1.1
%TAG !asdf! tag:stsci.edu:asdf/
---
$schema: "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0"
id: "http://stsci.edu/schemas/asdf/nugatory/nugatory-1.0.0"
tag: "tag:stsci.edu:asdf/nugatory/nugatory-1.0.0"
type: object
properties:
foobar:
$ref: "{}"
required: [foobar]
...
""".format(resolver.default_resolver('tag:stsci.edu:asdf/core/ndarray-1.0.0'))
schema_path = tmpdir.join('nugatory.yaml')
schema_path.write(schema_def.encode())
schema_tree = schema.load_schema(str(schema_path), resolve_references=True)
schema.check_schema(schema_tree)
def test_load_schema_with_file_url(tmpdir):
schema_def = """
%YAML 1.1
%TAG !asdf! tag:stsci.edu:asdf/
---
$schema: "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0"
id: "http://stsci.edu/schemas/asdf/nugatory/nugatory-1.0.0"
tag: "tag:stsci.edu:asdf/nugatory/nugatory-1.0.0"
type: object
properties:
foobar:
$ref: "{}"
required: [foobar]
...
""".format(resolver.default_resolver('tag:stsci.edu:asdf/core/ndarray-1.0.0'))
schema_path = tmpdir.join('nugatory.yaml')
schema_path.write(schema_def.encode())
schema_tree = schema.load_schema(str(schema_path), resolve_references=True)
schema.check_schema(schema_tree)
def test_load_schema_with_asdf_uri_scheme():
subschema_content = """%YAML 1.1
---
$schema: http://stsci.edu/schemas/asdf/asdf-schema-1.0.0
id: asdf://somewhere.org/schemas/bar
bar:
type: string
...
"""
content = """%YAML 1.1
---
$schema: http://stsci.edu/schemas/asdf/asdf-schema-1.0.0
id: asdf://somewhere.org/schemas/foo
definitions:
local_bar:
type: string
type: object
properties:
bar:
$ref: asdf://somewhere.org/schemas/bar#/bar
local_bar:
$ref: '#/definitions/local_bar'
...
"""
with asdf.config_context() as config:
config.add_resource_mapping(
{"asdf://somewhere.org/schemas/foo": content})
config.add_resource_mapping(
{"asdf://somewhere.org/schemas/bar": subschema_content})
schema_tree = schema.load_schema("asdf://somewhere.org/schemas/foo")
instance = {"bar": "baz", "local_bar": "foz"}
schema.validate(instance, schema=schema_tree)
with pytest.raises(ValidationError):
schema.validate({"bar": 12}, schema=schema_tree)
def test_nested_array_yaml(tmpdir):
schema_def = """
%YAML 1.1
---
type: object
properties:
stuff:
type: array
items:
type: array
items:
- type: integer
- type: string
- type: number
minItems: 3
maxItems: 3
...
"""
schema_path = tmpdir.join('nested.yaml')
schema_path.write(schema_def.encode())
schema_tree = schema.load_schema(str(schema_path))
schema.check_schema(schema_tree)
good = dict(stuff=[[1, 'hello', 2], [4, 'world', 9.7]])
schema.validate(good, schema=schema_tree)
bads = [
dict(stuff=[[1, 2, 3]]),
dict(stuff=[12, 'dldl']),
dict(stuff=[[12, 'dldl']]),
dict(stuff=[[1, 'hello', 2], [4, 5]]),
dict(stuff=[[1, 'hello', 2], [4, 5, 6]])
]
for b in bads:
with pytest.raises(ValidationError):
schema.validate(b, schema=schema_tree)
def test_nested_array_yaml(tmpdir):
schema_def = """
%YAML 1.1
---
type: object
properties:
stuff:
type: array
items:
type: array
items:
- type: integer
- type: string
- type: number
minItems: 3
maxItems: 3
...
"""
schema_path = tmpdir.join('nested.yaml')
schema_path.write(schema_def.encode())
schema_tree = schema.load_schema(str(schema_path))
schema.check_schema(schema_tree)
good = dict(stuff=[[1, 'hello', 2], [4, 'world', 9.7]])
schema.validate(good, schema=schema_tree)
bads = [
dict(stuff=[[1, 2, 3]]),
dict(stuff=[12,'dldl']),
dict(stuff=[[12, 'dldl']]),
dict(stuff=[[1, 'hello', 2], [4, 5]]),
dict(stuff=[[1, 'hello', 2], [4, 5, 6]])
]
for b in bads:
with pytest.raises(ValidationError):
schema.validate(b, schema=schema_tree)
def test_fits_without_sci():
schema = {
"allOf": [
mschema.load_schema("http://stsci.edu/schemas/jwst_datamodel/core.schema",
resolve_references=True),
{
"type": "object",
"properties": {
"coeffs": {
'max_ndim': 1,
'fits_hdu': 'COEFFS',
'datatype': 'float32'
}
}
}
]
}
hdulist = fits.HDUList()
hdulist.append(fits.PrimaryHDU())
hdulist.append(fits.ImageHDU(name='COEFFS', data=np.array([0.0], np.float32)))
with JwstDataModel(hdulist, schema=schema) as dm:
assert_array_equal(dm.coeffs, [0.0])
def test_schema_caching():
# Make sure that if we request the same URL, we get the *exact
# same* object, to ensure the cache is working.
s1 = schema.load_schema('http://stsci.edu/schemas/asdf/core/asdf-1.0.0')
s2 = schema.load_schema('http://stsci.edu/schemas/asdf/core/asdf-1.0.0')
assert s1 is s2
with pytest.raises(jsonschema.ValidationError):
with ImageModel((50, 50)) as dm:
dm.meta.date = 'Not an acceptable date'
def test_date2():
from astropy import time
with ImageModel((50, 50)) as dm:
assert isinstance(dm.meta.date, (time.Time, datetime.datetime))
'''
transformation_schema = {
"allOf": [
mschema.load_schema(
os.path.join(os.path.dirname(__file__),
"../schemas/image.schema.yaml"),
resolve_references=True),
{
"type": "object",
"properties": {
"meta": {
"type": "object",
"properties": {
"transformations": {
"title": "A list of transformations",
"type": "array",
"items": {
"title": "A transformation",
"type": "object",
"properties": {
"type": {
def test_load_schema_resolve_local_refs_deprecated():
custom_schema_path = helpers.get_test_data_path(
'custom_schema_definitions.yaml')
with pytest.deprecated_call():
schema.load_schema(custom_schema_path, resolve_local_refs=True)
def __init__(self, init=None, schema=None, extensions=None,
pass_invalid_values=False):
"""
Parameters
----------
init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None
- None: A default data model with no shape
- shape tuple: Initialize with empty data of the given
shape
- file path: Initialize from the given file (FITS or ASDF)
- readable file object: Initialize from the given file
object
- ``astropy.io.fits.HDUList``: Initialize from the given
`~astropy.io.fits.HDUList`.
- A numpy array: Used to initialize the data array
- dict: The object model tree for the data model
schema : tree of objects representing a JSON schema, or string naming a schema, optional
The schema to use to understand the elements on the model.
If not provided, the schema associated with this class
will be used.
extensions: classes extending the standard set of extensions
pass_invalid_values: If True, values that do not validate the schema can
be read and written, but with a warning message
"""
filename = os.path.abspath(inspect.getfile(self.__class__))
base_url = os.path.join(
os.path.dirname(filename), 'schemas', '')
if schema is None:
schema_path = os.path.join(base_url, self.schema_url)
schema = asdf_schema.load_schema(
schema_path, resolve_references=True)
self._schema = mschema.flatten_combiners(schema)
if extensions is not None:
extensions.extend(jwst_extensions)
else:
extensions = jwst_extensions[:]
self._extensions = extensions
if "PASS_INVALID_VALUES" in os.environ:
pass_invalid_values = os.environ["PASS_INVALID_VALUES"]
try:
self._pass_invalid_values = bool(int(pass_invalid_values))
except ValueError:
self._pass_invalid_values = False
else:
self._pass_invalid_values = pass_invalid_values
self._files_to_close = []
is_array = False
is_shape = False
shape = None
if init is None:
asdf = AsdfFile(extensions=extensions)
elif isinstance(init, dict):
asdf = AsdfFile(init, extensions=extensions)
elif isinstance(init, np.ndarray):
asdf = AsdfFile(extensions=extensions)
shape = init.shape
is_array = True
elif isinstance(init, self.__class__):
instance = copy.deepcopy(init._instance)
self._schema = init._schema
self._shape = init._shape
self._asdf = AsdfFile(instance, extensions=self._extensions)
self._instance = instance
self._ctx = self
self.__class__ = init.__class__
return
elif isinstance(init, DataModel):
raise TypeError(
"Passed in {0!r} is not of the expected subclass {1!r}".format(
init.__class__.__name__, self.__class__.__name__))
elif isinstance(init, AsdfFile):
asdf = init
elif isinstance(init, tuple):
for item in init:
if not isinstance(item, int):
raise ValueError("shape must be a tuple of ints")
shape = init
asdf = AsdfFile()
is_shape = True
elif isinstance(init, fits.HDUList):
asdf = fits_support.from_fits(init, self._schema,
extensions=self._extensions,
validate=False,
pass_invalid_values=self._pass_invalid_values)
elif isinstance(init, six.string_types):
if isinstance(init, bytes):
init = init.decode(sys.getfilesystemencoding())
try:
hdulist = fits.open(init)
except IOError:
try:
asdf = AsdfFile.open(init, extensions=self._extensions)
# TODO: Add json support
except ValueError:
raise IOError(
"File does not appear to be a FITS or ASDF file.")
else:
asdf = fits_support.from_fits(hdulist, self._schema,
extensions=self._extensions,
validate=False,
pass_invalid_values=self._pass_invalid_values)
self._files_to_close.append(hdulist)
self._shape = shape
self._instance = asdf.tree
self._asdf = asdf
self._ctx = self
# if the input model doesn't have a date set, use the current date/time
if self.meta.date is None:
self.meta.date = Time(datetime.datetime.now())
self.meta.date.format='isot'
self.meta.date = self.meta.date.value
# if the input is from a file, set the filename attribute
if isinstance(init, six.string_types):
self.meta.filename = os.path.basename(init)
if is_array:
primary_array_name = self.get_primary_array_name()
if primary_array_name is None:
raise TypeError(
"Array passed to model.__init__, but model has no primary "
"array in its schema")
setattr(self, primary_array_name, init)
if is_shape:
getattr(self, self.get_primary_array_name())
# Setting an invalid value should raise a ValueError
dm.meta.target.ra = "FOO"
def test_date2():
with ImageModel((50, 50), strict_validation=True) as dm:
time_obj = time.Time(dm.meta.date)
assert isinstance(time_obj, time.Time)
date_obj = datetime.strptime(dm.meta.date, '%Y-%m-%dT%H:%M:%S.%f')
assert isinstance(date_obj, datetime)
TRANSFORMATION_SCHEMA = {
"allOf": [
mschema.load_schema(os.path.join(URL_PREFIX, "image.schema"),
resolver=asdf.AsdfFile().resolver,
resolve_references=True),
{
"type": "object",
"properties": {
"meta": {
"type": "object",
"properties": {
"transformations": {
"title": "A list of transformations",
"type": "array",
"items": {
"title": "A transformation",
"type": "object",
"properties": {
"type": {
from asdf.schema import load_schema
from asdf import generic_io
from custom_module.custom_extension import CustomExtensions
from custom_module.custom_types import CustomClass
my_float = 12.45
my_list = [3, 4, 6]
my_dict = {"a": 1, "b": 42}
cc = CustomClass(2.45, [2, 8, 64])
tree = {
"cc": cc,
}
s = load_schema("custom_module/schemas/CustomClass-alternative.yaml")
s_string = generic_io.get_file(
"custom_module/schemas/CustomClass-alternative.yaml").read().decode(
'utf-8')
asdf.get_config().add_resource_mapping({
"http://CustomOrganization/schemas/CustomStandard/CustomClass-1.0.0":
s_string
})
print("Script --- Create AsdfFile instance")
af = asdf.AsdfFile(
tree,
extensions=CustomExtensions(),
custom_schema=f"custom_module\\schemas\\CustomSchema.yaml",
)
def __init__(self,
init=None,
schema=None,
extensions=None,
pass_invalid_values=False,
strict_validation=False,
**kwargs):
"""
Gets the path to the schema using a different prefix then calls the constructor of DataModel
Parameters
----------
init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None
- None: A default data model with no shape
- shape tuple: Initialize with empty data of the given
shape
- file path: Initialize from the given file (FITS or ASDF)
- readable file object: Initialize from the given file
object
- ``astropy.io.fits.HDUList``: Initialize from the given
`~astropy.io.fits.HDUList`.
- A numpy array: Used to initialize the data array
- dict: The object model tree for the data model
schema : tree of objects representing a JSON schema, or string naming a schema, optional
The schema to use to understand the elements on the model.
If not provided, the schema associated with this class
will be used.
extensions: classes extending the standard set of extensions, optional.
If an extension is defined, the prefix used should be 'url'.
pass_invalid_values: If true, values that do not validate the schema
will be added to the metadata. If false, they will be set to None
strict_validation: if true, an schema validation errors will generate
an excption. If false, they will generate a warning.
kwargs: Aadditional arguments passed to lower level functions
"""
URL_PREFIX = "http://oirlab.ucsd.edu/schemas/"
# Load the schema files
if schema is None:
schema_path = os.path.join(URL_PREFIX, self.schema_url)
# Create an AsdfFile so we can use its resolver for loading schemas
asdf_file = AsdfFile()
schema = asdf_schema.load_schema(schema_path,
resolver=asdf_file.resolver,
resolve_references=True)
super().__init__(init=init,
schema=schema,
extensions=extensions,
pass_invalid_values=pass_invalid_values,
strict_validation=strict_validation,
**kwargs)
def test_table_array_shape_ndim():
table_schema = {
"allOf": [
mschema.load_schema(os.path.join(URL_PREFIX, "image.schema"),
resolver=asdf.AsdfFile().resolver,
resolve_references=True),
{
"type": "object",
"properties": {
"table": {
'title': 'A structured table',
'fits_hdu': 'table',
'datatype': [
'bool8',
{'datatype': 'int16',
'name': 'my_int'},
{'datatype': 'float32',
'name': 'my_float1',
'shape': [3, 2]},
{'datatype': 'float32',
'name': 'my_float2',
'ndim': 2},
{'datatype': 'float32',
'name': 'my_float3'},
{'datatype': 'float32',
'name': 'my_float4'},
{'datatype': 'float32',
'name': 'my_float5'},
{'datatype': ['ascii', 64],
'name': 'my_string'}
]
}
}
}
]
}
with DataModel(schema=table_schema) as x:
x.table = [
(
True,
42,
[[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]],
[[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]],
[[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]],
[37.5, 38.0],
37.5,
'STRING'
)
]
assert x.table.dtype == [
('f0', '?'),
('my_int', '=i2'),
('my_float1', '=f4', (3, 2)),
('my_float2', '=f4', (3, 2)),
('my_float3', '=f4', (3, 2)),
('my_float4', '=f4', (2,)),
('my_float5', '=f4'),
('my_string', 'S64')
]
x.to_fits(TMP_FITS, overwrite=True)
with DataModel(TMP_FITS, schema=table_schema) as x:
assert x.table.dtype == [
('f0', '?'),
('my_int', '=i2'),
('my_float1', '=f4', (3, 2)),
('my_float2', '=f4', (3, 2)),
('my_float3', '=f4', (3, 2)),
('my_float4', '=f4', (2,)),
('my_float5', '=f4'),
('my_string', 'S64')
]
table_schema['allOf'][1]['properties']['table']['datatype'][3]['ndim'] = 3
with DataModel(schema=table_schema) as x:
with pytest.raises(ValueError) as e:
x.table = [
(
True,
42,
[[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]],
[[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]],
[[37.5, 38.0], [39.0, 40.0], [41.0, 42.0]],
[37.5, 38.0],
37.5,
'STRING'
)
]
assert str(e.value).startswith("Array has wrong number of dimensions.")
def __init__(self,
init=None,
schema=None,
extensions=None,
pass_invalid_values=False):
"""
Parameters
----------
init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None
- None: A default data model with no shape
- shape tuple: Initialize with empty data of the given
shape
- file path: Initialize from the given file (FITS or ASDF)
- readable file object: Initialize from the given file
object
- ``astropy.io.fits.HDUList``: Initialize from the given
`~astropy.io.fits.HDUList`.
- A numpy array: Used to initialize the data array
- dict: The object model tree for the data model
schema : tree of objects representing a JSON schema, or string naming a schema, optional
The schema to use to understand the elements on the model.
If not provided, the schema associated with this class
will be used.
extensions: classes extending the standard set of extensions
pass_invalid_values: If True, values that do not validate the schema can
be read and written, but with a warning message
"""
filename = os.path.abspath(inspect.getfile(self.__class__))
base_url = os.path.join(os.path.dirname(filename), 'schemas', '')
if schema is None:
schema_path = os.path.join(base_url, self.schema_url)
schema = asdf_schema.load_schema(schema_path,
resolve_references=True)
self._schema = mschema.flatten_combiners(schema)
if extensions is not None:
extensions.extend(jwst_extensions)
else:
extensions = jwst_extensions[:]
self._extensions = extensions
if "PASS_INVALID_VALUES" in os.environ:
pass_invalid_values = os.environ["PASS_INVALID_VALUES"]
try:
self._pass_invalid_values = bool(int(pass_invalid_values))
except ValueError:
self._pass_invalid_values = False
else:
self._pass_invalid_values = pass_invalid_values
self._files_to_close = []
is_array = False
is_shape = False
shape = None
if init is None:
asdf = AsdfFile(extensions=extensions)
elif isinstance(init, dict):
asdf = AsdfFile(init, extensions=extensions)
elif isinstance(init, np.ndarray):
asdf = AsdfFile(extensions=extensions)
shape = init.shape
is_array = True
elif isinstance(init, self.__class__):
instance = copy.deepcopy(init._instance)
self._schema = init._schema
self._shape = init._shape
self._asdf = AsdfFile(instance, extensions=self._extensions)
self._instance = instance
self._ctx = self
self.__class__ = init.__class__
return
elif isinstance(init, DataModel):
raise TypeError(
"Passed in {0!r} is not of the expected subclass {1!r}".format(
init.__class__.__name__, self.__class__.__name__))
elif isinstance(init, AsdfFile):
asdf = init
elif isinstance(init, tuple):
for item in init:
if not isinstance(item, int):
raise ValueError("shape must be a tuple of ints")
shape = init
asdf = AsdfFile()
is_shape = True
elif isinstance(init, fits.HDUList):
asdf = fits_support.from_fits(
init,
self._schema,
extensions=self._extensions,
validate=False,
pass_invalid_values=self._pass_invalid_values)
elif isinstance(init, six.string_types):
if isinstance(init, bytes):
init = init.decode(sys.getfilesystemencoding())
try:
hdulist = fits.open(init)
except IOError:
try:
asdf = AsdfFile.open(init, extensions=self._extensions)
# TODO: Add json support
except ValueError:
raise IOError(
"File does not appear to be a FITS or ASDF file.")
else:
asdf = fits_support.from_fits(
hdulist,
self._schema,
extensions=self._extensions,
validate=False,
pass_invalid_values=self._pass_invalid_values)
self._files_to_close.append(hdulist)
self._shape = shape
self._instance = asdf.tree
self._asdf = asdf
self._ctx = self
# if the input model doesn't have a date set, use the current date/time
if self.meta.date is None:
self.meta.date = Time(datetime.datetime.now())
self.meta.date.format = 'isot'
self.meta.date = self.meta.date.value
# if the input is from a file, set the filename attribute
if isinstance(init, six.string_types):
self.meta.filename = os.path.basename(init)
if is_array:
primary_array_name = self.get_primary_array_name()
if primary_array_name is None:
raise TypeError(
"Array passed to DataModel.__init__, but model has "
"no primary array in its schema")
setattr(self, primary_array_name, init)
if is_shape:
getattr(self, self.get_primary_array_name())
def __init__(self,
init=None,
schema=None,
extensions=None,
pass_invalid_values=False,
strict_validation=False):
"""
Parameters
----------
init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None
- None: A default data model with no shape
- shape tuple: Initialize with empty data of the given
shape
- file path: Initialize from the given file (FITS or ASDF)
- readable file object: Initialize from the given file
object
- ``astropy.io.fits.HDUList``: Initialize from the given
`~astropy.io.fits.HDUList`.
- A numpy array: Used to initialize the data array
- dict: The object model tree for the data model
schema : tree of objects representing a JSON schema, or string naming a schema, optional
The schema to use to understand the elements on the model.
If not provided, the schema associated with this class
will be used.
extensions: classes extending the standard set of extensions, optional.
If an extension is defined, the prefix used should be 'url'.
pass_invalid_values: If true, values that do not validate the schema
will be added to the metadata. If false, they will be set to None
strict_validation: if true, an schema validation errors will generate
an excption. If false, they will generate a warning.
"""
# Set the extensions
self._extensions = extensions
# Override value of validation parameters
# if environment value set
self._pass_invalid_values = self.get_envar("PASS_INVALID_VALUES",
pass_invalid_values)
self._strict_validation = self.get_envar("STRICT_VALIDATION",
strict_validation)
# Construct the path to the schema files
filename = os.path.abspath(inspect.getfile(self.__class__))
base_url = os.path.join(os.path.dirname(filename), 'schemas', '')
# Load the schema files
if schema is None:
schema_path = os.path.join(base_url, self.schema_url)
# Create an AsdfFile so we can use its resolver for loading schemas
asdf_file = AsdfFile(extensions=self._extensions)
if hasattr(asdf_file, 'resolver'):
file_resolver = asdf_file.resolver
else:
file_resolver = self.get_resolver(asdf_file)
schema = asdf_schema.load_schema(schema_path,
resolver=file_resolver,
resolve_references=True)
self._schema = mschema.flatten_combiners(schema)
# Provide the object as context to other classes and functions
self._ctx = self
# Determine what kind of input we have (init) and execute the
# proper code to intiailize the model
self._files_to_close = []
self._iscopy = False
is_array = False
is_shape = False
shape = None
if init is None:
asdf = AsdfFile(extensions=extensions)
elif isinstance(init, dict):
asdf = AsdfFile(init, extensions=self._extensions)
elif isinstance(init, np.ndarray):
asdf = AsdfFile(extensions=self._extensions)
shape = init.shape
is_array = True
elif isinstance(init, tuple):
for item in init:
if not isinstance(item, int):
raise ValueError("shape must be a tuple of ints")
shape = init
asdf = AsdfFile()
is_shape = True
elif isinstance(init, DataModel):
self.clone(self, init)
if not isinstance(init, self.__class__):
self.validate()
return
elif isinstance(init, AsdfFile):
asdf = init
elif isinstance(init, fits.HDUList):
asdf = fits_support.from_fits(init, self._schema, self._extensions,
self._ctx)
elif isinstance(init, (str, bytes)):
if isinstance(init, bytes):
init = init.decode(sys.getfilesystemencoding())
file_type = filetype.check(init)
if file_type == "fits":
hdulist = fits.open(init)
asdf = fits_support.from_fits(hdulist, self._schema,
self._extensions, self._ctx)
self._files_to_close.append(hdulist)
elif file_type == "asdf":
asdf = AsdfFile.open(init, extensions=self._extensions)
else:
# TODO handle json files as well
raise IOError(
"File does not appear to be a FITS or ASDF file.")
else:
raise ValueError("Can't initialize datamodel using {0}".format(
str(type(init))))
# Initialize object fields as determined from the code above
self._shape = shape
self._instance = asdf.tree
self._asdf = asdf
# Initalize class dependent hidden fields
self._no_asdf_extension = False
# Instantiate the primary array of the image
if is_array:
primary_array_name = self.get_primary_array_name()
if not primary_array_name:
raise TypeError(
"Array passed to DataModel.__init__, but model has "
"no primary array in its schema")
setattr(self, primary_array_name, init)
if is_shape:
if not self.get_primary_array_name():
raise TypeError(
"Shape passed to DataModel.__init__, but model has "
"no primary array in its schema")
# if the input is from a file, set the filename attribute
if isinstance(init, str):
self.meta.filename = os.path.basename(init)
elif isinstance(init, fits.HDUList):
info = init.fileinfo(0)
if info is not None:
filename = info.get('filename')
if filename is not None:
self.meta.filename = os.path.basename(filename)
# if the input model doesn't have a date set, use the current date/time
if not self.meta.hasattr('date'):
current_date = Time(datetime.datetime.now())
current_date.format = 'isot'
self.meta.date = current_date.value
# store the data model type, if not already set
klass = self.__class__.__name__
if klass != 'DataModel':
if not self.meta.hasattr('model_type'):
self.meta.model_type = klass
def test_table_with_metadata():
schema = {
"allOf": [
mschema.load_schema(os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True),
{
"type": "object",
"properties": {
"flux_table": {
"title":
"Photometric flux conversion table",
"fits_hdu":
"FLUX",
"datatype": [{
"name": "parameter",
"datatype": ['ascii', 7]
}, {
"name": "factor",
"datatype": "float64"
}, {
"name": "uncertainty",
"datatype": "float64"
}]
},
"meta": {
"type": "object",
"properties": {
"fluxinfo": {
"title":
"Information about the flux conversion",
"type": "object",
"properties": {
"exposure": {
"title":
"Description of exposure analyzed",
"type": "string",
"fits_hdu": "FLUX",
"fits_keyword": "FLUXEXP"
}
}
}
}
}
}
}
]
}
class FluxModel(DataModel):
def __init__(self, init=None, flux_table=None, **kwargs):
super(FluxModel, self).__init__(init=init, schema=schema, **kwargs)
if flux_table is not None:
self.flux_table = flux_table
flux_im = [
('F560W', 1.0e-5, 1.0e-7),
('F770W', 1.1e-5, 1.6e-7),
]
with FluxModel(flux_table=flux_im) as datamodel:
datamodel.meta.fluxinfo.exposure = 'Exposure info'
datamodel.save(TMP_FITS, overwrite=True)
del datamodel
from astropy.io import fits
with fits.open(TMP_FITS, memmap=False) as hdulist:
assert len(hdulist) == 3
assert isinstance(hdulist[1], fits.BinTableHDU)
assert hdulist[1].name == 'FLUX'
assert hdulist[2].name == 'ASDF'
def test_table_array_convert():
"""
Test that structured arrays are converted when necessary, and
reused as views when not.
"""
from jwst.datamodels import util
table_schema = {
"allOf": [
mschema.load_schema(
os.path.join(os.path.dirname(__file__),
"../schemas/image.schema.yaml"),
resolve_references=True),
{
"type": "object",
"properties": {
"table": {
'title': 'A structured table',
'fits_hdu': 'table',
'datatype': [
'bool8',
{'datatype': 'int16',
'name': 'my_int'},
{'datatype': ['ascii', 64],
'name': 'my_string'}
]
}
}
}
]
}
table = np.array(
[(42, 32000, 'foo')],
dtype=[
(str('f0'), str('?')),
(str('my_int'), str('=i2')),
(str('my_string'), str('S64'))
])
x = util.gentle_asarray(table, dtype=[
(str('f0'), str('?')),
(str('my_int'), str('=i2')),
(str('my_string'), str('S64'))
])
assert x is table
with DataModel(schema=table_schema) as x:
x.table = table
assert x.table is table
table = np.array(
[(42, 32000, 'foo')],
dtype=[
(str('f0'), str('?')),
(str('my_int'), str('=i2')),
(str('my_string'), str('S3'))
])
with DataModel(schema=table_schema) as x:
x.table = table
assert x.table is not table
assert x.table['my_string'][0] == table['my_string'][0]
def test_data_array(tmp_path):
"""Test lots of things
"""
path = str(tmp_path / "data_array.fits")
data_array_schema = {
"allOf": [
mschema.load_schema(
"http://stsci.edu/schemas/jwst_datamodel/core.schema",
resolve_references=True), {
"type": "object",
"properties": {
"arr": {
'title': 'An array of data',
'type': 'array',
"fits_hdu": ["FOO", "DQ"],
"items": {
"title": "entry",
"type": "object",
"properties": {
"data": {
"fits_hdu": "FOO",
"default": 0.0,
"max_ndim": 2,
"datatype": "float64"
},
"dq": {
"fits_hdu": "DQ",
"default": 1,
"datatype": "uint8"
},
}
}
}
}
}
]
}
array1 = np.random.rand(5, 5)
array2 = np.random.rand(5, 5)
array3 = np.random.rand(5, 5)
with JwstDataModel(schema=data_array_schema) as x:
x.arr.append(x.arr.item())
x.arr[0].data = array1
assert len(x.arr) == 1
x.arr.append(x.arr.item(data=array2))
assert len(x.arr) == 2
x.arr.append({})
assert len(x.arr) == 3
x.arr[2].data = array3
del x.arr[1]
assert len(x.arr) == 2
x.save(path)
with JwstDataModel(path, schema=data_array_schema) as x:
assert len(x.arr) == 2
assert_array_almost_equal(x.arr[0].data, array1)
assert_array_almost_equal(x.arr[1].data, array3)
del x.arr[0]
assert len(x.arr) == 1
x.arr = []
assert len(x.arr) == 0
x.arr.append({'data': np.empty((5, 5))})
assert len(x.arr) == 1
x.arr.extend([
x.arr.item(data=np.empty((5, 5))),
x.arr.item(data=np.empty((5, 5)),
dq=np.empty((5, 5), dtype=np.uint8))
])
assert len(x.arr) == 3
del x.arr[1]
assert len(x.arr) == 2
x.save(path)
from astropy.io import fits
with fits.open(path) as hdulist:
x = set()
for hdu in hdulist:
x.add((hdu.header.get('EXTNAME'), hdu.header.get('EXTVER')))
assert x == set([('FOO', 2), ('FOO', 1), ('ASDF', None), ('DQ', 2),
(None, None)])
def test_data_array():
data_array_schema = {
"allOf": [
mschema.load_schema(
os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True),
{
"type": "object",
"properties": {
"arr": {
'title': 'An array of data',
'type': 'array',
"fits_hdu": ["FOO", "DQ"],
"items": {
"title": "entry",
"type": "object",
"properties": {
"data": {
"fits_hdu": "FOO",
"default": 0.0,
"max_ndim": 2,
"datatype": "float64"
},
"dq": {
"fits_hdu": "DQ",
"default": 1,
"datatype": "uint8"
},
}
}
}
}
}
]
}
array1 = np.random.rand(5, 5)
array2 = np.random.rand(5, 5)
array3 = np.random.rand(5, 5)
with DataModel(schema=data_array_schema) as x:
x.arr.append(x.arr.item())
x.arr[0].data = array1
assert len(x.arr) == 1
x.arr.append(x.arr.item(data=array2))
assert len(x.arr) == 2
x.arr.append({})
assert len(x.arr) == 3
x.arr[2].data = array3
del x.arr[1]
assert len(x.arr) == 2
x.to_fits(TMP_FITS, overwrite=True)
with DataModel(TMP_FITS, schema=data_array_schema) as x:
assert len(x.arr) == 2
assert_array_almost_equal(x.arr[0].data, array1)
assert_array_almost_equal(x.arr[1].data, array3)
del x.arr[0]
assert len(x.arr) == 1
x.arr = []
assert len(x.arr) == 0
x.arr.append({'data': np.empty((5, 5))})
assert len(x.arr) == 1
x.arr.extend([
x.arr.item(data=np.empty((5, 5))),
x.arr.item(data=np.empty((5, 5)),
dq=np.empty((5, 5), dtype=np.uint8))])
assert len(x.arr) == 3
del x.arr[1]
assert len(x.arr) == 2
x.to_fits(TMP_FITS2, overwrite=True)
from astropy.io import fits
with fits.open(TMP_FITS2) as hdulist:
x = set()
for hdu in hdulist:
x.add((hdu.header.get('EXTNAME'),
hdu.header.get('EXTVER')))
print(x)
assert x == set(
[('FOO', 2), ('FOO', 1), ('ASDF', None), ('DQ', 2),
(None, None)])
def test_datamodel_schema_entry_points():
"""Test that entry points for datamodels BaseExtension work with asdf"""
resolver = asdf.AsdfFile().resolver
mschema.load_schema('http://jwst.stsci.edu/schemas/image.schema.yaml',
resolver=resolver,
resolve_references=True)
with pytest.raises(jsonschema.ValidationError):
with ImageModel((50, 50)) as dm:
dm.meta.date = 'Not an acceptable date'
def test_date2():
from astropy import time
with ImageModel((50, 50)) as dm:
assert isinstance(dm.meta.date, (time.Time, datetime.datetime))
'''
transformation_schema = {
"allOf": [
mschema.load_schema(os.path.join(os.path.dirname(__file__),
"../schemas/image.schema.yaml"),
resolve_references=True),
{
"type": "object",
"properties": {
"meta": {
"type": "object",
"properties": {
"transformations": {
"title": "A list of transformations",
"type": "array",
"items": {
"title": "A transformation",
"type": "object",
"properties": {
"type": {
def __init__(self,
init=None,
schema=None,
memmap=False,
pass_invalid_values=None,
strict_validation=None,
ignore_missing_extensions=True,
**kwargs):
"""
Parameters
----------
init : str, tuple, `~astropy.io.fits.HDUList`, ndarray, dict, None
- None : Create a default data model with no shape.
- tuple : Shape of the data array.
Initialize with empty data array with shape specified by the.
- file path: Initialize from the given file (FITS or ASDF)
- readable file object: Initialize from the given file
object
- `~astropy.io.fits.HDUList` : Initialize from the given
`~astropy.io.fits.HDUList`.
- A numpy array: Used to initialize the data array
- dict: The object model tree for the data model
schema : dict, str (optional)
Tree of objects representing a JSON schema, or string naming a schema.
The schema to use to understand the elements on the model.
If not provided, the schema associated with this class
will be used.
memmap : bool
Turn memmap of FITS file on or off. (default: False). Ignored for
ASDF files.
pass_invalid_values : bool or None
If `True`, values that do not validate the schema
will be added to the metadata. If `False`, they will be set to `None`.
If `None`, value will be taken from the environmental PASS_INVALID_VALUES.
Otherwise the default value is `False`.
strict_validation : bool or None
If `True`, schema validation errors will generate
an exception. If `False`, they will generate a warning.
If `None`, value will be taken from the environmental STRICT_VALIDATION.
Otherwise, the default value is `False`.
ignore_missing_extensions : bool
When `False`, raise warnings when a file is read that
contains metadata about extensions that are not available.
Defaults to `True`.
kwargs : dict
Additional arguments passed to lower level functions.
"""
# Override value of validation parameters if not explicitly set.
if not pass_invalid_values:
pass_invalid_values = self._get_envar_as_boolean(
"PASS_INVALID_VALUES", False)
self._pass_invalid_values = pass_invalid_values
if not strict_validation:
strict_validation = self._get_envar_as_boolean(
"STRICT_VALIDATION", False)
self._strict_validation = strict_validation
self._ignore_missing_extensions = ignore_missing_extensions
kwargs.update({'ignore_missing_extensions': ignore_missing_extensions})
# Load the schema files
if schema is None:
# Create an AsdfFile so we can use its resolver for loading schemas
asdf_file = AsdfFile()
schema = asdf_schema.load_schema(self.schema_url,
resolver=asdf_file.resolver,
resolve_references=True)
self._schema = mschema.merge_property_trees(schema)
# Provide the object as context to other classes and functions
self._ctx = self
# Determine what kind of input we have (init) and execute the
# proper code to intiailize the model
self._files_to_close = []
self._iscopy = False
is_array = False
is_shape = False
shape = None
if init is None:
asdffile = self.open_asdf(init=None, **kwargs)
elif isinstance(init, dict):
asdffile = self.open_asdf(init=init, **kwargs)
elif isinstance(init, np.ndarray):
asdffile = self.open_asdf(init=None, **kwargs)
shape = init.shape
is_array = True
elif isinstance(init, tuple):
for item in init:
if not isinstance(item, int):
raise ValueError("shape must be a tuple of ints")
shape = init
is_shape = True
asdffile = self.open_asdf(init=None, **kwargs)
elif isinstance(init, DataModel):
asdffile = None
self.clone(self, init)
if not isinstance(init, self.__class__):
self.validate()
return
elif isinstance(init, AsdfFile):
asdffile = init
elif isinstance(init, fits.HDUList):
asdffile = fits_support.from_fits(init, self._schema, self._ctx,
**kwargs)
elif isinstance(init, (str, bytes)):
if isinstance(init, bytes):
init = init.decode(sys.getfilesystemencoding())
file_type = filetype.check(init)
if file_type == "fits":
if s3_utils.is_s3_uri(init):
init_fitsopen = s3_utils.get_object(init)
memmap = None
else:
init_fitsopen = init
with fits.open(init_fitsopen, memmap=memmap) as hdulist:
asdffile = fits_support.from_fits(hdulist, self._schema,
self._ctx, **kwargs)
self._files_to_close.append(hdulist)
elif file_type == "asdf":
asdffile = self.open_asdf(init=init, **kwargs)
else:
# TODO handle json files as well
raise IOError(
"File does not appear to be a FITS or ASDF file.")
else:
raise ValueError("Can't initialize datamodel using {0}".format(
str(type(init))))
# Initialize object fields as determined from the code above
self._shape = shape
self._instance = asdffile.tree
self._asdf = asdffile
# Initalize class dependent hidden fields
self._no_asdf_extension = False
# Instantiate the primary array of the image
if is_array:
primary_array_name = self.get_primary_array_name()
if not primary_array_name:
raise TypeError(
"Array passed to DataModel.__init__, but model has "
"no primary array in its schema")
setattr(self, primary_array_name, init)
# If a shape has been given, initialize the primary array.
if is_shape:
primary_array_name = self.get_primary_array_name()
if not primary_array_name:
raise TypeError(
"Shape passed to DataModel.__init__, but model has "
"no primary array in its schema")
# Initialization occurs when the primary array is first
# referenced. Do so now.
getattr(self, primary_array_name)
# if the input is from a file, set the filename attribute
if isinstance(init, str):
self.meta.filename = os.path.basename(init)
elif isinstance(init, fits.HDUList):
info = init.fileinfo(0)
if info is not None:
filename = info.get('filename')
if filename is not None:
self.meta.filename = os.path.basename(filename)
# if the input model doesn't have a date set, use the current date/time
if not self.meta.hasattr('date'):
current_date = Time(datetime.datetime.now())
current_date.format = 'isot'
self.meta.date = current_date.value
# store the data model type, if not already set
klass = self.__class__.__name__
if klass != 'DataModel':
if not self.meta.hasattr('model_type'):
self.meta.model_type = klass
# initialize arrays from keyword arguments when they are present
for attr, value in kwargs.items():
if value is not None:
subschema = properties._get_schema_for_property(
self._schema, attr)
if 'datatype' in subschema:
setattr(self, attr, value)
def runtest(self):
# Make sure that each schema itself is valid.
schema_tree = schema.load_schema(self.schema_path,
resolver=_resolver,
resolve_references=True)
schema.check_schema(schema_tree)
def test_table_with_metadata():
schema = {
"allOf": [
mschema.load_schema(
os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True),
{"type": "object",
"properties": {
"flux_table": {
"title": "Photometric flux conversion table",
"fits_hdu": "FLUX",
"datatype":
[
{"name": "parameter", "datatype": ['ascii', 7]},
{"name": "factor", "datatype": "float64"},
{"name": "uncertainty", "datatype": "float64"}
]
},
"meta": {
"type": "object",
"properties": {
"fluxinfo": {
"title": "Information about the flux conversion",
"type": "object",
"properties": {
"exposure": {
"title": "Description of exposure analyzed",
"type": "string",
"fits_hdu": "FLUX",
"fits_keyword": "FLUXEXP"
}
}
}
}
}
}
}
]
}
class FluxModel(DataModel):
def __init__(self, init=None, flux_table=None, **kwargs):
super(FluxModel, self).__init__(init=init, schema=schema, **kwargs)
if flux_table is not None:
self.flux_table = flux_table
flux_im = [
('F560W', 1.0e-5, 1.0e-7),
('F770W', 1.1e-5, 1.6e-7),
]
with FluxModel(flux_table=flux_im) as datamodel:
datamodel.meta.fluxinfo.exposure = 'Exposure info'
datamodel.save(TMP_FITS, clobber=True)
del datamodel
from astropy.io import fits
hdulist = fits.open(TMP_FITS)
assert len(hdulist) == 3
assert isinstance(hdulist[1], fits.BinTableHDU)
assert hdulist[1].name == 'FLUX'
assert isinstance(hdulist[2], fits.ImageHDU)
assert hdulist[2].name == 'ASDF'
def test_datamodel_schema_entry_points():
"""Test that entry points for DataModelExtension works as expected"""
resolver = asdf.AsdfFile().resolver
mschema.load_schema('http://stsci.edu/schemas/jwst_datamodel/image.schema',
resolver=resolver,
resolve_references=True)
def test_replace_table():
from astropy.io import fits
schema_narrow = {
"allOf": [
mschema.load_schema(
os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True),
{
"type": "object",
"properties": {
"data": {
"title": "relative sensitivity table",
"fits_hdu": "RELSENS",
"datatype": [
{"name": "TYPE", "datatype": ["ascii", 16]},
{"name": "T_OFFSET", "datatype": "float32"},
{"name": "DECAY_PEAK", "datatype": "float32"},
{"name": "DECAY_FREQ", "datatype": "float32"},
{"name": "TAU", "datatype": "float32"}
]
}
}
}
]
}
schema_wide = {
"allOf": [
mschema.load_schema(
os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True),
{
"type": "object",
"properties": {
"data": {
"title": "relative sensitivity table",
"fits_hdu": "RELSENS",
"datatype": [
{"name": "TYPE", "datatype": ["ascii", 16]},
{"name": "T_OFFSET", "datatype": "float64"},
{"name": "DECAY_PEAK", "datatype": "float64"},
{"name": "DECAY_FREQ", "datatype": "float64"},
{"name": "TAU", "datatype": "float64"}
]
}
}
}
]
}
x = np.array([("string", 1., 2., 3., 4.)],
dtype=[(str('TYPE'), str('S16')),
(str('T_OFFSET'), np.float32),
(str('DECAY_PEAK'), np.float32),
(str('DECAY_FREQ'), np.float32),
(str('TAU'), np.float32)])
m = DataModel(schema=schema_narrow)
m.data = x
m.to_fits(TMP_FITS, clobber=True)
with fits.open(TMP_FITS) as hdulist:
assert repr(list(x)) == repr(list(np.asarray(hdulist[1].data)))
assert hdulist[1].data.dtype[1].str == '>f4'
assert hdulist[1].header['TFORM2'] == 'E'
with DataModel(TMP_FITS,
schema=schema_wide) as m:
foo = m.data
m.to_fits(TMP_FITS2, clobber=True)
with fits.open(TMP_FITS2) as hdulist:
assert repr(list(x)) == repr(list(np.asarray(hdulist[1].data)))
assert hdulist[1].data.dtype[1].str == '>f8'
assert hdulist[1].header['TFORM2'] == 'D'
def test_replace_table():
from astropy.io import fits
schema_narrow = {
"allOf": [
mschema.load_schema(os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True), {
"type": "object",
"properties": {
"data": {
"title":
"relative sensitivity table",
"fits_hdu":
"RELSENS",
"datatype": [{
"name":
"TYPE",
"datatype": ["ascii", 16]
}, {
"name": "T_OFFSET",
"datatype": "float32"
}, {
"name": "DECAY_PEAK",
"datatype": "float32"
}, {
"name": "DECAY_FREQ",
"datatype": "float32"
}, {
"name": "TAU",
"datatype": "float32"
}]
}
}
}
]
}
schema_wide = {
"allOf": [
mschema.load_schema(os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True), {
"type": "object",
"properties": {
"data": {
"title":
"relative sensitivity table",
"fits_hdu":
"RELSENS",
"datatype": [{
"name":
"TYPE",
"datatype": ["ascii", 16]
}, {
"name": "T_OFFSET",
"datatype": "float64"
}, {
"name": "DECAY_PEAK",
"datatype": "float64"
}, {
"name": "DECAY_FREQ",
"datatype": "float64"
}, {
"name": "TAU",
"datatype": "float64"
}]
}
}
}
]
}
x = np.array([("string", 1., 2., 3., 4.)],
dtype=[('TYPE', 'S16'), ('T_OFFSET', np.float32),
('DECAY_PEAK', np.float32), ('DECAY_FREQ', np.float32),
('TAU', np.float32)])
m = DataModel(schema=schema_narrow)
m.data = x
m.to_fits(TMP_FITS, overwrite=True)
with fits.open(TMP_FITS, memmap=False) as hdulist:
assert records_equal(x, np.asarray(hdulist[1].data))
assert hdulist[1].data.dtype[1].str == '>f4'
assert hdulist[1].header['TFORM2'] == 'E'
with DataModel(TMP_FITS, schema=schema_wide) as m:
m.to_fits(TMP_FITS2, overwrite=True)
with fits.open(TMP_FITS2, memmap=False) as hdulist:
assert records_equal(x, np.asarray(hdulist[1].data))
assert hdulist[1].data.dtype[1].str == '>f8'
assert hdulist[1].header['TFORM2'] == 'D'
def __init__(self,
init=None,
schema=None,
memmap=False,
pass_invalid_values=None,
strict_validation=None,
validate_on_assignment=None,
ignore_missing_extensions=True,
**kwargs):
"""
Parameters
----------
init : str, tuple, `~astropy.io.fits.HDUList`, ndarray, dict, None
- None : Create a default data model with no shape.
- tuple : Shape of the data array.
Initialize with empty data array with shape specified by the.
- file path: Initialize from the given file (FITS or ASDF)
- readable file object: Initialize from the given file
object
- `~astropy.io.fits.HDUList` : Initialize from the given
`~astropy.io.fits.HDUList`.
- A numpy array: Used to initialize the data array
- dict: The object model tree for the data model
schema : dict, str (optional)
Tree of objects representing a JSON schema, or string naming a schema.
The schema to use to understand the elements on the model.
If not provided, the schema associated with this class
will be used.
memmap : bool
Turn memmap of FITS file on or off. (default: False). Ignored for
ASDF files.
pass_invalid_values : bool or None
If `True`, values that do not validate the schema
will be added to the metadata. If `False`, they will be set to `None`.
If `None`, value will be taken from the environmental PASS_INVALID_VALUES.
Otherwise the default value is `False`.
strict_validation : bool or None
If `True`, schema validation errors will generate
an exception. If `False`, they will generate a warning.
If `None`, value will be taken from the environmental STRICT_VALIDATION.
Otherwise, the default value is `False`.
validate_on_assignment : bool or None
Defaults to 'None'.
If `None`, value will be taken from the environmental VALIDATE_ON_ASSIGNMENT,
defaulting to 'True' if no environment variable is set.
If 'True', attribute assignments are validated at the time of assignment.
Validation errors generate warnings and values will be set to `None`.
If 'False', schema validation occurs only once at the time of write.
Validation errors generate warnings.
ignore_missing_extensions : bool
When `False`, raise warnings when a file is read that
contains metadata about extensions that are not available.
Defaults to `True`.
kwargs : dict
Additional keyword arguments passed to lower level functions. These arguments
are generally file format-specific. Arguments of note are:
- FITS
skip_fits_update - bool or None
`True` to skip updating the ASDF tree from the FITS headers, if possible.
If `None`, value will be taken from the environmental SKIP_FITS_UPDATE.
Otherwise, the default value is `True`.
"""
# Override value of validation parameters if not explicitly set.
if pass_invalid_values is None:
pass_invalid_values = get_envar_as_boolean("PASS_INVALID_VALUES",
False)
self._pass_invalid_values = pass_invalid_values
if strict_validation is None:
strict_validation = get_envar_as_boolean("STRICT_VALIDATION",
False)
if validate_on_assignment is None:
validate_on_assignment = get_envar_as_boolean(
"VALIDATE_ON_ASSIGNMENT", True)
self._strict_validation = strict_validation
self._ignore_missing_extensions = ignore_missing_extensions
self._validate_on_assignment = validate_on_assignment
kwargs.update({'ignore_missing_extensions': ignore_missing_extensions})
# Load the schema files
if schema is None:
if self.schema_url is None:
schema = _DEFAULT_SCHEMA
else:
# Create an AsdfFile so we can use its resolver for loading schemas
schema = asdf_schema.load_schema(self.schema_url,
resolve_references=True)
self._schema = mschema.merge_property_trees(schema)
# Provide the object as context to other classes and functions
self._ctx = self
# Initialize with an empty AsdfFile instance as this is needed for
# reading in FITS files where validate._check_value() gets called, and
# ctx needs to have an _asdf attribute.
self._asdf = AsdfFile()
# Determine what kind of input we have (init) and execute the
# proper code to intiailize the model
self._files_to_close = []
self._iscopy = False
is_array = False
is_shape = False
shape = None
if init is None:
asdffile = self.open_asdf(init=None, **kwargs)
elif isinstance(init, dict):
asdffile = self.open_asdf(init=init, **kwargs)
elif isinstance(init, np.ndarray):
asdffile = self.open_asdf(init=None, **kwargs)
shape = init.shape
is_array = True
elif isinstance(init, tuple):
for item in init:
if not isinstance(item, int):
raise ValueError("shape must be a tuple of ints")
shape = init
is_shape = True
asdffile = self.open_asdf(init=None, **kwargs)
elif isinstance(init, DataModel):
asdffile = None
self.clone(self, init)
if not isinstance(init, self.__class__):
self.validate()
return
elif isinstance(init, AsdfFile):
asdffile = init
elif isinstance(init, fits.HDUList):
asdffile = fits_support.from_fits(init, self._schema, self._ctx,
**kwargs)
elif isinstance(init, (str, bytes, PurePath)):
if isinstance(init, PurePath):
init = str(init)
if isinstance(init, bytes):
init = init.decode(sys.getfilesystemencoding())
file_type = filetype.check(init)
if file_type == "fits":
if s3_utils.is_s3_uri(init):
init_fitsopen = s3_utils.get_object(init)
memmap = None
else:
init_fitsopen = init
hdulist = fits.open(init_fitsopen, memmap=memmap)
asdffile = fits_support.from_fits(hdulist, self._schema,
self._ctx, **kwargs)
self._files_to_close.append(hdulist)
elif file_type == "asdf":
asdffile = self.open_asdf(init=init, **kwargs)
else:
# TODO handle json files as well
raise IOError(
"File does not appear to be a FITS or ASDF file.")
else:
raise ValueError("Can't initialize datamodel using {0}".format(
str(type(init))))
# Initialize object fields as determined from the code above
self._shape = shape
self._instance = asdffile.tree
self._asdf = asdffile
# Initalize class dependent hidden fields
self._no_asdf_extension = False
# Instantiate the primary array of the image
if is_array:
primary_array_name = self.get_primary_array_name()
if not primary_array_name:
raise TypeError(
"Array passed to DataModel.__init__, but model has "
"no primary array in its schema")
setattr(self, primary_array_name, init)
# If a shape has been given, initialize the primary array.
if is_shape:
primary_array_name = self.get_primary_array_name()
if not primary_array_name:
raise TypeError(
"Shape passed to DataModel.__init__, but model has "
"no primary array in its schema")
# Initialization occurs when the primary array is first
# referenced. Do so now.
getattr(self, primary_array_name)
# initialize arrays from keyword arguments when they are present
for attr, value in kwargs.items():
if value is not None:
subschema = properties._get_schema_for_property(
self._schema, attr)
if 'datatype' in subschema:
setattr(self, attr, value)
# Call hook that sets model properties
self.on_init(init)
def __init__(self,
init=None,
schema=None,
extensions=None,
pass_invalid_values=False):
"""
Parameters
----------
init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None
- None: A default data model with no shape
- shape tuple: Initialize with empty data of the given
shape
- file path: Initialize from the given file (FITS or ASDF)
- readable file object: Initialize from the given file
object
- ``astropy.io.fits.HDUList``: Initialize from the given
`~astropy.io.fits.HDUList`.
- A numpy array: Used to initialize the data array
- dict: The object model tree for the data model
schema : tree of objects representing a JSON schema, or string naming a schema, optional
The schema to use to understand the elements on the model.
If not provided, the schema associated with this class
will be used.
extensions: classes extending the standard set of extensions, optional.
If an extension is defined, the prefix used should be 'url'.
pass_invalid_values: If true, values that do not validate the schema can
be read and written and only a warning will be generated
"""
# Set the extensions
if extensions is None:
extensions = jwst_extensions[:]
else:
extensions.extend(jwst_extensions)
self._extensions = extensions
# Override value of pass_invalid value if environment value set
if "PASS_INVALID_VALUES" in os.environ:
pass_invalid_values = os.environ["PASS_INVALID_VALUES"]
try:
pass_invalid_values = bool(int(pass_invalid_values))
except ValueError:
pass_invalid_values = False
self._pass_invalid_values = pass_invalid_values
# Construct the path to the schema files
filename = os.path.abspath(inspect.getfile(self.__class__))
base_url = os.path.join(os.path.dirname(filename), 'schemas', '')
# Load the schema files
if schema is None:
schema_path = os.path.join(base_url, self.schema_url)
extension_list = asdf_extension.AsdfExtensionList(self._extensions)
schema = asdf_schema.load_schema(
schema_path,
resolver=extension_list.url_mapping,
resolve_references=True)
self._schema = mschema.flatten_combiners(schema)
# Determine what kind of input we have (init) and execute the
# proper code to intiailize the model
self._files_to_close = []
self._iscopy = False
is_array = False
is_shape = False
shape = None
if init is None:
asdf = AsdfFile(extensions=extensions)
elif isinstance(init, dict):
asdf = AsdfFile(init, extensions=extensions)
elif isinstance(init, np.ndarray):
asdf = AsdfFile(extensions=extensions)
shape = init.shape
is_array = True
elif isinstance(init, self.__class__):
self.clone(self, init)
return
elif isinstance(init, DataModel):
raise TypeError(
"Passed in {0!r} is not of the expected subclass {1!r}".format(
init.__class__.__name__, self.__class__.__name__))
elif isinstance(init, AsdfFile):
asdf = init
elif isinstance(init, tuple):
for item in init:
if not isinstance(item, int):
raise ValueError("shape must be a tuple of ints")
shape = init
asdf = AsdfFile()
is_shape = True
elif isinstance(init, fits.HDUList):
asdf = fits_support.from_fits(init, self._schema, extensions,
pass_invalid_values)
elif isinstance(init, (six.string_types, bytes)):
if isinstance(init, bytes):
init = init.decode(sys.getfilesystemencoding())
file_type = filetype.check(init)
if file_type == "fits":
hdulist = fits.open(init)
asdf = fits_support.from_fits(hdulist, self._schema,
extensions, pass_invalid_values)
self._files_to_close.append(hdulist)
elif file_type == "asdf":
asdf = AsdfFile.open(init, extensions=extensions)
else:
# TODO handle json files as well
raise IOError(
"File does not appear to be a FITS or ASDF file.")
else:
raise ValueError("Can't initialize datamodel using {0}".format(
str(type(init))))
# Initialize object fields as determined fro the code above
self._shape = shape
self._instance = asdf.tree
self._asdf = asdf
self._ctx = self
# if the input is from a file, set the filename attribute
if isinstance(init, six.string_types):
self.meta.filename = os.path.basename(init)
elif isinstance(init, fits.HDUList):
info = init.fileinfo(0)
if info is not None:
filename = info.get('filename')
if filename is not None:
self.meta.filename = os.path.basename(filename)
# if the input model doesn't have a date set, use the current date/time
if self.meta.date is None:
self.meta.date = Time(datetime.datetime.now())
if hasattr(self.meta.date, 'value'):
self.meta.date.format = 'isot'
self.meta.date = str(self.meta.date.value)
# store the data model type, if not already set
if hasattr(self.meta, 'model_type'):
if self.meta.model_type is None:
self.meta.model_type = self.__class__.__name__
else:
self.meta.model_type = None
if is_array:
primary_array_name = self.get_primary_array_name()
if primary_array_name is None:
raise TypeError(
"Array passed to DataModel.__init__, but model has "
"no primary array in its schema")
setattr(self, primary_array_name, init)
# TODO this code looks useless
if is_shape:
getattr(self, self.get_primary_array_name())
def test_table_array_convert():
"""
Test that structured arrays are converted when necessary, and
reused as views when not.
"""
from jwst.datamodels import util
table_schema = {
"allOf": [
mschema.load_schema(
os.path.join(os.path.dirname(__file__),
"../schemas/image.schema.yaml"),
resolve_references=True),
{
"type": "object",
"properties": {
"table": {
'title': 'A structured table',
'fits_hdu': 'table',
'datatype': [
'bool8',
{'datatype': 'int16',
'name': 'my_int'},
{'datatype': ['ascii', 64],
'name': 'my_string'}
]
}
}
}
]
}
table = np.array(
[(42, 32000, 'foo')],
dtype=[
(str('f0'), str('?')),
(str('my_int'), str('=i2')),
(str('my_string'), str('S64'))
])
x = util.gentle_asarray(table, dtype=[
(str('f0'), str('?')),
(str('my_int'), str('=i2')),
(str('my_string'), str('S64'))
])
assert x is table
with DataModel(schema=table_schema) as x:
x.table = table
assert x.table is not table
table = np.array(
[(42, 32000, 'foo')],
dtype=[
(str('f0'), str('?')),
(str('my_int'), str('=i2')),
(str('my_string'), str('S3'))
])
with DataModel(schema=table_schema) as x:
x.table = table
assert x.table is not table
assert x.table['my_string'][0] != table['my_string'][0]
def test_data_array():
data_array_schema = {
"allOf": [
mschema.load_schema(os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True), {
"type": "object",
"properties": {
"arr": {
'title': 'An array of data',
'type': 'array',
"fits_hdu": ["FOO", "DQ"],
"items": {
"title": "entry",
"type": "object",
"properties": {
"data": {
"fits_hdu": "FOO",
"default": 0.0,
"max_ndim": 2,
"datatype": "float64"
},
"dq": {
"fits_hdu": "DQ",
"default": 1,
"datatype": "uint8"
},
}
}
}
}
}
]
}
array1 = np.random.rand(5, 5)
array2 = np.random.rand(5, 5)
array3 = np.random.rand(5, 5)
with DataModel(schema=data_array_schema) as x:
x.arr.append(x.arr.item())
x.arr[0].data = array1
assert len(x.arr) == 1
x.arr.append(x.arr.item(data=array2))
assert len(x.arr) == 2
x.arr.append({})
assert len(x.arr) == 3
x.arr[2].data = array3
del x.arr[1]
assert len(x.arr) == 2
x.to_fits(TMP_FITS, clobber=True)
with DataModel(TMP_FITS, schema=data_array_schema) as x:
assert len(x.arr) == 2
assert_array_almost_equal(x.arr[0].data, array1)
assert_array_almost_equal(x.arr[1].data, array3)
del x.arr[0]
assert len(x.arr) == 1
x.arr = []
assert len(x.arr) == 0
x.arr.append({'data': np.empty((5, 5))})
assert len(x.arr) == 1
x.arr.extend([
x.arr.item(data=np.empty((5, 5))),
x.arr.item(data=np.empty((5, 5)),
dq=np.empty((5, 5), dtype=np.uint8))
])
assert len(x.arr) == 3
del x.arr[1]
assert len(x.arr) == 2
x.to_fits(TMP_FITS2, clobber=True)
from astropy.io import fits
with fits.open(TMP_FITS2) as hdulist:
x = set()
for hdu in hdulist:
x.add((hdu.header.get('EXTNAME'), hdu.header.get('EXTVER')))
print(x)
assert x == set([('FOO', 2), ('FOO', 1), ('ASDF', None), ('DQ', 2),
(None, None)])
def __init__(self, init=None, schema=None,
pass_invalid_values=False, strict_validation=False,
**kwargs):
"""
Parameters
----------
init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, None
- None: A default data model with no shape
- shape tuple: Initialize with empty data of the given
shape
- file path: Initialize from the given file (FITS or ASDF)
- readable file object: Initialize from the given file
object
- ``astropy.io.fits.HDUList``: Initialize from the given
`~astropy.io.fits.HDUList`.
- A numpy array: Used to initialize the data array
- dict: The object model tree for the data model
schema : tree of objects representing a JSON schema, or string naming a schema, optional
The schema to use to understand the elements on the model.
If not provided, the schema associated with this class
will be used.
pass_invalid_values: If true, values that do not validate the schema
will be added to the metadata. If false, they will be set to None
strict_validation: if true, an schema validation errors will generate
an excption. If false, they will generate a warning.
kwargs: Aadditional arguments passed to lower level functions
"""
# Override value of validation parameters
# if environment value set
self._pass_invalid_values = self.get_envar("PASS_INVALID_VALUES",
pass_invalid_values)
self._strict_validation = self.get_envar("STRICT_VALIDATION",
strict_validation)
# Load the schema files
if schema is None:
schema_path = os.path.join(URL_PREFIX, self.schema_url)
# Create an AsdfFile so we can use its resolver for loading schemas
asdf_file = AsdfFile()
schema = asdf_schema.load_schema(schema_path,
resolver=asdf_file.resolver,
resolve_references=True)
self._schema = mschema.merge_property_trees(schema)
# Provide the object as context to other classes and functions
self._ctx = self
# Determine what kind of input we have (init) and execute the
# proper code to intiailize the model
self._files_to_close = []
self._iscopy = False
is_array = False
is_shape = False
shape = None
if init is None:
asdffile = self.open_asdf(init=None, **kwargs)
elif isinstance(init, dict):
asdffile = self.open_asdf(init=init, **kwargs)
elif isinstance(init, np.ndarray):
asdffile = self.open_asdf(init=None, **kwargs)
shape = init.shape
is_array = True
elif isinstance(init, tuple):
for item in init:
if not isinstance(item, int):
raise ValueError("shape must be a tuple of ints")
shape = init
is_shape = True
asdffile = self.open_asdf(init=None, **kwargs)
elif isinstance(init, DataModel):
asdffile = None
self.clone(self, init)
if not isinstance(init, self.__class__):
self.validate()
return
elif isinstance(init, AsdfFile):
asdffile = init
elif isinstance(init, fits.HDUList):
asdffile = fits_support.from_fits(init, self._schema, self._ctx)
elif isinstance(init, (str, bytes)):
if isinstance(init, bytes):
init = init.decode(sys.getfilesystemencoding())
file_type = filetype.check(init)
if file_type == "fits":
hdulist = fits.open(init)
asdffile = fits_support.from_fits(hdulist,
self._schema,
self._ctx,
**kwargs)
self._files_to_close.append(hdulist)
elif file_type == "asdf":
asdffile = self.open_asdf(init=init, **kwargs)
else:
# TODO handle json files as well
raise IOError(
"File does not appear to be a FITS or ASDF file.")
else:
raise ValueError(
"Can't initialize datamodel using {0}".format(str(type(init))))
# Initialize object fields as determined from the code above
self._shape = shape
self._instance = asdffile.tree
self._asdf = asdffile
# Initalize class dependent hidden fields
self._no_asdf_extension = False
# Instantiate the primary array of the image
if is_array:
primary_array_name = self.get_primary_array_name()
if not primary_array_name:
raise TypeError(
"Array passed to DataModel.__init__, but model has "
"no primary array in its schema")
setattr(self, primary_array_name, init)
if is_shape:
if not self.get_primary_array_name():
raise TypeError(
"Shape passed to DataModel.__init__, but model has "
"no primary array in its schema")
# if the input is from a file, set the filename attribute
if isinstance(init, str):
self.meta.filename = os.path.basename(init)
elif isinstance(init, fits.HDUList):
info = init.fileinfo(0)
if info is not None:
filename = info.get('filename')
if filename is not None:
self.meta.filename = os.path.basename(filename)
# if the input model doesn't have a date set, use the current date/time
if not self.meta.hasattr('date'):
current_date = Time(datetime.datetime.now())
current_date.format = 'isot'
self.meta.date = current_date.value
# store the data model type, if not already set
klass = self.__class__.__name__
if klass != 'DataModel':
if not self.meta.hasattr('model_type'):
self.meta.model_type = klass
# initialize arrays from keyword arguments when they are present
for attr, value in kwargs.items():
if value is not None:
subschema = properties._get_schema_for_property(self._schema,
attr)
if 'datatype' in subschema:
setattr(self, attr, value)
