def test_external_reference_invalid_fragment(tmpdir):
exttree = {
'list_of_stuff': [
'foobar',
42,
np.array([7, 8, 9], np.float)
]
}
external_path = os.path.join(str(tmpdir), 'external.asdf')
ff = asdf.AsdfFile(exttree)
ff.write_to(external_path)
tree = {
'foo': {
'$ref': 'external.asdf#/list_of_stuff/a'
}
}
with asdf.AsdfFile(tree, uri=util.filepath_to_url(
os.path.join(str(tmpdir), 'main.asdf'))) as ff:
with pytest.raises(ValueError):
ff.resolve_references()
tree = {
'foo': {
'$ref': 'external.asdf#/list_of_stuff/3'
}
}
with asdf.AsdfFile(tree, uri=util.filepath_to_url(
os.path.join(str(tmpdir), 'main.asdf'))) as ff:
with pytest.raises(ValueError):
ff.resolve_references()
def url_mapping(self):
return [
(URL_PREFIX,
util.filepath_to_url(SCHEMA_PATH) + '/{url_suffix}.yaml'),
('http://stsci.edu/schemas/fits-schema/',
util.filepath_to_url(METASCHEMA_PATH) + '/{url_suffix}.yaml'),
]
def test_external_reference_invalid_fragment(tmpdir):
exttree = {
'list_of_stuff': [
'foobar',
42,
np.array([7, 8, 9], np.float)
]
}
external_path = os.path.join(str(tmpdir), 'external.asdf')
ff = asdf.AsdfFile(exttree)
ff.write_to(external_path)
tree = {
'foo': {
'$ref': 'external.asdf#/list_of_stuff/a'
}
}
with asdf.AsdfFile(tree, uri=util.filepath_to_url(
os.path.join(str(tmpdir), 'main.asdf'))) as ff:
with pytest.raises(ValueError):
ff.resolve_references()
tree = {
'foo': {
'$ref': 'external.asdf#/list_of_stuff/3'
}
}
with asdf.AsdfFile(tree, uri=util.filepath_to_url(
os.path.join(str(tmpdir), 'main.asdf'))) as ff:
with pytest.raises(ValueError):
ff.resolve_references()
def test_schema_example(filename, example):
"""Pytest to check validity of a specific example within schema file
Parameters:
-----------
filename : name of the schema file containing example to be tested
example: string representing example
This function is called with a range of parameters by pytest's
'parametrize' utility in order to account for all examples in all schema
files.
"""
if not HAS_GWCS and re.search(r'frame-\d\.\d\.\d\.yaml', filename):
return pytest.skip
standard_version = _find_standard_version(filename)
# Make sure that the examples in the schema files (and thus the
# ASDF standard document) are valid.
buff = helpers.yaml_to_asdf('example: ' + example.strip(),
standard_version=standard_version)
ff = asdf.AsdfFile(uri=util.filepath_to_url(os.path.abspath(filename)),
extensions=TestExtension())
# Fake an external file
ff2 = asdf.AsdfFile({'data': np.empty((1024 * 1024 * 8), dtype=np.uint8)})
ff._external_asdf_by_uri[util.filepath_to_url(
os.path.abspath(
os.path.join(os.path.dirname(filename), 'external.asdf')))] = ff2
# Add some dummy blocks so that the ndarray examples work
for i in range(3):
b = block.Block(np.zeros((1024 * 1024 * 8), dtype=np.uint8))
b._used = True
ff.blocks.add(b)
b._array_storage = "streamed"
try:
with catch_warnings() as w:
ff._open_impl(ff, buff)
# Do not tolerate any warnings that occur during schema validation,
# other than a few that we expect to occur under certain circumstances
_assert_warnings(w)
except:
print("From file:", filename)
raise
# Just test we can write it out. A roundtrip test
# wouldn't always yield the correct result, so those have
# to be covered by "real" unit tests.
if b'external.asdf' not in buff.getvalue():
buff = io.BytesIO()
ff.write_to(buff)
def runtest(self):
from asdf import AsdfFile, block, util
from asdf.tests import helpers
name, version = parse_schema_filename(self.filename)
if should_skip(name, version):
return
standard_version = self._find_standard_version(name, version)
# Make sure that the examples in the schema files (and thus the
# ASDF standard document) are valid.
buff = helpers.yaml_to_asdf(
'example: ' + self.example.strip(), standard_version=standard_version)
ff = AsdfFile(
uri=util.filepath_to_url(os.path.abspath(self.filename)),
ignore_unrecognized_tag=self.ignore_unrecognized_tag,
ignore_version_mismatch=self.ignore_version_mismatch,
)
# Fake an external file
ff2 = AsdfFile({'data': np.empty((1024*1024*8), dtype=np.uint8)})
ff._external_asdf_by_uri[
util.filepath_to_url(
os.path.abspath(
os.path.join(
os.path.dirname(self.filename), 'external.asdf')))] = ff2
# Add some dummy blocks so that the ndarray examples work
for i in range(3):
b = block.Block(np.zeros((1024*1024*8), dtype=np.uint8))
b._used = True
ff.blocks.add(b)
b._array_storage = "streamed"
try:
with pytest.warns(None) as w:
ff._open_impl(ff, buff, mode='rw')
# Do not tolerate any warnings that occur during schema validation
assert len(w) == 0, helpers.display_warnings(w)
except Exception:
print("From file:", self.filename)
raise
# Just test we can write it out. A roundtrip test
# wouldn't always yield the correct result, so those have
# to be covered by "real" unit tests.
if b'external.asdf' not in buff.getvalue():
buff = io.BytesIO()
ff.write_to(buff)
def runtest(self):
from asdf import AsdfFile, block, util
from asdf.tests import helpers
from .extension import TestExtension
name, version = parse_schema_filename(self.filename)
if should_skip(name, version):
return
standard_version = self._find_standard_version(name, version)
# Make sure that the examples in the schema files (and thus the
# ASDF standard document) are valid.
buff = helpers.yaml_to_asdf(
'example: ' + self.example.strip(), standard_version=standard_version)
ff = AsdfFile(
uri=util.filepath_to_url(os.path.abspath(self.filename)),
extensions=TestExtension())
# Fake an external file
ff2 = AsdfFile({'data': np.empty((1024*1024*8), dtype=np.uint8)})
ff._external_asdf_by_uri[
util.filepath_to_url(
os.path.abspath(
os.path.join(
os.path.dirname(self.filename), 'external.asdf')))] = ff2
# Add some dummy blocks so that the ndarray examples work
for i in range(3):
b = block.Block(np.zeros((1024*1024*8), dtype=np.uint8))
b._used = True
ff.blocks.add(b)
b._array_storage = "streamed"
try:
with pytest.warns(None) as w:
import warnings
ff._open_impl(ff, buff, mode='rw')
# Do not tolerate any warnings that occur during schema validation
assert len(w) == 0, helpers.display_warnings(w)
except Exception:
print("From file:", self.filename)
raise
# Just test we can write it out. A roundtrip test
# wouldn't always yield the correct result, so those have
# to be covered by "real" unit tests.
if b'external.asdf' not in buff.getvalue():
buff = io.BytesIO()
ff.write_to(buff)
def test_internal_reference(tmpdir):
testfile = os.path.join(str(tmpdir), 'test.asdf')
tree = {
'foo': 2,
'bar': {'$ref': '#'}
}
ff = asdf.AsdfFile(tree)
ff.find_references()
assert isinstance(ff.tree['bar'], reference.Reference)
ff.resolve_references()
assert ff.tree['bar']['foo'] == 2
tree = {
'foo': 2
}
ff = asdf.AsdfFile(
tree, uri=util.filepath_to_url(os.path.abspath(testfile)))
ff.tree['bar'] = ff.make_reference([])
buff = io.BytesIO()
ff.write_to(buff)
buff.seek(0)
ff = asdf.AsdfFile()
content = asdf.AsdfFile()._open_impl(ff, buff, _get_yaml_content=True)
assert b"{$ref: ''}" in content
def url_mapping(self):
print("CustomExtension --- Return url mapping")
return [(
"http://CustomOrganization/schemas/CustomStandard/",
util.filepath_to_url(os.path.dirname(__file__)) +
"/schemas/{url_suffix}.yaml",
)]
def test_internal_reference(tmpdir):
testfile = os.path.join(str(tmpdir), 'test.asdf')
tree = {
'foo': 2,
'bar': {'$ref': '#'}
}
ff = asdf.AsdfFile(tree)
ff.find_references()
assert isinstance(ff.tree['bar'], reference.Reference)
ff.resolve_references()
assert ff.tree['bar']['foo'] == 2
tree = {
'foo': 2
}
ff = asdf.AsdfFile(
tree, uri=util.filepath_to_url(os.path.abspath(testfile)))
ff.tree['bar'] = ff.make_reference([])
buff = io.BytesIO()
ff.write_to(buff)
buff.seek(0)
ff = asdf.AsdfFile()
content = asdf.AsdfFile()._open_impl(ff, buff, _get_yaml_content=True)
assert b"{$ref: ''}" in content
def get_read_fd():
f = generic_io.get_file(path, mode='r')
assert isinstance(f, generic_io.RealFile)
assert f._uri == util.filepath_to_url(path)
# This is to check for a "feature" in Python 3.x that reading zero
# bytes from a socket causes it to stop. We have code in generic_io.py
# to workaround it.
f.read(0)
return f
def get_read_fd():
f = generic_io.get_file(path, mode='r')
assert isinstance(f, generic_io.RealFile)
assert f._uri == util.filepath_to_url(path)
# This is to check for a "feature" in Python 3.x that reading zero
# bytes from a socket causes it to stop. We have code in generic_io.py
# to workaround it.
f.read(0)
return f
def test_external_reference_invalid(tmpdir):
tree = {'foo': {'$ref': 'fail.asdf'}}
ff = asdf.AsdfFile(tree)
with pytest.raises(ValueError):
ff.resolve_references()
ff = asdf.AsdfFile(tree, uri="http://nowhere.com/")
with pytest.raises(IOError):
ff.resolve_references()
ff = asdf.AsdfFile(tree,
uri=util.filepath_to_url(
os.path.join(str(tmpdir), 'main.asdf')))
with pytest.raises(IOError):
ff.resolve_references()
class RomanDataModelExtension(AsdfExtension):
"""
Extension that maps datamodel schema URIs to their corresponding
locations on the disk. This allows the asdf package to locate
the schema content when we request validation against one of these
URIs.
"""
# The AsdfExtension interface requires types and tag_mapping attributes,
# but we won't actually be using them here.
types = []
tag_mapping = []
url_mapping = [
(URI_PREFIX,
util.filepath_to_url(str(SCHEMAS_ROOT)) + "/{url_suffix}.yaml"),
]
def test_version_mismatch_file(tmpdir):
testfile = os.path.join(str(tmpdir), 'mismatch.asdf')
yaml = """
a: !core/complex-42.0.0
0j
"""
buff = helpers.yaml_to_asdf(yaml)
with open(testfile, 'wb') as handle:
handle.write(buff.read())
expected_uri = util.filepath_to_url(str(testfile))
with pytest.warns(AsdfConversionWarning,
match="tag:stsci.edu:asdf/core/complex"):
with asdf.open(testfile, ignore_version_mismatch=False) as ff:
assert ff._fname == expected_uri
assert isinstance(ff.tree['a'], complex)
def test_external_reference_invalid(tmpdir):
tree = {
'foo': {
'$ref': 'fail.asdf'
}
}
ff = asdf.AsdfFile(tree)
with pytest.raises(ValueError):
ff.resolve_references()
ff = asdf.AsdfFile(tree, uri="http://httpstat.us/404")
with pytest.raises(IOError):
ff.resolve_references()
ff = asdf.AsdfFile(tree, uri=util.filepath_to_url(
os.path.join(str(tmpdir), 'main.asdf')))
with pytest.raises(IOError):
ff.resolve_references()
def url_mapping(self):
return [('http://stsci.edu/schemas/asdf/core/complex-42.0.0',
util.filepath_to_url(TEST_DATA_PATH) +
'/complex-42.0.0.yaml')]
import os
from pathlib import Path
from asdf.util import filepath_to_url
SCHEMA_PATH = str(Path(__file__).resolve().parents[0] / "schemas")
WELDX_SCHEMA_URI_BASE = "http://weldx.bam.de/schemas/"
WELDX_TAG_BASE = "tag:weldx.bam.de:weldx"
WELDX_URL_MAPPING = [
(
WELDX_SCHEMA_URI_BASE,
filepath_to_url(os.path.join(SCHEMA_PATH, "weldx.bam.de"))
+ "/{url_suffix}.yaml",
)
]
from .tags.transform.compound import *
from .tags.transform.polynomial import *
from .tags.transform.projections import *
from .tags.transform.tabular import *
from .tags.unit.quantity import *
from .tags.unit.unit import *
from .types import _astropy_types, _astropy_asdf_types
__all__ = ['AstropyExtension', 'AstropyAsdfExtension']
ASTROPY_SCHEMA_URI_BASE = 'http://astropy.org/schemas/'
SCHEMA_PATH = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'schemas'))
ASTROPY_URL_MAPPING = [
(ASTROPY_SCHEMA_URI_BASE,
filepath_to_url(os.path.join(SCHEMA_PATH, 'astropy.org')) +
'/{url_suffix}.yaml')
]
# This extension is used to register custom types that have both tags and
# schemas defined by Astropy.
class AstropyExtension(AsdfExtension):
@property
def types(self):
return _astropy_types
@property
def tag_mapping(self):
return [('tag:astropy.org:astropy',
ASTROPY_SCHEMA_URI_BASE + 'astropy{tag_suffix}')]
from .tags.transform.tabular import *
from .tags.unit.quantity import *
from .tags.unit.unit import *
from .tags.unit.equivalency import *
from .types import _astropy_types, _astropy_asdf_types
__all__ = ['AstropyExtension', 'AstropyAsdfExtension']
ASTROPY_SCHEMA_URI_BASE = 'http://astropy.org/schemas/'
SCHEMA_PATH = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'data', 'schemas'))
ASTROPY_URL_MAPPING = [
(ASTROPY_SCHEMA_URI_BASE,
filepath_to_url(
os.path.join(SCHEMA_PATH, 'astropy.org')) +
'/{url_suffix}.yaml')]
# This extension is used to register custom types that have both tags and
# schemas defined by Astropy.
class AstropyExtension(AsdfExtension):
@property
def types(self):
return _astropy_types
@property
def tag_mapping(self):
return [('tag:astropy.org:astropy',
ASTROPY_SCHEMA_URI_BASE + 'astropy{tag_suffix}')]
def get_write_fd():
f = generic_io.get_file(open(path, 'wb'), mode='w', close=True)
assert isinstance(f, generic_io.RealFile)
assert f._uri == util.filepath_to_url(path)
return f
def test_external_reference(tmpdir):
exttree = {
'cool_stuff': {
'a': np.array([0, 1, 2], np.float),
'b': np.array([3, 4, 5], np.float)
},
'list_of_stuff': ['foobar', 42,
np.array([7, 8, 9], np.float)]
}
external_path = os.path.join(str(tmpdir), 'external.asdf')
ext = asdf.AsdfFile(exttree)
# Since we're testing with small arrays, force all arrays to be stored
# in internal blocks rather than letting some of them be automatically put
# inline.
ext.write_to(external_path, all_array_storage='internal')
external_path = os.path.join(str(tmpdir), 'external2.asdf')
ff = asdf.AsdfFile(exttree)
ff.write_to(external_path, all_array_storage='internal')
tree = {
# The special name "data" here must be an array. This is
# included so that such validation can be ignored when we just
# have a "$ref".
'data': {
'$ref': 'external.asdf#/cool_stuff/a'
},
'science_data': {
'$ref': 'external.asdf#/cool_stuff/a'
},
'science_data2': {
'$ref': 'external2.asdf#/cool_stuff/a'
},
'foobar': {
'$ref': 'external.asdf#/list_of_stuff/0',
},
'answer': {
'$ref': 'external.asdf#/list_of_stuff/1'
},
'array': {
'$ref': 'external.asdf#/list_of_stuff/2',
},
'whole_thing': {
'$ref': 'external.asdf#'
},
'myself': {
'$ref': '#',
},
'internal': {
'$ref': '#science_data'
}
}
def do_asserts(ff):
assert 'unloaded' in repr(ff.tree['science_data'])
assert 'unloaded' in str(ff.tree['science_data'])
assert len(ff._external_asdf_by_uri) == 0
assert_array_equal(ff.tree['science_data'], exttree['cool_stuff']['a'])
assert len(ff._external_asdf_by_uri) == 1
with pytest.raises((ValueError, RuntimeError)):
# Assignment destination is readonly
ff.tree['science_data'][0] = 42
assert_array_equal(ff.tree['science_data2'],
exttree['cool_stuff']['a'])
assert len(ff._external_asdf_by_uri) == 2
assert ff.tree['foobar']() == 'foobar'
assert ff.tree['answer']() == 42
assert_array_equal(ff.tree['array'], exttree['list_of_stuff'][2])
assert_tree_match(ff.tree['whole_thing'](), exttree)
assert_array_equal(ff.tree['whole_thing']['cool_stuff']['a'],
exttree['cool_stuff']['a'])
assert_array_equal(ff.tree['myself']['science_data'],
exttree['cool_stuff']['a'])
# Make sure that referencing oneself doesn't make another call
# to disk.
assert len(ff._external_asdf_by_uri) == 2
assert_array_equal(ff.tree['internal'], exttree['cool_stuff']['a'])
with asdf.AsdfFile(tree,
uri=util.filepath_to_url(
os.path.join(str(tmpdir), 'main.asdf'))) as ff:
do_asserts(ff)
internal_path = os.path.join(str(tmpdir), 'main.asdf')
ff.write_to(internal_path)
with asdf.open(internal_path) as ff:
do_asserts(ff)
with asdf.open(internal_path) as ff:
assert len(ff._external_asdf_by_uri) == 0
ff.resolve_references()
assert len(ff._external_asdf_by_uri) == 2
assert isinstance(ff.tree['data'], ndarray.NDArrayType)
assert isinstance(ff.tree['science_data'], ndarray.NDArrayType)
assert_array_equal(ff.tree['science_data'], exttree['cool_stuff']['a'])
assert_array_equal(ff.tree['science_data2'],
exttree['cool_stuff']['a'])
assert ff.tree['foobar'] == 'foobar'
assert ff.tree['answer'] == 42
assert_array_equal(ff.tree['array'], exttree['list_of_stuff'][2])
assert_tree_match(ff.tree['whole_thing'], exttree)
assert_array_equal(ff.tree['whole_thing']['cool_stuff']['a'],
exttree['cool_stuff']['a'])
assert_array_equal(ff.tree['myself']['science_data'],
exttree['cool_stuff']['a'])
assert_array_equal(ff.tree['internal'], exttree['cool_stuff']['a'])
def get_write_fd():
f = generic_io.get_file(open(path, 'wb'), mode='w', close=True)
assert isinstance(f, generic_io.RealFile)
assert f._uri == util.filepath_to_url(path)
return f
def get_read_fd():
f = generic_io.get_file(io.open(path, 'r+b'), mode='rw', close=True)
assert isinstance(f, generic_io.RealFile)
assert f._uri == util.filepath_to_url(path)
return f
def url_mapping(self):
return [(
'http://nowhere.org/schemas/custom/',
util.filepath_to_url(TEST_DATA_PATH) + '/{url_suffix}.yaml')]
def url_mapping(self):
return [('http://jwst.stsci.edu/schemas/',
util.filepath_to_url(SCHEMA_PATH) + '/{url_suffix}')]
def url_mapping(self):
return [
(URL_PREFIX, util.filepath_to_url(SCHEMA_PATH) + '/{url_suffix}'),
('http://stsci.edu/schemas/fits-schema/',
util.filepath_to_url(METASCHEMA_PATH) + '/{url_suffix}.yaml'),
]
from asdf import AsdfExtension
from asdf.util import filepath_to_url
from .types import SunPyType
from .tags.map import *
from .tags.coordinates import *
__all__ = ['SunpyExtension']
SUNPY_SCHEMA_URI_BASE = 'http://sunpy.org/schemas/'
SCHEMA_PATH = Path(__file__).parent / "schemas"
SUNPY_URL_MAPPING = [
(SUNPY_SCHEMA_URI_BASE,
filepath_to_url(str(SCHEMA_PATH / "sunpy.org")) + "/{url_suffix}.yaml")
]
# This extension is used to register custom types that have both tags and
# schemas defined in SunPy
class SunpyExtension(AsdfExtension):
@property
def types(self):
return SunPyType._tags
@property
def tag_mapping(self):
return [('tag:sunpy.org:sunpy',
SUNPY_SCHEMA_URI_BASE + 'sunpy{tag_suffix}')]
def url_mapping(self):
return [
("http://stsci.edu/schemas/stpipe", util.filepath_to_url(SCHEMAS_PATH) + "/{url_suffix}.yaml"),
]
def url_mapping(self):
return [('http://nowhere.org/schemas/custom/',
util.filepath_to_url(helpers.get_test_data_path('')) +
'/{url_suffix}.yaml')]
"""Legacy asdf extension code to add custom validators."""
from asdf.extension import AsdfExtension
from asdf.types import CustomType
from asdf.util import filepath_to_url
from .constants import SCHEMA_PATH, WELDX_SCHEMA_URI_BASE, WELDX_TAG_URI_BASE
from .validators import wx_property_tag_validator, wx_shape_validator, wx_unit_validator
WELDX_URL_MAPPING = [(
WELDX_SCHEMA_URI_BASE,
filepath_to_url(str(SCHEMA_PATH)) + "/{url_suffix}.yaml",
)]
class WeldxLegacyValidatorType(CustomType):
"""Dummy legacy class to register weldx validators using legacy asdf API."""
organization = "weldx.bam.de"
standard = "weldx"
name = "legacy/validators"
version = "0.1.0"
types = []
validators = {
"wx_property_tag": wx_property_tag_validator,
"wx_unit": wx_unit_validator,
"wx_shape": wx_shape_validator,
}
versioned_siblings = []
def get_read_fd():
# Must open with mode=rw in order to get memmapped data
f = generic_io.get_file(open(path, 'r+b'), mode='rw', close=True)
assert isinstance(f, generic_io.RealFile)
assert f._uri == util.filepath_to_url(path)
return f
def url_mapping(self):
return [('http://stsci.edu/schemas/jwst_pipeline/',
util.filepath_to_url(os.path.join(SCHEMA_PATH, "stsci.edu")) +
'/jwst_pipeline/{url_suffix}.yaml')]
def get_write_fd():
f = generic_io.get_file(path, mode='w')
assert isinstance(f, generic_io.RealFile)
assert f._uri == util.filepath_to_url(path)
return f
def url_mapping(self):
return [
('http://nowhere.org/schemas/custom/',
util.filepath_to_url(TEST_DATA_PATH) + '/{url_suffix}.yaml')
]
def url_mapping(self):
return [(
"http://QualityOrganization/schemas/default/",
util.filepath_to_url(os.path.dirname(__file__)) +
"/schemas/standards/" + self._standard + "/{url_suffix}.yaml",
)]
def get_write_fd():
f = generic_io.get_file(path, mode='w')
assert isinstance(f, generic_io.RealFile)
assert f._uri == util.filepath_to_url(path)
return f
def url_mapping(self):
return [('http://nowhere.org/schemas/custom/',
util.filepath_to_url(helpers.get_test_data_path('')) +
'/{url_suffix}.yaml')]
def url_mapping(self):
return [(URL_PREFIX,
util.filepath_to_url(SCHEMA_PATH) + "/{url_suffix}")]
def url_mapping(self):
return [
('http://stsci.edu/schemas/asdf/core/complex-42.0.0',
util.filepath_to_url(TEST_DATA_PATH) + '/complex-42.0.0.yaml')
]
def test_external_reference(tmpdir):
exttree = {
'cool_stuff': {
'a': np.array([0, 1, 2], np.float),
'b': np.array([3, 4, 5], np.float)
},
'list_of_stuff': [
'foobar',
42,
np.array([7, 8, 9], np.float)
]
}
external_path = os.path.join(str(tmpdir), 'external.asdf')
ext = asdf.AsdfFile(exttree)
# Since we're testing with small arrays, force all arrays to be stored
# in internal blocks rather than letting some of them be automatically put
# inline.
ext.write_to(external_path, all_array_storage='internal')
external_path = os.path.join(str(tmpdir), 'external2.asdf')
ff = asdf.AsdfFile(exttree)
ff.write_to(external_path, all_array_storage='internal')
tree = {
# The special name "data" here must be an array. This is
# included so that such validation can be ignored when we just
# have a "$ref".
'data': {
'$ref': 'external.asdf#/cool_stuff/a'
},
'science_data': {
'$ref': 'external.asdf#/cool_stuff/a'
},
'science_data2': {
'$ref': 'external2.asdf#/cool_stuff/a'
},
'foobar': {
'$ref': 'external.asdf#/list_of_stuff/0',
},
'answer': {
'$ref': 'external.asdf#/list_of_stuff/1'
},
'array': {
'$ref': 'external.asdf#/list_of_stuff/2',
},
'whole_thing': {
'$ref': 'external.asdf#'
},
'myself': {
'$ref': '#',
},
'internal': {
'$ref': '#science_data'
}
}
def do_asserts(ff):
assert 'unloaded' in repr(ff.tree['science_data'])
assert 'unloaded' in str(ff.tree['science_data'])
assert len(ff._external_asdf_by_uri) == 0
assert_array_equal(ff.tree['science_data'], exttree['cool_stuff']['a'])
assert len(ff._external_asdf_by_uri) == 1
with pytest.raises((ValueError, RuntimeError)):
# Assignment destination is readonly
ff.tree['science_data'][0] = 42
assert_array_equal(ff.tree['science_data2'], exttree['cool_stuff']['a'])
assert len(ff._external_asdf_by_uri) == 2
assert ff.tree['foobar']() == 'foobar'
assert ff.tree['answer']() == 42
assert_array_equal(ff.tree['array'], exttree['list_of_stuff'][2])
assert_tree_match(ff.tree['whole_thing'](), exttree)
assert_array_equal(
ff.tree['whole_thing']['cool_stuff']['a'],
exttree['cool_stuff']['a'])
assert_array_equal(
ff.tree['myself']['science_data'],
exttree['cool_stuff']['a'])
# Make sure that referencing oneself doesn't make another call
# to disk.
assert len(ff._external_asdf_by_uri) == 2
assert_array_equal(ff.tree['internal'], exttree['cool_stuff']['a'])
with asdf.AsdfFile(tree, uri=util.filepath_to_url(
os.path.join(str(tmpdir), 'main.asdf'))) as ff:
do_asserts(ff)
internal_path = os.path.join(str(tmpdir), 'main.asdf')
ff.write_to(internal_path)
with asdf.open(internal_path) as ff:
do_asserts(ff)
with asdf.open(internal_path) as ff:
assert len(ff._external_asdf_by_uri) == 0
ff.resolve_references()
assert len(ff._external_asdf_by_uri) == 2
assert isinstance(ff.tree['data'], ndarray.NDArrayType)
assert isinstance(ff.tree['science_data'], ndarray.NDArrayType)
assert_array_equal(ff.tree['science_data'], exttree['cool_stuff']['a'])
assert_array_equal(ff.tree['science_data2'], exttree['cool_stuff']['a'])
assert ff.tree['foobar'] == 'foobar'
assert ff.tree['answer'] == 42
assert_array_equal(ff.tree['array'], exttree['list_of_stuff'][2])
assert_tree_match(ff.tree['whole_thing'], exttree)
assert_array_equal(
ff.tree['whole_thing']['cool_stuff']['a'],
exttree['cool_stuff']['a'])
assert_array_equal(
ff.tree['myself']['science_data'],
exttree['cool_stuff']['a'])
assert_array_equal(ff.tree['internal'], exttree['cool_stuff']['a'])
def get_read_fd():
f = generic_io.get_file(io.open(path, 'r+b'), mode='rw', close=True)
assert isinstance(f, generic_io.RealFile)
assert f._uri == util.filepath_to_url(path)
return f
import urllib
from asdf.util import filepath_to_url
from asdf.extension import AsdfExtension
from astropy.io.misc.asdf.extension import ASTROPY_SCHEMA_URI_BASE
from .tags.spectra import *
from .types import _specutils_types
SCHEMA_PATH = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'schemas'))
SPECUTILS_URL_MAPPING = [
(urllib.parse.urljoin(ASTROPY_SCHEMA_URI_BASE, 'specutils/'),
filepath_to_url(
os.path.join(SCHEMA_PATH, 'astropy.org', 'specutils')) +
'/{url_suffix}.yaml')]
class SpecutilsExtension(AsdfExtension):
"""
Defines specutils types and schema locations to be used by ASDF
"""
@property
def types(self):
"""
Collection of tag types that are used by ASDF for serialization
"""
return _specutils_types
@property
from asdf import AsdfExtension
from asdf.util import filepath_to_url
from .types import SunPyType
from .tags.map import *
from .tags.coordinates import *
__all__ = ['SunpyExtension']
SUNPY_SCHEMA_URI_BASE = 'http://sunpy.org/schemas/'
SCHEMA_PATH = Path(__file__).parent / "schemas"
SUNPY_URL_MAPPING = [(SUNPY_SCHEMA_URI_BASE,
filepath_to_url(str(SCHEMA_PATH / "sunpy.org")) + "/{url_suffix}.yaml")]
# This extension is used to register custom types that have both tags and
# schemas defined in SunPy
class SunpyExtension(AsdfExtension):
@property
def types(self):
return SunPyType._tags
@property
def tag_mapping(self):
return [('tag:sunpy.org:sunpy',
SUNPY_SCHEMA_URI_BASE + 'sunpy{tag_suffix}')]
@property
def get_read_fd():
# Must open with mode=rw in order to get memmapped data
f = generic_io.get_file(open(path, 'r+b'), mode='rw', close=True)
assert isinstance(f, generic_io.RealFile)
assert f._uri == util.filepath_to_url(path)
return f
def url_mapping(self):
return [('http://jwst.stsci.edu/schemas/',
util.filepath_to_url(SCHEMA_PATH) +
'/{url_suffix}')]
from .tags.target import *
from .tags.instrument import *
from .tags.detector2d_ccd import *
from .tags.subarray import *
from .tags.proposal import *
from .tags.observer import *
from .types import _astronomy_datamodel_types, _astronomy_datamodel_asdf_types
__all__ = ['AstronomyDataModelExtension', 'AstronomyDataModelAsdfExtension']
ASTRONOMY_DATAMODEL_SCHEMA_URI_BASE = 'http://astroasdf.org/astronomy_datamodel/'
SCHEMA_PATH = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'asdf_astronomy_schemas/schemas'))
ASTRONOMY_DATAMODEL_URL_MAPPING = [
(ASTRONOMY_DATAMODEL_SCHEMA_URI_BASE,
os.path.join(filepath_to_url(os.path.join(SCHEMA_PATH, 'astroasdf.org')),
'{url_suffix}.yaml'))
]
# This extension is used to register custom types that have both tags and
# schemas defined by Astropy.
class AstronomyDataModelExtension(AsdfExtension):
@property
def types(self):
return _astronomy_datamodel_types
@property
def tag_mapping(self):
return [('tag:astroasdf.org:astronomy_datamodel',
ASTRONOMY_DATAMODEL_SCHEMA_URI_BASE + '{tag_suffix}')]
