def test_backwards_compat_gcrs():
obsgeoloc = (
3.0856775814671916e+16,
9.257032744401574e+16,
6.1713551629343834e+19
)
obsgeovel = (2.0, 1.0, 8.0)
old_frame_yaml = """
frames:
- !wcs/celestial_frame-1.0.0
axes_names: [lon, lat]
name: CelestialFrame
reference_frame:
type: GCRS
obsgeoloc:
- [%f, %f, %f]
- !unit/unit-1.0.0 m
obsgeovel:
- [%f, %f, %f]
- !unit/unit-1.0.0 m s-1
obstime: !time/time-1.0.0 2010-01-01 00:00:00.000
unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg]
""" % (obsgeovel + obsgeoloc)
new_frame_yaml = """
frames:
- !wcs/celestial_frame-1.1.0
axes_names: [lon, lat]
name: CelestialFrame
reference_frame:
type: GCRS
obsgeoloc:
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f}
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f}
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f}
obsgeovel:
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f}
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f}
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f}
obstime: !time/time-1.1.0 2010-01-01 00:00:00.000
unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg]
""" % (obsgeovel + obsgeoloc)
old_buff = helpers.yaml_to_asdf(old_frame_yaml)
old_asdf = AsdfFile.open(old_buff)
old_frame = old_asdf.tree['frames'][0]
old_loc = old_frame.reference_frame.obsgeoloc
old_vel = old_frame.reference_frame.obsgeovel
new_buff = helpers.yaml_to_asdf(new_frame_yaml)
new_asdf = AsdfFile.open(new_buff)
new_frame = new_asdf.tree['frames'][0]
new_loc = new_frame.reference_frame.obsgeoloc
new_vel = new_frame.reference_frame.obsgeovel
assert (old_loc.x == new_loc.x and old_loc.y == new_loc.y and
old_loc.z == new_loc.z)
assert (old_vel.x == new_vel.x and old_vel.y == new_vel.y and
old_vel.z == new_vel.z)
def test_backwards_compat_gcrs():
obsgeoloc = (
3.0856775814671916e+16,
9.257032744401574e+16,
6.1713551629343834e+19
)
obsgeovel = (2.0, 1.0, 8.0)
old_frame_yaml = """
frames:
- !wcs/celestial_frame-1.0.0
axes_names: [lon, lat]
name: CelestialFrame
reference_frame:
type: GCRS
obsgeoloc:
- [%f, %f, %f]
- !unit/unit-1.0.0 m
obsgeovel:
- [%f, %f, %f]
- !unit/unit-1.0.0 m s-1
obstime: !time/time-1.0.0 2010-01-01 00:00:00.000
unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg]
""" % (obsgeovel + obsgeoloc)
new_frame_yaml = """
frames:
- !wcs/celestial_frame-1.1.0
axes_names: [lon, lat]
name: CelestialFrame
reference_frame:
type: GCRS
obsgeoloc:
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f}
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f}
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f}
obsgeovel:
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f}
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f}
- !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f}
obstime: !time/time-1.1.0 2010-01-01 00:00:00.000
unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg]
""" % (obsgeovel + obsgeoloc)
old_buff = helpers.yaml_to_asdf(old_frame_yaml)
old_asdf = AsdfFile.open(old_buff)
old_frame = old_asdf.tree['frames'][0]
old_loc = old_frame.reference_frame.obsgeoloc
old_vel = old_frame.reference_frame.obsgeovel
new_buff = helpers.yaml_to_asdf(new_frame_yaml)
new_asdf = AsdfFile.open(new_buff)
new_frame = new_asdf.tree['frames'][0]
new_loc = new_frame.reference_frame.obsgeoloc
new_vel = new_frame.reference_frame.obsgeovel
assert (old_loc.x == new_loc.x and old_loc.y == new_loc.y and
old_loc.z == new_loc.z)
assert (old_vel.x == new_vel.x and old_vel.y == new_vel.y and
old_vel.z == new_vel.z)
def test_nonexistent_tag(tmpdir):
"""
This tests the case where a node is tagged with a type that apparently
comes from an extension that is known, but the type itself can't be found.
This could occur when a more recent version of an installed package
provides the new type, but an older version of the package is installed.
ASDF should still be able to open the file in this case, but it won't be
able to restore the type.
The bug that prompted this test results from attempting to load a schema
file that doesn't exist, which is why this test belongs in this file.
"""
# This shouldn't ever happen, but it's a useful test case
yaml = """
a: !core/doesnt_exist-1.0.0
hello
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(AsdfWarning, match="Unable to locate schema file"):
with asdf.open(buff) as af:
assert str(af['a']) == 'hello'
# This is a more realistic case since we're using an external extension
yaml = """
a: !<tag:nowhere.org:custom/doesnt_exist-1.0.0>
hello
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(AsdfWarning, match="Unable to locate schema file"):
with asdf.open(buff, extensions=CustomExtension()) as af:
assert str(af['a']) == 'hello'
def test_extension_version_warning():
yaml = """
history:
extensions:
- !core/extension_metadata-1.0.0
extension_class: asdf.extension.BuiltinExtension
software: !core/software-1.0.0
name: asdf
version: 100.0.3
"""
buff = yaml_to_asdf(yaml)
with pytest.warns(
AsdfWarning,
match=
"File was created with extension class 'asdf.extension.BuiltinExtension'"
):
with asdf.open(buff):
pass
buff.seek(0)
# Make sure suppressing the warning works too
with assert_no_warnings():
with asdf.open(buff, ignore_missing_extensions=True):
pass
def make_complex_asdf(string):
yaml = """
a: !core/complex-1.0.0
{}
""".format(string)
return helpers.yaml_to_asdf(yaml)
def test_extension_version_warning():
yaml = """
history:
extensions:
- !core/extension_metadata-1.0.0
extension_class: asdf.extension.BuiltinExtension
software: !core/software-1.0.0
name: asdf
version: 100.0.3
"""
buff = yaml_to_asdf(yaml)
with pytest.warns(None) as warnings:
with asdf.open(buff) as af:
pass
assert len(warnings) == 1, display_warnings(warnings)
assert str(warnings[0].message).startswith(
"File was created with extension 'asdf.extension.BuiltinExtension' "
"from package asdf-100.0.3")
buff.seek(0)
# Make sure suppressing the warning works too
with pytest.warns(None) as warnings:
with asdf.open(buff, ignore_missing_extensions=True) as af:
pass
assert len(warnings) == 0, display_warnings(warnings)
def test_fill_and_remove_defaults():
class DefaultType(dict, asdftypes.CustomType):
name = 'default'
organization = 'nowhere.org'
version = (1, 0, 0)
standard = 'custom'
class DefaultTypeExtension(CustomExtension):
@property
def types(self):
return [DefaultType]
yaml = """
custom: !<tag:nowhere.org:custom/default-1.0.0>
b: {}
"""
buff = helpers.yaml_to_asdf(yaml)
with asdf.AsdfFile.open(buff, extensions=[DefaultTypeExtension()]) as ff:
assert 'a' in ff.tree['custom']
assert ff.tree['custom']['a'] == 42
assert ff.tree['custom']['b']['c'] == 82
buff.seek(0)
with asdf.AsdfFile.open(buff, extensions=[DefaultTypeExtension()],
do_not_fill_defaults=True) as ff:
assert 'a' not in ff.tree['custom']
assert 'c' not in ff.tree['custom']['b']
ff.fill_defaults()
assert 'a' in ff.tree['custom']
assert ff.tree['custom']['a'] == 42
assert 'c' in ff.tree['custom']['b']
assert ff.tree['custom']['b']['c'] == 82
ff.remove_defaults()
assert 'a' not in ff.tree['custom']
assert 'c' not in ff.tree['custom']['b']
def test_type_missing_dependencies():
class MissingType(asdftypes.AsdfType):
name = 'missing'
organization = 'nowhere.org'
version = (1, 1, 0)
standard = 'custom'
types = ['asdfghjkl12345.foo']
requires = ["ASDFGHJKL12345"]
class DefaultTypeExtension(CustomExtension):
@property
def types(self):
return [MissingType]
yaml = """
custom: !<tag:nowhere.org:custom/missing-1.1.0>
b: {foo: 42}
"""
buff = helpers.yaml_to_asdf(yaml)
with catch_warnings() as w:
with asdf.AsdfFile.open(buff,
extensions=[DefaultTypeExtension()]) as ff:
assert ff.tree['custom']['b']['foo'] == 42
assert len(w) == 1
def test_type_missing_dependencies():
pytest.importorskip('astropy', '3.0.0')
class MissingType(types.CustomType):
name = 'missing'
organization = 'nowhere.org'
version = (1, 1, 0)
standard = 'custom'
types = ['asdfghjkl12345.foo']
requires = ["ASDFGHJKL12345"]
class DefaultTypeExtension(CustomExtension):
@property
def types(self):
return [MissingType]
yaml = """
custom: !<tag:nowhere.org:custom/missing-1.1.0>
b: {foo: 42}
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(
AsdfConversionWarning,
match="Failed to convert tag:nowhere.org:custom/missing-1.1.0"):
with asdf.open(buff, extensions=[DefaultTypeExtension()]) as ff:
assert ff.tree['custom']['b']['foo'] == 42
def run_schema_example_test(organization, standard, name, version, check_func=None):
import asdf
from asdf.tests import helpers
from asdf.types import format_tag
from asdf.schema import load_schema
tag = format_tag(organization, standard, version, name)
uri = asdf.resolver.default_tag_to_url_mapping(tag)
r = asdf.AsdfFile().resolver
examples = []
schema = load_schema(uri, resolver=r)
for node in asdf.treeutil.iter_tree(schema):
if (isinstance(node, dict) and
'examples' in node and
isinstance(node['examples'], list)):
for desc, example in node['examples']:
examples.append(example)
for example in examples:
buff = helpers.yaml_to_asdf('example: ' + example.strip())
ff = asdf.AsdfFile(uri=uri)
# Add some dummy blocks so that the ndarray examples work
for i in range(3):
b = asdf.block.Block(np.zeros((1024*1024*8), dtype=np.uint8))
b._used = True
ff.blocks.add(b)
ff._open_impl(ff, buff, mode='r')
if check_func:
check_func(ff)
def run_schema_example_test(organization,
standard,
name,
version,
check_func=None):
import asdf
from asdf.tests import helpers
from asdf.types import format_tag
from asdf.schema import load_schema
tag = format_tag(organization, standard, version, name)
uri = asdf.extension.default_extensions.extension_list.tag_mapping(tag)
r = asdf.extension.get_default_resolver()
examples = []
schema = load_schema(uri, resolver=r)
for node in asdf.treeutil.iter_tree(schema):
if (isinstance(node, dict) and 'examples' in node
and isinstance(node['examples'], list)):
for desc, example in node['examples']:
examples.append(example)
for example in examples:
buff = helpers.yaml_to_asdf('example: ' + example.strip())
ff = asdf.AsdfFile(uri=uri)
# Add some dummy blocks so that the ndarray examples work
for i in range(3):
b = asdf.block.Block(np.zeros((1024 * 1024 * 8), dtype=np.uint8))
b._used = True
ff.blocks.add(b)
ff._open_impl(ff, buff, mode='r')
if check_func:
check_func(ff)
def test_read_large_literal(value):
yaml = f"integer: {value}"
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(AsdfWarning, match="Invalid integer literal value"):
with asdf.open(buff) as af:
assert af['integer'] == value
yaml = f"{value}: foo"
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(AsdfWarning, match="Invalid integer literal value"):
with asdf.open(buff) as af:
assert af[value] == "foo"
def test_deserialize_compound_user_inverse(tmpdir):
"""
Confirm that we are able to correctly reconstruct a
compound model with a user inverse set on one of its
component models.
Due to code in TransformType that facilitates circular
inverses, the user inverse of the component model is
not available at the time that the CompoundModel is
constructed.
"""
yaml = """
model: !transform/concatenate-1.2.0
forward:
- !transform/shift-1.2.0
inverse: !transform/shift-1.2.0 {offset: 5.0}
offset: -10.0
- !transform/shift-1.2.0 {offset: -20.0}
"""
buff = helpers.yaml_to_asdf(yaml)
with asdf.open(buff) as af:
model = af["model"]
assert model.has_inverse()
assert model.inverse(-5, -20) == (0, 0)
def run_schema_example_test(organization, standard, name, version, check_func=None):
import asdf
from asdf.tests import helpers
from asdf.types import format_tag
from asdf.resolver import default_resolver
tag = format_tag(organization, standard, version, name)
schema_path = urllib.parse.urlparse(default_resolver(tag)).path
with open(schema_path, 'rb') as ff:
schema = yaml.load(ff)
examples = []
for node in asdf.treeutil.iter_tree(schema):
if (isinstance(node, dict) and
'examples' in node and
isinstance(node['examples'], list)):
for desc, example in node['examples']:
examples.append(example)
for example in examples:
buff = helpers.yaml_to_asdf('example: ' + example.strip())
ff = asdf.AsdfFile(uri=schema_path)
# Add some dummy blocks so that the ndarray examples work
for i in range(3):
b = asdf.block.Block(np.zeros((1024*1024*8), dtype=np.uint8))
b._used = True
ff.blocks.add(b)
ff._open_impl(ff, buff, mode='r')
if check_func:
check_func(ff)
def test_fill_and_remove_defaults():
class DefaultType(dict, types.CustomType):
name = 'default'
organization = 'nowhere.org'
version = (1, 0, 0)
standard = 'custom'
class DefaultTypeExtension(CustomExtension):
@property
def types(self):
return [DefaultType]
yaml = """
custom: !<tag:nowhere.org:custom/default-1.0.0>
b: {}
"""
buff = helpers.yaml_to_asdf(yaml)
with asdf.open(buff, extensions=[DefaultTypeExtension()]) as ff:
assert 'a' in ff.tree['custom']
assert ff.tree['custom']['a'] == 42
assert ff.tree['custom']['b']['c'] == 82
buff.seek(0)
with asdf.open(buff, extensions=[DefaultTypeExtension()],
do_not_fill_defaults=True) as ff:
assert 'a' not in ff.tree['custom']
assert 'c' not in ff.tree['custom']['b']
ff.fill_defaults()
assert 'a' in ff.tree['custom']
assert ff.tree['custom']['a'] == 42
assert 'c' in ff.tree['custom']['b']
assert ff.tree['custom']['b']['c'] == 82
ff.remove_defaults()
assert 'a' not in ff.tree['custom']
assert 'c' not in ff.tree['custom']['b']
def test_type_missing_dependencies():
class MissingType(asdftypes.CustomType):
name = 'missing'
organization = 'nowhere.org'
version = (1, 1, 0)
standard = 'custom'
types = ['asdfghjkl12345.foo']
requires = ["ASDFGHJKL12345"]
class DefaultTypeExtension(CustomExtension):
@property
def types(self):
return [MissingType]
yaml = """
custom: !<tag:nowhere.org:custom/missing-1.1.0>
b: {foo: 42}
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(None) as w:
with asdf.AsdfFile.open(buff, extensions=[DefaultTypeExtension()]) as ff:
assert ff.tree['custom']['b']['foo'] == 42
assert len(w) == 1
def make_complex_asdf(string):
yaml = """
a: !core/complex-1.0.0
{}
""".format(string)
return helpers.yaml_to_asdf(yaml)
def test_extension_version_warning():
yaml = """
history:
extensions:
- !core/extension_metadata-1.0.0
extension_class: asdf.extension.BuiltinExtension
software: !core/software-1.0.0
name: asdf
version: 100.0.3
"""
buff = yaml_to_asdf(yaml)
with pytest.warns(None) as warnings:
with asdf.open(buff) as af:
pass
assert len(warnings) == 1, display_warnings(warnings)
assert str(warnings[0].message).startswith(
"File was created with extension 'asdf.extension.BuiltinExtension' "
"from package asdf-100.0.3")
buff.seek(0)
# Make sure suppressing the warning works too
with pytest.warns(None) as warnings:
with asdf.open(buff, ignore_missing_extensions=True) as af:
pass
assert len(warnings) == 0, display_warnings(warnings)
def test_old_format(recwarn):
warnings.simplefilter("always")
content = helpers.yaml_to_asdf("""
extension: !core/extension_metadata-1.0.0
extension_class: asdf.extension.BuiltinExtension
software: {name: asdf, version: 2.5.0}
""")
with asdf.open(content) as af:
extension = af.tree["extension"]
assert isinstance(extension, asdf.tags.core.ExtensionMetadata)
assert extension.extension_class == "asdf.extension.BuiltinExtension"
assert isinstance(extension.package, asdf.tags.core.Software)
assert extension.package["name"] == "asdf"
assert extension.package["version"] == "2.5.0"
software = extension.software
assert isinstance(software, asdf.tags.core.Software)
assert software["name"] == "asdf"
assert software["version"] == "2.5.0"
# One warning for opening a file in the old format, another for
# accessing the 'software' property:
assert len(recwarn) == 2
assert recwarn.pop(asdf.exceptions.AsdfDeprecationWarning)
assert recwarn.pop(asdf.exceptions.AsdfDeprecationWarning)
def test_nonexistent_tag(tmpdir):
"""
This tests the case where a node is tagged with a type that apparently
comes from an extension that is known, but the type itself can't be found.
This could occur when a more recent version of an installed package
provides the new type, but an older version of the package is installed.
ASDF should still be able to open the file in this case, but it won't be
able to restore the type.
The bug that prompted this test results from attempting to load a schema
file that doesn't exist, which is why this test belongs in this file.
"""
# This shouldn't ever happen, but it's a useful test case
yaml = """
a: !core/doesnt_exist-1.0.0
hello
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(None) as w:
with asdf.open(buff) as af:
assert str(af['a']) == 'hello'
# Currently there are 3 warnings since one occurs on each of the
# validation passes. It would be good to consolidate these
# eventually
assert len(w) == 3, helpers.display_warnings(w)
assert str(w[0].message).startswith("Unable to locate schema file")
assert str(w[1].message).startswith("Unable to locate schema file")
assert str(w[2].message).startswith(af['a']._tag)
# This is a more realistic case since we're using an external extension
yaml = """
a: !<tag:nowhere.org:custom/doesnt_exist-1.0.0>
hello
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(None) as w:
with asdf.open(buff, extensions=CustomExtension()) as af:
assert str(af['a']) == 'hello'
assert len(w) == 3, helpers.display_warnings(w)
assert str(w[0].message).startswith("Unable to locate schema file")
assert str(w[1].message).startswith("Unable to locate schema file")
assert str(w[2].message).startswith(af['a']._tag)
def test_nonexistent_tag(tmpdir):
"""
This tests the case where a node is tagged with a type that apparently
comes from an extension that is known, but the type itself can't be found.
This could occur when a more recent version of an installed package
provides the new type, but an older version of the package is installed.
ASDF should still be able to open the file in this case, but it won't be
able to restore the type.
The bug that prompted this test results from attempting to load a schema
file that doesn't exist, which is why this test belongs in this file.
"""
# This shouldn't ever happen, but it's a useful test case
yaml = """
a: !core/doesnt_exist-1.0.0
hello
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(None) as w:
with asdf.open(buff) as af:
assert str(af['a']) == 'hello'
# Currently there are 3 warnings since one occurs on each of the
# validation passes. It would be good to consolidate these
# eventually
assert len(w) == 3, helpers.display_warnings(w)
assert str(w[0].message).startswith("Unable to locate schema file")
assert str(w[1].message).startswith("Unable to locate schema file")
assert str(w[2].message).startswith(af['a']._tag)
# This is a more realistic case since we're using an external extension
yaml = """
a: !<tag:nowhere.org:custom/doesnt_exist-1.0.0>
hello
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(None) as w:
with asdf.open(buff, extensions=CustomExtension()) as af:
assert str(af['a']) == 'hello'
assert len(w) == 3, helpers.display_warnings(w)
assert str(w[0].message).startswith("Unable to locate schema file")
assert str(w[1].message).startswith("Unable to locate schema file")
assert str(w[2].message).startswith(af['a']._tag)
def test_invalid_complex():
yaml = """
a: !core/complex-1.0.0
3 + 4i
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.raises(ValueError):
with asdf.AsdfFile.open(buff):
pass
def roundtrip_quantity(yaml, quantity):
buff = helpers.yaml_to_asdf(yaml)
with asdf.AsdfFile.open(buff) as ff:
assert (ff.tree['quantity'] == quantity).all()
buff2 = io.BytesIO()
ff.write_to(buff2)
buff2.seek(0)
with asdf.AsdfFile.open(buff2) as ff:
assert (ff.tree['quantity'] == quantity).all()
def test_read_large_literal():
value = 1 << 64
yaml = """integer: {}""".format(value)
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(AsdfWarning, match="Invalid integer literal value"):
with asdf.open(buff) as af:
assert af['integer'] == value
def roundtrip_quantity(yaml, quantity):
buff = helpers.yaml_to_asdf(yaml)
with asdf.AsdfFile.open(buff) as ff:
assert (ff.tree['quantity'] == quantity).all()
buff2 = io.BytesIO()
ff.write_to(buff2)
buff2.seek(0)
with asdf.AsdfFile.open(buff2) as ff:
assert (ff.tree['quantity'] == quantity).all()
def test_schema_example(filename, example):
"""Pytest to check validity of a specific example within schema file
Parameters:
-----------
filename : name of the schema file containing example to be tested
example: string representing example
This function is called with a range of parameters by pytest's
'parametrize' utility in order to account for all examples in all schema
files.
"""
if not HAS_GWCS and re.search(r'frame-\d\.\d\.\d\.yaml', filename):
return pytest.skip
standard_version = _find_standard_version(filename)
# Make sure that the examples in the schema files (and thus the
# ASDF standard document) are valid.
buff = helpers.yaml_to_asdf('example: ' + example.strip(),
standard_version=standard_version)
ff = asdf.AsdfFile(uri=util.filepath_to_url(os.path.abspath(filename)),
extensions=TestExtension())
# Fake an external file
ff2 = asdf.AsdfFile({'data': np.empty((1024 * 1024 * 8), dtype=np.uint8)})
ff._external_asdf_by_uri[util.filepath_to_url(
os.path.abspath(
os.path.join(os.path.dirname(filename), 'external.asdf')))] = ff2
# Add some dummy blocks so that the ndarray examples work
for i in range(3):
b = block.Block(np.zeros((1024 * 1024 * 8), dtype=np.uint8))
b._used = True
ff.blocks.add(b)
b._array_storage = "streamed"
try:
with catch_warnings() as w:
ff._open_impl(ff, buff)
# Do not tolerate any warnings that occur during schema validation,
# other than a few that we expect to occur under certain circumstances
_assert_warnings(w)
except:
print("From file:", filename)
raise
# Just test we can write it out. A roundtrip test
# wouldn't always yield the correct result, so those have
# to be covered by "real" unit tests.
if b'external.asdf' not in buff.getvalue():
buff = io.BytesIO()
ff.write_to(buff)
def test_invalid_nested():
class CustomType(str, asdftypes.CustomType):
name = 'custom'
organization = 'nowhere.org'
version = (1, 0, 0)
standard = 'custom'
class CustomTypeExtension(CustomExtension):
@property
def types(self):
return [CustomType]
yaml = """
custom: !<tag:nowhere.org:custom/custom-1.0.0>
foo
"""
buff = helpers.yaml_to_asdf(yaml)
# This should cause a warning but not an error because without explicitly
# providing an extension, our custom type will not be recognized and will
# simply be converted to a raw type.
with pytest.warns(None) as warning:
with asdf.AsdfFile.open(buff):
pass
assert len(warning) == 1
buff.seek(0)
with pytest.raises(ValidationError):
with asdf.AsdfFile.open(buff, extensions=[CustomTypeExtension()]):
pass
# Make sure tags get validated inside of other tags that know
# nothing about them.
yaml = """
array: !core/ndarray-1.0.0
data: [0, 1, 2]
custom: !<tag:nowhere.org:custom/custom-1.0.0>
foo
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.raises(ValidationError):
with asdf.AsdfFile.open(buff, extensions=[CustomTypeExtension()]):
pass
def test_invalid_nested():
class CustomType(str, asdftypes.AsdfType):
name = 'custom'
organization = 'nowhere.org'
version = (1, 0, 0)
standard = 'custom'
class CustomTypeExtension(CustomExtension):
@property
def types(self):
return [CustomType]
yaml = """
custom: !<tag:nowhere.org:custom/custom-1.0.0>
foo
"""
buff = helpers.yaml_to_asdf(yaml)
# This should cause a warning but not an error because without explicitly
# providing an extension, our custom type will not be recognized and will
# simply be converted to a raw type.
with catch_warnings() as warning:
with asdf.AsdfFile.open(buff):
pass
assert len(warning) == 1
buff.seek(0)
with pytest.raises(ValidationError):
with asdf.AsdfFile.open(buff, extensions=[CustomTypeExtension()]):
pass
# Make sure tags get validated inside of other tags that know
# nothing about them.
yaml = """
array: !core/ndarray-1.0.0
data: [0, 1, 2]
custom: !<tag:nowhere.org:custom/custom-1.0.0>
foo
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.raises(ValidationError):
with asdf.AsdfFile.open(buff, extensions=[CustomTypeExtension()]):
pass
def test_inline_shape_mismatch():
content = """
arr: !core/ndarray-1.0.0
data: [1, 2, 3]
shape: [2]
"""
buff = helpers.yaml_to_asdf(content)
with pytest.raises(ValueError):
with asdf.open(buff) as ff:
pass
def test_inline_shape_mismatch():
content = """
arr: !core/ndarray-1.0.0
data: [1, 2, 3]
shape: [2]
"""
buff = helpers.yaml_to_asdf(content)
with pytest.raises(ValueError):
with asdf.AsdfFile.open(buff) as ff:
pass
def test_mask_arbitrary():
content = """
arr: !core/ndarray-1.0.0
data: [[1, 2, 3, 1234], [5, 6, 7, 8]]
mask: 1234
"""
buff = helpers.yaml_to_asdf(content)
with asdf.open(buff) as ff:
assert_array_equal(
ff.tree['arr'].mask,
[[False, False, False, True], [False, False, False, False]])
def test_mask_nan():
content = """
arr: !core/ndarray-1.0.0
data: [[1, 2, 3, .NaN], [5, 6, 7, 8]]
mask: .NaN
"""
buff = helpers.yaml_to_asdf(content)
with asdf.AsdfFile.open(buff) as ff:
assert_array_equal(
ff.tree['arr'].mask,
[[False, False, False, True], [False, False, False, False]])
def test_mask_datatype(tmpdir):
content = """
arr: !core/ndarray-1.0.0
data: [1, 2, 3]
dtype: int32
mask: !core/ndarray-1.0.0
data: [true, true, false]
"""
buff = helpers.yaml_to_asdf(content)
with asdf.open(buff) as ff:
pass
def test_mask_nan():
content = """
arr: !core/ndarray-1.0.0
data: [[1, 2, 3, .NaN], [5, 6, 7, 8]]
mask: .NaN
"""
buff = helpers.yaml_to_asdf(content)
with asdf.open(buff) as ff:
assert_array_equal(
ff.tree['arr'].mask,
[[False, False, False, True], [False, False, False, False]])
def test_mask_datatype(tmpdir):
content = """
arr: !core/ndarray-1.0.0
data: [1, 2, 3]
dtype: int32
mask: !core/ndarray-1.0.0
data: [true, true, false]
"""
buff = helpers.yaml_to_asdf(content)
with asdf.AsdfFile.open(buff) as ff:
pass
def test_mask_arbitrary():
content = """
arr: !core/ndarray-1.0.0
data: [[1, 2, 3, 1234], [5, 6, 7, 8]]
mask: 1234
"""
buff = helpers.yaml_to_asdf(content)
with asdf.AsdfFile.open(buff) as ff:
assert_array_equal(
ff.tree['arr'].mask,
[[False, False, False, True], [False, False, False, False]])
def runtest(self):
from asdf import AsdfFile, block, util
from asdf.tests import helpers
name, version = parse_schema_filename(self.filename)
if should_skip(name, version):
return
standard_version = self._find_standard_version(name, version)
# Make sure that the examples in the schema files (and thus the
# ASDF standard document) are valid.
buff = helpers.yaml_to_asdf(
'example: ' + self.example.strip(), standard_version=standard_version)
ff = AsdfFile(
uri=util.filepath_to_url(os.path.abspath(self.filename)),
ignore_unrecognized_tag=self.ignore_unrecognized_tag,
ignore_version_mismatch=self.ignore_version_mismatch,
)
# Fake an external file
ff2 = AsdfFile({'data': np.empty((1024*1024*8), dtype=np.uint8)})
ff._external_asdf_by_uri[
util.filepath_to_url(
os.path.abspath(
os.path.join(
os.path.dirname(self.filename), 'external.asdf')))] = ff2
# Add some dummy blocks so that the ndarray examples work
for i in range(3):
b = block.Block(np.zeros((1024*1024*8), dtype=np.uint8))
b._used = True
ff.blocks.add(b)
b._array_storage = "streamed"
try:
with pytest.warns(None) as w:
ff._open_impl(ff, buff, mode='rw')
# Do not tolerate any warnings that occur during schema validation
assert len(w) == 0, helpers.display_warnings(w)
except Exception:
print("From file:", self.filename)
raise
# Just test we can write it out. A roundtrip test
# wouldn't always yield the correct result, so those have
# to be covered by "real" unit tests.
if b'external.asdf' not in buff.getvalue():
buff = io.BytesIO()
ff.write_to(buff)
def test_invalid_mask_datatype(tmpdir):
content = """
arr: !core/ndarray-1.0.0
data: [1, 2, 3]
dtype: int32
mask: !core/ndarray-1.0.0
data: ['a', 'b', 'c']
"""
buff = helpers.yaml_to_asdf(content)
with pytest.raises(jsonschema.ValidationError):
with asdf.open(buff) as ff:
pass
def test_inline_structured():
content = """
arr: !core/ndarray-1.0.0
datatype: [['ascii', 4], uint16, uint16, ['ascii', 4]]
data: [[M110, 110, 205, And],
[ M31, 31, 224, And],
[ M32, 32, 221, And],
[M103, 103, 581, Cas]]"""
buff = helpers.yaml_to_asdf(content)
with asdf.open(buff) as ff:
assert ff.tree['arr']['f1'].dtype.char == 'H'
def test_copy_inline():
yaml = """
x0: !core/ndarray-1.0.0
data: [-1.0, 1.0]
"""
buff = helpers.yaml_to_asdf(yaml)
with asdf.AsdfFile.open(buff) as infile:
with asdf.AsdfFile() as f:
f.tree['a'] = infile.tree['x0']
f.tree['b'] = f.tree['a']
f.write_to(io.BytesIO())
def test_inline_structured():
content = """
arr: !core/ndarray-1.0.0
datatype: [['ascii', 4], uint16, uint16, ['ascii', 4]]
data: [[M110, 110, 205, And],
[ M31, 31, 224, And],
[ M32, 32, 221, And],
[M103, 103, 581, Cas]]"""
buff = helpers.yaml_to_asdf(content)
with asdf.AsdfFile.open(buff) as ff:
assert ff.tree['arr']['f1'].dtype.char == 'H'
def test_invalid_mask_datatype(tmpdir):
content = """
arr: !core/ndarray-1.0.0
data: [1, 2, 3]
dtype: int32
mask: !core/ndarray-1.0.0
data: ['a', 'b', 'c']
"""
buff = helpers.yaml_to_asdf(content)
with pytest.raises(jsonschema.ValidationError):
with asdf.AsdfFile.open(buff) as ff:
pass
def test_copy_inline():
yaml = """
x0: !core/ndarray-1.0.0
data: [-1.0, 1.0]
"""
buff = helpers.yaml_to_asdf(yaml)
with asdf.open(buff) as infile:
with asdf.AsdfFile() as f:
f.tree['a'] = infile.tree['x0']
f.tree['b'] = f.tree['a']
f.write_to(io.BytesIO())
def runtest(self):
from asdf import AsdfFile, block, util
from asdf.tests import helpers
from .extension import TestExtension
name, version = parse_schema_filename(self.filename)
if should_skip(name, version):
return
standard_version = self._find_standard_version(name, version)
# Make sure that the examples in the schema files (and thus the
# ASDF standard document) are valid.
buff = helpers.yaml_to_asdf(
'example: ' + self.example.strip(), standard_version=standard_version)
ff = AsdfFile(
uri=util.filepath_to_url(os.path.abspath(self.filename)),
extensions=TestExtension())
# Fake an external file
ff2 = AsdfFile({'data': np.empty((1024*1024*8), dtype=np.uint8)})
ff._external_asdf_by_uri[
util.filepath_to_url(
os.path.abspath(
os.path.join(
os.path.dirname(self.filename), 'external.asdf')))] = ff2
# Add some dummy blocks so that the ndarray examples work
for i in range(3):
b = block.Block(np.zeros((1024*1024*8), dtype=np.uint8))
b._used = True
ff.blocks.add(b)
b._array_storage = "streamed"
try:
with pytest.warns(None) as w:
import warnings
ff._open_impl(ff, buff, mode='rw')
# Do not tolerate any warnings that occur during schema validation
assert len(w) == 0, helpers.display_warnings(w)
except Exception:
print("From file:", self.filename)
raise
# Just test we can write it out. A roundtrip test
# wouldn't always yield the correct result, so those have
# to be covered by "real" unit tests.
if b'external.asdf' not in buff.getvalue():
buff = io.BytesIO()
ff.write_to(buff)
def test_unit():
yaml = """
unit: !unit/unit-1.0.0 "2.1798721 10-18kg m2 s-2"
"""
buff = helpers.yaml_to_asdf(yaml)
with asdf.AsdfFile.open(buff) as ff:
assert ff.tree['unit'].is_equivalent(u.Ry)
buff2 = io.BytesIO()
ff.write_to(buff2)
buff2.seek(0)
with asdf.AsdfFile.open(buff2) as ff:
assert ff.tree['unit'].is_equivalent(u.Ry)
def test_old_history(tmpdir):
"""Make sure that old versions of the history format are still accepted"""
yaml = """
history:
- !core/history_entry-1.0.0
description: "Here's a test of old history entries"
software: !core/software-1.0.0
name: foo
version: 1.2.3
"""
buff = yaml_to_asdf(yaml)
with asdf.open(buff) as af:
assert len(af.tree['history']) == 1
def test_unit():
yaml = """
unit: !unit/unit-1.0.0 "2.1798721 10-18kg m2 s-2"
"""
buff = helpers.yaml_to_asdf(yaml)
with asdf.open(buff) as ff:
assert ff.tree['unit'].is_equivalent(u.Ry)
buff2 = io.BytesIO()
ff.write_to(buff2)
buff2.seek(0)
with asdf.open(buff2) as ff:
assert ff.tree['unit'].is_equivalent(u.Ry)
def test_read_large_literal():
value = 1 << 64
yaml = """integer: {}""".format(value)
buff = helpers.yaml_to_asdf(yaml)
with pytest.warns(UserWarning) as w:
with asdf.open(buff) as af:
assert af['integer'] == value
# We get two warnings: one for validation time, and one when defaults
# are filled. It seems like we could improve this architecture, though...
assert len(w) == 2
assert str(w[0].message).startswith('Invalid integer literal value')
assert str(w[1].message).startswith('Invalid integer literal value')
def test_tagging_scalars():
yaml = """
unit: !unit/unit-1.0.0
m
not_unit:
m
"""
from astropy import units as u
buff = helpers.yaml_to_asdf(yaml)
with asdf.AsdfFile.open(buff) as ff:
assert isinstance(ff.tree['unit'], u.UnitBase)
assert not isinstance(ff.tree['not_unit'], u.UnitBase)
assert isinstance(ff.tree['not_unit'], str)
assert ff.tree == {'unit': u.m, 'not_unit': 'm'}
def test_mismatched_columns():
yaml = """
table: !core/table
columns:
- !core/column
data: !core/ndarray
data: [0, 1, 2]
name: a
- !core/column
data: !core/ndarray
data: [0, 1, 2, 3]
name: b
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.raises(ValueError):
with asdf.AsdfFile.open(buff) as ff:
pass
def test_tag_reference_validation():
class DefaultTypeExtension(CustomExtension):
@property
def types(self):
return [TagReferenceType]
yaml = """
custom: !<tag:nowhere.org:custom/tag_reference-1.0.0>
name:
"Something"
things: !core/ndarray-1.0.0
data: [1, 2, 3]
"""
buff = helpers.yaml_to_asdf(yaml)
with asdf.AsdfFile.open(buff, extensions=[DefaultTypeExtension()]) as ff:
custom = ff.tree['custom']
assert custom['name'] == "Something"
assert_array_equal(custom['things'], [1, 2, 3])
def test_tagging_scalars():
yaml = """
unit: !unit/unit-1.0.0
m
not_unit:
m
"""
from astropy import units as u
buff = helpers.yaml_to_asdf(yaml)
with asdf.AsdfFile.open(buff) as ff:
assert isinstance(ff.tree['unit'], u.UnitBase)
assert not isinstance(ff.tree['not_unit'], u.UnitBase)
assert isinstance(ff.tree['not_unit'], str)
assert ff.tree == {
'unit': u.m,
'not_unit': 'm'
}
def test_missing_extension_warning():
yaml = """
history:
extensions:
- !core/extension_metadata-1.0.0
extension_class: foo.bar.FooBar
software: !core/software-1.0.0
name: foo
version: 1.2.3
"""
buff = yaml_to_asdf(yaml)
with pytest.warns(None) as warnings:
with asdf.open(buff) as af:
pass
assert len(warnings) == 1, display_warnings(warnings)
assert str(warnings[0].message).startswith(
"File was created with extension 'foo.bar.FooBar'")
def test_foreign_tag_reference_validation():
class ForeignTagReferenceType(asdftypes.CustomType):
name = 'foreign_tag_reference'
organization = 'nowhere.org'
version = (1, 0, 0)
standard = 'custom'
@classmethod
def from_tree(cls, tree, ctx):
node = {}
node['a'] = yamlutil.tagged_tree_to_custom_tree(tree['a'], ctx)
node['b'] = yamlutil.tagged_tree_to_custom_tree(tree['b'], ctx)
return node
class ForeignTypeExtension(CustomExtension):
@property
def types(self):
return [TagReferenceType, ForeignTagReferenceType]
yaml = """
custom: !<tag:nowhere.org:custom/foreign_tag_reference-1.0.0>
a: !<tag:nowhere.org:custom/tag_reference-1.0.0>
name:
"Something"
things: !core/ndarray-1.0.0
data: [1, 2, 3]
b: !<tag:nowhere.org:custom/tag_reference-1.0.0>
name:
"Anything"
things: !core/ndarray-1.0.0
data: [4, 5, 6]
"""
buff = helpers.yaml_to_asdf(yaml)
with asdf.AsdfFile.open(buff, extensions=ForeignTypeExtension()) as ff:
a = ff.tree['custom']['a']
b = ff.tree['custom']['b']
assert a['name'] == 'Something'
assert_array_equal(a['things'], [1, 2, 3])
assert b['name'] == 'Anything'
assert_array_equal(b['things'], [4, 5, 6])
def test_strict_extension_check():
yaml = """
history:
extensions:
- !core/extension_metadata-1.0.0
extension_class: foo.bar.FooBar
software: !core/software-1.0.0
name: foo
version: 1.2.3
"""
buff = yaml_to_asdf(yaml)
with pytest.raises(RuntimeError):
with asdf.open(buff, strict_extension_check=True) as af:
pass
# Make sure to test for incompatibility with ignore_missing_extensions
with pytest.raises(ValueError):
with asdf.open(buff, strict_extension_check=True, ignore_missing_extensions=True) as af:
pass
def test_mismatched_columns():
yaml = """
table: !<tag:astropy.org:astropy/table/table-1.0.0>
columns:
- !core/column-1.0.0
data: !core/ndarray-1.0.0
data: [0, 1, 2]
name: a
- !core/column-1.0.0
data: !core/ndarray-1.0.0
data: [0, 1, 2, 3]
name: b
colnames: [a, b]
"""
buff = helpers.yaml_to_asdf(yaml)
with pytest.raises(ValueError) as err:
with asdf.open(buff) as ff:
pass
assert 'Inconsistent data column lengths' in str(err)
def test_old_history(tmpdir):
"""Make sure that old versions of the history format are still accepted"""
yaml = """
history:
- !core/history_entry-1.0.0
description: "Here's a test of old history entries"
software: !core/software-1.0.0
name: foo
version: 1.2.3
"""
buff = yaml_to_asdf(yaml)
with asdf.open(buff) as af:
assert len(af.tree['history']) == 1
# Test the history entry retrieval API
entries = af.get_history_entries()
assert len(entries) == 1
assert isinstance(entries, list)
assert isinstance(entries[0], HistoryEntry)
assert entries[0]['description'] == "Here's a test of old history entries"
assert entries[0]['software']['name'] == 'foo'
def test_ndim_validation(tmpdir):
content = """
obj: !<tag:nowhere.org:custom/ndim-1.0.0>
a: !core/ndarray-1.0.0
data: [1, 2, 3]
"""
buff = helpers.yaml_to_asdf(content)
with pytest.raises(jsonschema.ValidationError):
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
content = """
obj: !<tag:nowhere.org:custom/ndim-1.0.0>
a: !core/ndarray-1.0.0
data: [[1, 2, 3]]
"""
buff = helpers.yaml_to_asdf(content)
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
content = """
obj: !<tag:nowhere.org:custom/ndim-1.0.0>
a: !core/ndarray-1.0.0
shape: [1, 3]
data: [[1, 2, 3]]
"""
buff = helpers.yaml_to_asdf(content)
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
content = """
obj: !<tag:nowhere.org:custom/ndim-1.0.0>
b: !core/ndarray-1.0.0
data: [1, 2, 3]
"""
buff = helpers.yaml_to_asdf(content)
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
content = """
obj: !<tag:nowhere.org:custom/ndim-1.0.0>
b: !core/ndarray-1.0.0
data: [[1, 2, 3]]
"""
buff = helpers.yaml_to_asdf(content)
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
content = """
obj: !<tag:nowhere.org:custom/ndim-1.0.0>
b: !core/ndarray-1.0.0
data: [[[1, 2, 3]]]
"""
buff = helpers.yaml_to_asdf(content)
with pytest.raises(jsonschema.ValidationError):
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
def test_structured_datatype_validation(tmpdir):
content = """
obj: !<tag:nowhere.org:custom/datatype-1.0.0>
c: !core/ndarray-1.0.0
data: [[1, 'a'], [2, 'b'], [3, 'c']]
datatype:
- name: a
datatype: int8
- name: b
datatype: ['ascii', 8]
"""
buff = helpers.yaml_to_asdf(content)
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
content = """
obj: !<tag:nowhere.org:custom/datatype-1.0.0>
c: !core/ndarray-1.0.0
data: [[1, 'a'], [2, 'b'], [3, 'c']]
datatype:
- name: a
datatype: int64
- name: b
datatype: ['ascii', 8]
"""
buff = helpers.yaml_to_asdf(content)
with pytest.raises(jsonschema.ValidationError):
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
content = """
obj: !<tag:nowhere.org:custom/datatype-1.0.0>
c: !core/ndarray-1.0.0
data: [[1, 'a', 0], [2, 'b', 1], [3, 'c', 2]]
datatype:
- name: a
datatype: int8
- name: b
datatype: ['ascii', 8]
- name: c
datatype: float64
"""
buff = helpers.yaml_to_asdf(content)
with pytest.raises(jsonschema.ValidationError):
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
content = """
obj: !<tag:nowhere.org:custom/datatype-1.0.0>
c: !core/ndarray-1.0.0
data: [1, 2, 3]
"""
buff = helpers.yaml_to_asdf(content)
with pytest.raises(jsonschema.ValidationError):
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
content = """
obj: !<tag:nowhere.org:custom/datatype-1.0.0>
d: !core/ndarray-1.0.0
data: [[1, 'a'], [2, 'b'], [3, 'c']]
datatype:
- name: a
datatype: int8
- name: b
datatype: ['ascii', 8]
"""
buff = helpers.yaml_to_asdf(content)
with pytest.raises(jsonschema.ValidationError):
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
content = """
obj: !<tag:nowhere.org:custom/datatype-1.0.0>
d: !core/ndarray-1.0.0
data: [[1, 'a'], [2, 'b'], [3, 'c']]
datatype:
- name: a
datatype: int16
- name: b
datatype: ['ascii', 16]
"""
buff = helpers.yaml_to_asdf(content)
with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff:
pass
