def test_backwards_compat_gcrs(): obsgeoloc = ( 3.0856775814671916e+16, 9.257032744401574e+16, 6.1713551629343834e+19 ) obsgeovel = (2.0, 1.0, 8.0) old_frame_yaml = """ frames: - !wcs/celestial_frame-1.0.0 axes_names: [lon, lat] name: CelestialFrame reference_frame: type: GCRS obsgeoloc: - [%f, %f, %f] - !unit/unit-1.0.0 m obsgeovel: - [%f, %f, %f] - !unit/unit-1.0.0 m s-1 obstime: !time/time-1.0.0 2010-01-01 00:00:00.000 unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] """ % (obsgeovel + obsgeoloc) new_frame_yaml = """ frames: - !wcs/celestial_frame-1.1.0 axes_names: [lon, lat] name: CelestialFrame reference_frame: type: GCRS obsgeoloc: - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} obsgeovel: - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} obstime: !time/time-1.1.0 2010-01-01 00:00:00.000 unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] """ % (obsgeovel + obsgeoloc) old_buff = helpers.yaml_to_asdf(old_frame_yaml) old_asdf = AsdfFile.open(old_buff) old_frame = old_asdf.tree['frames'][0] old_loc = old_frame.reference_frame.obsgeoloc old_vel = old_frame.reference_frame.obsgeovel new_buff = helpers.yaml_to_asdf(new_frame_yaml) new_asdf = AsdfFile.open(new_buff) new_frame = new_asdf.tree['frames'][0] new_loc = new_frame.reference_frame.obsgeoloc new_vel = new_frame.reference_frame.obsgeovel assert (old_loc.x == new_loc.x and old_loc.y == new_loc.y and old_loc.z == new_loc.z) assert (old_vel.x == new_vel.x and old_vel.y == new_vel.y and old_vel.z == new_vel.z)
def test_backwards_compat_gcrs(): obsgeoloc = ( 3.0856775814671916e+16, 9.257032744401574e+16, 6.1713551629343834e+19 ) obsgeovel = (2.0, 1.0, 8.0) old_frame_yaml = """ frames: - !wcs/celestial_frame-1.0.0 axes_names: [lon, lat] name: CelestialFrame reference_frame: type: GCRS obsgeoloc: - [%f, %f, %f] - !unit/unit-1.0.0 m obsgeovel: - [%f, %f, %f] - !unit/unit-1.0.0 m s-1 obstime: !time/time-1.0.0 2010-01-01 00:00:00.000 unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] """ % (obsgeovel + obsgeoloc) new_frame_yaml = """ frames: - !wcs/celestial_frame-1.1.0 axes_names: [lon, lat] name: CelestialFrame reference_frame: type: GCRS obsgeoloc: - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} obsgeovel: - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} obstime: !time/time-1.1.0 2010-01-01 00:00:00.000 unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] """ % (obsgeovel + obsgeoloc) old_buff = helpers.yaml_to_asdf(old_frame_yaml) old_asdf = AsdfFile.open(old_buff) old_frame = old_asdf.tree['frames'][0] old_loc = old_frame.reference_frame.obsgeoloc old_vel = old_frame.reference_frame.obsgeovel new_buff = helpers.yaml_to_asdf(new_frame_yaml) new_asdf = AsdfFile.open(new_buff) new_frame = new_asdf.tree['frames'][0] new_loc = new_frame.reference_frame.obsgeoloc new_vel = new_frame.reference_frame.obsgeovel assert (old_loc.x == new_loc.x and old_loc.y == new_loc.y and old_loc.z == new_loc.z) assert (old_vel.x == new_vel.x and old_vel.y == new_vel.y and old_vel.z == new_vel.z)
def test_nonexistent_tag(tmpdir): """ This tests the case where a node is tagged with a type that apparently comes from an extension that is known, but the type itself can't be found. This could occur when a more recent version of an installed package provides the new type, but an older version of the package is installed. ASDF should still be able to open the file in this case, but it won't be able to restore the type. The bug that prompted this test results from attempting to load a schema file that doesn't exist, which is why this test belongs in this file. """ # This shouldn't ever happen, but it's a useful test case yaml = """ a: !core/doesnt_exist-1.0.0 hello """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(AsdfWarning, match="Unable to locate schema file"): with asdf.open(buff) as af: assert str(af['a']) == 'hello' # This is a more realistic case since we're using an external extension yaml = """ a: !<tag:nowhere.org:custom/doesnt_exist-1.0.0> hello """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(AsdfWarning, match="Unable to locate schema file"): with asdf.open(buff, extensions=CustomExtension()) as af: assert str(af['a']) == 'hello'
def test_extension_version_warning(): yaml = """ history: extensions: - !core/extension_metadata-1.0.0 extension_class: asdf.extension.BuiltinExtension software: !core/software-1.0.0 name: asdf version: 100.0.3 """ buff = yaml_to_asdf(yaml) with pytest.warns( AsdfWarning, match= "File was created with extension class 'asdf.extension.BuiltinExtension'" ): with asdf.open(buff): pass buff.seek(0) # Make sure suppressing the warning works too with assert_no_warnings(): with asdf.open(buff, ignore_missing_extensions=True): pass
def make_complex_asdf(string): yaml = """ a: !core/complex-1.0.0 {} """.format(string) return helpers.yaml_to_asdf(yaml)
def test_extension_version_warning(): yaml = """ history: extensions: - !core/extension_metadata-1.0.0 extension_class: asdf.extension.BuiltinExtension software: !core/software-1.0.0 name: asdf version: 100.0.3 """ buff = yaml_to_asdf(yaml) with pytest.warns(None) as warnings: with asdf.open(buff) as af: pass assert len(warnings) == 1, display_warnings(warnings) assert str(warnings[0].message).startswith( "File was created with extension 'asdf.extension.BuiltinExtension' " "from package asdf-100.0.3") buff.seek(0) # Make sure suppressing the warning works too with pytest.warns(None) as warnings: with asdf.open(buff, ignore_missing_extensions=True) as af: pass assert len(warnings) == 0, display_warnings(warnings)
def test_fill_and_remove_defaults(): class DefaultType(dict, asdftypes.CustomType): name = 'default' organization = 'nowhere.org' version = (1, 0, 0) standard = 'custom' class DefaultTypeExtension(CustomExtension): @property def types(self): return [DefaultType] yaml = """ custom: !<tag:nowhere.org:custom/default-1.0.0> b: {} """ buff = helpers.yaml_to_asdf(yaml) with asdf.AsdfFile.open(buff, extensions=[DefaultTypeExtension()]) as ff: assert 'a' in ff.tree['custom'] assert ff.tree['custom']['a'] == 42 assert ff.tree['custom']['b']['c'] == 82 buff.seek(0) with asdf.AsdfFile.open(buff, extensions=[DefaultTypeExtension()], do_not_fill_defaults=True) as ff: assert 'a' not in ff.tree['custom'] assert 'c' not in ff.tree['custom']['b'] ff.fill_defaults() assert 'a' in ff.tree['custom'] assert ff.tree['custom']['a'] == 42 assert 'c' in ff.tree['custom']['b'] assert ff.tree['custom']['b']['c'] == 82 ff.remove_defaults() assert 'a' not in ff.tree['custom'] assert 'c' not in ff.tree['custom']['b']
def test_type_missing_dependencies(): class MissingType(asdftypes.AsdfType): name = 'missing' organization = 'nowhere.org' version = (1, 1, 0) standard = 'custom' types = ['asdfghjkl12345.foo'] requires = ["ASDFGHJKL12345"] class DefaultTypeExtension(CustomExtension): @property def types(self): return [MissingType] yaml = """ custom: !<tag:nowhere.org:custom/missing-1.1.0> b: {foo: 42} """ buff = helpers.yaml_to_asdf(yaml) with catch_warnings() as w: with asdf.AsdfFile.open(buff, extensions=[DefaultTypeExtension()]) as ff: assert ff.tree['custom']['b']['foo'] == 42 assert len(w) == 1
def test_type_missing_dependencies(): pytest.importorskip('astropy', '3.0.0') class MissingType(types.CustomType): name = 'missing' organization = 'nowhere.org' version = (1, 1, 0) standard = 'custom' types = ['asdfghjkl12345.foo'] requires = ["ASDFGHJKL12345"] class DefaultTypeExtension(CustomExtension): @property def types(self): return [MissingType] yaml = """ custom: !<tag:nowhere.org:custom/missing-1.1.0> b: {foo: 42} """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns( AsdfConversionWarning, match="Failed to convert tag:nowhere.org:custom/missing-1.1.0"): with asdf.open(buff, extensions=[DefaultTypeExtension()]) as ff: assert ff.tree['custom']['b']['foo'] == 42
def run_schema_example_test(organization, standard, name, version, check_func=None): import asdf from asdf.tests import helpers from asdf.types import format_tag from asdf.schema import load_schema tag = format_tag(organization, standard, version, name) uri = asdf.resolver.default_tag_to_url_mapping(tag) r = asdf.AsdfFile().resolver examples = [] schema = load_schema(uri, resolver=r) for node in asdf.treeutil.iter_tree(schema): if (isinstance(node, dict) and 'examples' in node and isinstance(node['examples'], list)): for desc, example in node['examples']: examples.append(example) for example in examples: buff = helpers.yaml_to_asdf('example: ' + example.strip()) ff = asdf.AsdfFile(uri=uri) # Add some dummy blocks so that the ndarray examples work for i in range(3): b = asdf.block.Block(np.zeros((1024*1024*8), dtype=np.uint8)) b._used = True ff.blocks.add(b) ff._open_impl(ff, buff, mode='r') if check_func: check_func(ff)
def run_schema_example_test(organization, standard, name, version, check_func=None): import asdf from asdf.tests import helpers from asdf.types import format_tag from asdf.schema import load_schema tag = format_tag(organization, standard, version, name) uri = asdf.extension.default_extensions.extension_list.tag_mapping(tag) r = asdf.extension.get_default_resolver() examples = [] schema = load_schema(uri, resolver=r) for node in asdf.treeutil.iter_tree(schema): if (isinstance(node, dict) and 'examples' in node and isinstance(node['examples'], list)): for desc, example in node['examples']: examples.append(example) for example in examples: buff = helpers.yaml_to_asdf('example: ' + example.strip()) ff = asdf.AsdfFile(uri=uri) # Add some dummy blocks so that the ndarray examples work for i in range(3): b = asdf.block.Block(np.zeros((1024 * 1024 * 8), dtype=np.uint8)) b._used = True ff.blocks.add(b) ff._open_impl(ff, buff, mode='r') if check_func: check_func(ff)
def test_read_large_literal(value): yaml = f"integer: {value}" buff = helpers.yaml_to_asdf(yaml) with pytest.warns(AsdfWarning, match="Invalid integer literal value"): with asdf.open(buff) as af: assert af['integer'] == value yaml = f"{value}: foo" buff = helpers.yaml_to_asdf(yaml) with pytest.warns(AsdfWarning, match="Invalid integer literal value"): with asdf.open(buff) as af: assert af[value] == "foo"
def test_deserialize_compound_user_inverse(tmpdir): """ Confirm that we are able to correctly reconstruct a compound model with a user inverse set on one of its component models. Due to code in TransformType that facilitates circular inverses, the user inverse of the component model is not available at the time that the CompoundModel is constructed. """ yaml = """ model: !transform/concatenate-1.2.0 forward: - !transform/shift-1.2.0 inverse: !transform/shift-1.2.0 {offset: 5.0} offset: -10.0 - !transform/shift-1.2.0 {offset: -20.0} """ buff = helpers.yaml_to_asdf(yaml) with asdf.open(buff) as af: model = af["model"] assert model.has_inverse() assert model.inverse(-5, -20) == (0, 0)
def run_schema_example_test(organization, standard, name, version, check_func=None): import asdf from asdf.tests import helpers from asdf.types import format_tag from asdf.resolver import default_resolver tag = format_tag(organization, standard, version, name) schema_path = urllib.parse.urlparse(default_resolver(tag)).path with open(schema_path, 'rb') as ff: schema = yaml.load(ff) examples = [] for node in asdf.treeutil.iter_tree(schema): if (isinstance(node, dict) and 'examples' in node and isinstance(node['examples'], list)): for desc, example in node['examples']: examples.append(example) for example in examples: buff = helpers.yaml_to_asdf('example: ' + example.strip()) ff = asdf.AsdfFile(uri=schema_path) # Add some dummy blocks so that the ndarray examples work for i in range(3): b = asdf.block.Block(np.zeros((1024*1024*8), dtype=np.uint8)) b._used = True ff.blocks.add(b) ff._open_impl(ff, buff, mode='r') if check_func: check_func(ff)
def test_fill_and_remove_defaults(): class DefaultType(dict, types.CustomType): name = 'default' organization = 'nowhere.org' version = (1, 0, 0) standard = 'custom' class DefaultTypeExtension(CustomExtension): @property def types(self): return [DefaultType] yaml = """ custom: !<tag:nowhere.org:custom/default-1.0.0> b: {} """ buff = helpers.yaml_to_asdf(yaml) with asdf.open(buff, extensions=[DefaultTypeExtension()]) as ff: assert 'a' in ff.tree['custom'] assert ff.tree['custom']['a'] == 42 assert ff.tree['custom']['b']['c'] == 82 buff.seek(0) with asdf.open(buff, extensions=[DefaultTypeExtension()], do_not_fill_defaults=True) as ff: assert 'a' not in ff.tree['custom'] assert 'c' not in ff.tree['custom']['b'] ff.fill_defaults() assert 'a' in ff.tree['custom'] assert ff.tree['custom']['a'] == 42 assert 'c' in ff.tree['custom']['b'] assert ff.tree['custom']['b']['c'] == 82 ff.remove_defaults() assert 'a' not in ff.tree['custom'] assert 'c' not in ff.tree['custom']['b']
def test_type_missing_dependencies(): class MissingType(asdftypes.CustomType): name = 'missing' organization = 'nowhere.org' version = (1, 1, 0) standard = 'custom' types = ['asdfghjkl12345.foo'] requires = ["ASDFGHJKL12345"] class DefaultTypeExtension(CustomExtension): @property def types(self): return [MissingType] yaml = """ custom: !<tag:nowhere.org:custom/missing-1.1.0> b: {foo: 42} """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(None) as w: with asdf.AsdfFile.open(buff, extensions=[DefaultTypeExtension()]) as ff: assert ff.tree['custom']['b']['foo'] == 42 assert len(w) == 1
def make_complex_asdf(string): yaml = """ a: !core/complex-1.0.0 {} """.format(string) return helpers.yaml_to_asdf(yaml)
def test_extension_version_warning(): yaml = """ history: extensions: - !core/extension_metadata-1.0.0 extension_class: asdf.extension.BuiltinExtension software: !core/software-1.0.0 name: asdf version: 100.0.3 """ buff = yaml_to_asdf(yaml) with pytest.warns(None) as warnings: with asdf.open(buff) as af: pass assert len(warnings) == 1, display_warnings(warnings) assert str(warnings[0].message).startswith( "File was created with extension 'asdf.extension.BuiltinExtension' " "from package asdf-100.0.3") buff.seek(0) # Make sure suppressing the warning works too with pytest.warns(None) as warnings: with asdf.open(buff, ignore_missing_extensions=True) as af: pass assert len(warnings) == 0, display_warnings(warnings)
def test_old_format(recwarn): warnings.simplefilter("always") content = helpers.yaml_to_asdf(""" extension: !core/extension_metadata-1.0.0 extension_class: asdf.extension.BuiltinExtension software: {name: asdf, version: 2.5.0} """) with asdf.open(content) as af: extension = af.tree["extension"] assert isinstance(extension, asdf.tags.core.ExtensionMetadata) assert extension.extension_class == "asdf.extension.BuiltinExtension" assert isinstance(extension.package, asdf.tags.core.Software) assert extension.package["name"] == "asdf" assert extension.package["version"] == "2.5.0" software = extension.software assert isinstance(software, asdf.tags.core.Software) assert software["name"] == "asdf" assert software["version"] == "2.5.0" # One warning for opening a file in the old format, another for # accessing the 'software' property: assert len(recwarn) == 2 assert recwarn.pop(asdf.exceptions.AsdfDeprecationWarning) assert recwarn.pop(asdf.exceptions.AsdfDeprecationWarning)
def test_nonexistent_tag(tmpdir): """ This tests the case where a node is tagged with a type that apparently comes from an extension that is known, but the type itself can't be found. This could occur when a more recent version of an installed package provides the new type, but an older version of the package is installed. ASDF should still be able to open the file in this case, but it won't be able to restore the type. The bug that prompted this test results from attempting to load a schema file that doesn't exist, which is why this test belongs in this file. """ # This shouldn't ever happen, but it's a useful test case yaml = """ a: !core/doesnt_exist-1.0.0 hello """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(None) as w: with asdf.open(buff) as af: assert str(af['a']) == 'hello' # Currently there are 3 warnings since one occurs on each of the # validation passes. It would be good to consolidate these # eventually assert len(w) == 3, helpers.display_warnings(w) assert str(w[0].message).startswith("Unable to locate schema file") assert str(w[1].message).startswith("Unable to locate schema file") assert str(w[2].message).startswith(af['a']._tag) # This is a more realistic case since we're using an external extension yaml = """ a: !<tag:nowhere.org:custom/doesnt_exist-1.0.0> hello """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(None) as w: with asdf.open(buff, extensions=CustomExtension()) as af: assert str(af['a']) == 'hello' assert len(w) == 3, helpers.display_warnings(w) assert str(w[0].message).startswith("Unable to locate schema file") assert str(w[1].message).startswith("Unable to locate schema file") assert str(w[2].message).startswith(af['a']._tag)
def test_nonexistent_tag(tmpdir): """ This tests the case where a node is tagged with a type that apparently comes from an extension that is known, but the type itself can't be found. This could occur when a more recent version of an installed package provides the new type, but an older version of the package is installed. ASDF should still be able to open the file in this case, but it won't be able to restore the type. The bug that prompted this test results from attempting to load a schema file that doesn't exist, which is why this test belongs in this file. """ # This shouldn't ever happen, but it's a useful test case yaml = """ a: !core/doesnt_exist-1.0.0 hello """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(None) as w: with asdf.open(buff) as af: assert str(af['a']) == 'hello' # Currently there are 3 warnings since one occurs on each of the # validation passes. It would be good to consolidate these # eventually assert len(w) == 3, helpers.display_warnings(w) assert str(w[0].message).startswith("Unable to locate schema file") assert str(w[1].message).startswith("Unable to locate schema file") assert str(w[2].message).startswith(af['a']._tag) # This is a more realistic case since we're using an external extension yaml = """ a: !<tag:nowhere.org:custom/doesnt_exist-1.0.0> hello """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(None) as w: with asdf.open(buff, extensions=CustomExtension()) as af: assert str(af['a']) == 'hello' assert len(w) == 3, helpers.display_warnings(w) assert str(w[0].message).startswith("Unable to locate schema file") assert str(w[1].message).startswith("Unable to locate schema file") assert str(w[2].message).startswith(af['a']._tag)
def test_invalid_complex(): yaml = """ a: !core/complex-1.0.0 3 + 4i """ buff = helpers.yaml_to_asdf(yaml) with pytest.raises(ValueError): with asdf.AsdfFile.open(buff): pass
def roundtrip_quantity(yaml, quantity): buff = helpers.yaml_to_asdf(yaml) with asdf.AsdfFile.open(buff) as ff: assert (ff.tree['quantity'] == quantity).all() buff2 = io.BytesIO() ff.write_to(buff2) buff2.seek(0) with asdf.AsdfFile.open(buff2) as ff: assert (ff.tree['quantity'] == quantity).all()
def test_read_large_literal(): value = 1 << 64 yaml = """integer: {}""".format(value) buff = helpers.yaml_to_asdf(yaml) with pytest.warns(AsdfWarning, match="Invalid integer literal value"): with asdf.open(buff) as af: assert af['integer'] == value
def roundtrip_quantity(yaml, quantity): buff = helpers.yaml_to_asdf(yaml) with asdf.AsdfFile.open(buff) as ff: assert (ff.tree['quantity'] == quantity).all() buff2 = io.BytesIO() ff.write_to(buff2) buff2.seek(0) with asdf.AsdfFile.open(buff2) as ff: assert (ff.tree['quantity'] == quantity).all()
def test_schema_example(filename, example): """Pytest to check validity of a specific example within schema file Parameters: ----------- filename : name of the schema file containing example to be tested example: string representing example This function is called with a range of parameters by pytest's 'parametrize' utility in order to account for all examples in all schema files. """ if not HAS_GWCS and re.search(r'frame-\d\.\d\.\d\.yaml', filename): return pytest.skip standard_version = _find_standard_version(filename) # Make sure that the examples in the schema files (and thus the # ASDF standard document) are valid. buff = helpers.yaml_to_asdf('example: ' + example.strip(), standard_version=standard_version) ff = asdf.AsdfFile(uri=util.filepath_to_url(os.path.abspath(filename)), extensions=TestExtension()) # Fake an external file ff2 = asdf.AsdfFile({'data': np.empty((1024 * 1024 * 8), dtype=np.uint8)}) ff._external_asdf_by_uri[util.filepath_to_url( os.path.abspath( os.path.join(os.path.dirname(filename), 'external.asdf')))] = ff2 # Add some dummy blocks so that the ndarray examples work for i in range(3): b = block.Block(np.zeros((1024 * 1024 * 8), dtype=np.uint8)) b._used = True ff.blocks.add(b) b._array_storage = "streamed" try: with catch_warnings() as w: ff._open_impl(ff, buff) # Do not tolerate any warnings that occur during schema validation, # other than a few that we expect to occur under certain circumstances _assert_warnings(w) except: print("From file:", filename) raise # Just test we can write it out. A roundtrip test # wouldn't always yield the correct result, so those have # to be covered by "real" unit tests. if b'external.asdf' not in buff.getvalue(): buff = io.BytesIO() ff.write_to(buff)
def test_invalid_nested(): class CustomType(str, asdftypes.CustomType): name = 'custom' organization = 'nowhere.org' version = (1, 0, 0) standard = 'custom' class CustomTypeExtension(CustomExtension): @property def types(self): return [CustomType] yaml = """ custom: !<tag:nowhere.org:custom/custom-1.0.0> foo """ buff = helpers.yaml_to_asdf(yaml) # This should cause a warning but not an error because without explicitly # providing an extension, our custom type will not be recognized and will # simply be converted to a raw type. with pytest.warns(None) as warning: with asdf.AsdfFile.open(buff): pass assert len(warning) == 1 buff.seek(0) with pytest.raises(ValidationError): with asdf.AsdfFile.open(buff, extensions=[CustomTypeExtension()]): pass # Make sure tags get validated inside of other tags that know # nothing about them. yaml = """ array: !core/ndarray-1.0.0 data: [0, 1, 2] custom: !<tag:nowhere.org:custom/custom-1.0.0> foo """ buff = helpers.yaml_to_asdf(yaml) with pytest.raises(ValidationError): with asdf.AsdfFile.open(buff, extensions=[CustomTypeExtension()]): pass
def test_invalid_nested(): class CustomType(str, asdftypes.AsdfType): name = 'custom' organization = 'nowhere.org' version = (1, 0, 0) standard = 'custom' class CustomTypeExtension(CustomExtension): @property def types(self): return [CustomType] yaml = """ custom: !<tag:nowhere.org:custom/custom-1.0.0> foo """ buff = helpers.yaml_to_asdf(yaml) # This should cause a warning but not an error because without explicitly # providing an extension, our custom type will not be recognized and will # simply be converted to a raw type. with catch_warnings() as warning: with asdf.AsdfFile.open(buff): pass assert len(warning) == 1 buff.seek(0) with pytest.raises(ValidationError): with asdf.AsdfFile.open(buff, extensions=[CustomTypeExtension()]): pass # Make sure tags get validated inside of other tags that know # nothing about them. yaml = """ array: !core/ndarray-1.0.0 data: [0, 1, 2] custom: !<tag:nowhere.org:custom/custom-1.0.0> foo """ buff = helpers.yaml_to_asdf(yaml) with pytest.raises(ValidationError): with asdf.AsdfFile.open(buff, extensions=[CustomTypeExtension()]): pass
def test_inline_shape_mismatch(): content = """ arr: !core/ndarray-1.0.0 data: [1, 2, 3] shape: [2] """ buff = helpers.yaml_to_asdf(content) with pytest.raises(ValueError): with asdf.open(buff) as ff: pass
def test_inline_shape_mismatch(): content = """ arr: !core/ndarray-1.0.0 data: [1, 2, 3] shape: [2] """ buff = helpers.yaml_to_asdf(content) with pytest.raises(ValueError): with asdf.AsdfFile.open(buff) as ff: pass
def test_mask_arbitrary(): content = """ arr: !core/ndarray-1.0.0 data: [[1, 2, 3, 1234], [5, 6, 7, 8]] mask: 1234 """ buff = helpers.yaml_to_asdf(content) with asdf.open(buff) as ff: assert_array_equal( ff.tree['arr'].mask, [[False, False, False, True], [False, False, False, False]])
def test_mask_nan(): content = """ arr: !core/ndarray-1.0.0 data: [[1, 2, 3, .NaN], [5, 6, 7, 8]] mask: .NaN """ buff = helpers.yaml_to_asdf(content) with asdf.AsdfFile.open(buff) as ff: assert_array_equal( ff.tree['arr'].mask, [[False, False, False, True], [False, False, False, False]])
def test_mask_datatype(tmpdir): content = """ arr: !core/ndarray-1.0.0 data: [1, 2, 3] dtype: int32 mask: !core/ndarray-1.0.0 data: [true, true, false] """ buff = helpers.yaml_to_asdf(content) with asdf.open(buff) as ff: pass
def test_mask_nan(): content = """ arr: !core/ndarray-1.0.0 data: [[1, 2, 3, .NaN], [5, 6, 7, 8]] mask: .NaN """ buff = helpers.yaml_to_asdf(content) with asdf.open(buff) as ff: assert_array_equal( ff.tree['arr'].mask, [[False, False, False, True], [False, False, False, False]])
def test_mask_datatype(tmpdir): content = """ arr: !core/ndarray-1.0.0 data: [1, 2, 3] dtype: int32 mask: !core/ndarray-1.0.0 data: [true, true, false] """ buff = helpers.yaml_to_asdf(content) with asdf.AsdfFile.open(buff) as ff: pass
def test_mask_arbitrary(): content = """ arr: !core/ndarray-1.0.0 data: [[1, 2, 3, 1234], [5, 6, 7, 8]] mask: 1234 """ buff = helpers.yaml_to_asdf(content) with asdf.AsdfFile.open(buff) as ff: assert_array_equal( ff.tree['arr'].mask, [[False, False, False, True], [False, False, False, False]])
def runtest(self): from asdf import AsdfFile, block, util from asdf.tests import helpers name, version = parse_schema_filename(self.filename) if should_skip(name, version): return standard_version = self._find_standard_version(name, version) # Make sure that the examples in the schema files (and thus the # ASDF standard document) are valid. buff = helpers.yaml_to_asdf( 'example: ' + self.example.strip(), standard_version=standard_version) ff = AsdfFile( uri=util.filepath_to_url(os.path.abspath(self.filename)), ignore_unrecognized_tag=self.ignore_unrecognized_tag, ignore_version_mismatch=self.ignore_version_mismatch, ) # Fake an external file ff2 = AsdfFile({'data': np.empty((1024*1024*8), dtype=np.uint8)}) ff._external_asdf_by_uri[ util.filepath_to_url( os.path.abspath( os.path.join( os.path.dirname(self.filename), 'external.asdf')))] = ff2 # Add some dummy blocks so that the ndarray examples work for i in range(3): b = block.Block(np.zeros((1024*1024*8), dtype=np.uint8)) b._used = True ff.blocks.add(b) b._array_storage = "streamed" try: with pytest.warns(None) as w: ff._open_impl(ff, buff, mode='rw') # Do not tolerate any warnings that occur during schema validation assert len(w) == 0, helpers.display_warnings(w) except Exception: print("From file:", self.filename) raise # Just test we can write it out. A roundtrip test # wouldn't always yield the correct result, so those have # to be covered by "real" unit tests. if b'external.asdf' not in buff.getvalue(): buff = io.BytesIO() ff.write_to(buff)
def test_invalid_mask_datatype(tmpdir): content = """ arr: !core/ndarray-1.0.0 data: [1, 2, 3] dtype: int32 mask: !core/ndarray-1.0.0 data: ['a', 'b', 'c'] """ buff = helpers.yaml_to_asdf(content) with pytest.raises(jsonschema.ValidationError): with asdf.open(buff) as ff: pass
def test_inline_structured(): content = """ arr: !core/ndarray-1.0.0 datatype: [['ascii', 4], uint16, uint16, ['ascii', 4]] data: [[M110, 110, 205, And], [ M31, 31, 224, And], [ M32, 32, 221, And], [M103, 103, 581, Cas]]""" buff = helpers.yaml_to_asdf(content) with asdf.open(buff) as ff: assert ff.tree['arr']['f1'].dtype.char == 'H'
def test_copy_inline(): yaml = """ x0: !core/ndarray-1.0.0 data: [-1.0, 1.0] """ buff = helpers.yaml_to_asdf(yaml) with asdf.AsdfFile.open(buff) as infile: with asdf.AsdfFile() as f: f.tree['a'] = infile.tree['x0'] f.tree['b'] = f.tree['a'] f.write_to(io.BytesIO())
def test_inline_structured(): content = """ arr: !core/ndarray-1.0.0 datatype: [['ascii', 4], uint16, uint16, ['ascii', 4]] data: [[M110, 110, 205, And], [ M31, 31, 224, And], [ M32, 32, 221, And], [M103, 103, 581, Cas]]""" buff = helpers.yaml_to_asdf(content) with asdf.AsdfFile.open(buff) as ff: assert ff.tree['arr']['f1'].dtype.char == 'H'
def test_invalid_mask_datatype(tmpdir): content = """ arr: !core/ndarray-1.0.0 data: [1, 2, 3] dtype: int32 mask: !core/ndarray-1.0.0 data: ['a', 'b', 'c'] """ buff = helpers.yaml_to_asdf(content) with pytest.raises(jsonschema.ValidationError): with asdf.AsdfFile.open(buff) as ff: pass
def test_copy_inline(): yaml = """ x0: !core/ndarray-1.0.0 data: [-1.0, 1.0] """ buff = helpers.yaml_to_asdf(yaml) with asdf.open(buff) as infile: with asdf.AsdfFile() as f: f.tree['a'] = infile.tree['x0'] f.tree['b'] = f.tree['a'] f.write_to(io.BytesIO())
def runtest(self): from asdf import AsdfFile, block, util from asdf.tests import helpers from .extension import TestExtension name, version = parse_schema_filename(self.filename) if should_skip(name, version): return standard_version = self._find_standard_version(name, version) # Make sure that the examples in the schema files (and thus the # ASDF standard document) are valid. buff = helpers.yaml_to_asdf( 'example: ' + self.example.strip(), standard_version=standard_version) ff = AsdfFile( uri=util.filepath_to_url(os.path.abspath(self.filename)), extensions=TestExtension()) # Fake an external file ff2 = AsdfFile({'data': np.empty((1024*1024*8), dtype=np.uint8)}) ff._external_asdf_by_uri[ util.filepath_to_url( os.path.abspath( os.path.join( os.path.dirname(self.filename), 'external.asdf')))] = ff2 # Add some dummy blocks so that the ndarray examples work for i in range(3): b = block.Block(np.zeros((1024*1024*8), dtype=np.uint8)) b._used = True ff.blocks.add(b) b._array_storage = "streamed" try: with pytest.warns(None) as w: import warnings ff._open_impl(ff, buff, mode='rw') # Do not tolerate any warnings that occur during schema validation assert len(w) == 0, helpers.display_warnings(w) except Exception: print("From file:", self.filename) raise # Just test we can write it out. A roundtrip test # wouldn't always yield the correct result, so those have # to be covered by "real" unit tests. if b'external.asdf' not in buff.getvalue(): buff = io.BytesIO() ff.write_to(buff)
def test_unit(): yaml = """ unit: !unit/unit-1.0.0 "2.1798721 10-18kg m2 s-2" """ buff = helpers.yaml_to_asdf(yaml) with asdf.AsdfFile.open(buff) as ff: assert ff.tree['unit'].is_equivalent(u.Ry) buff2 = io.BytesIO() ff.write_to(buff2) buff2.seek(0) with asdf.AsdfFile.open(buff2) as ff: assert ff.tree['unit'].is_equivalent(u.Ry)
def test_old_history(tmpdir): """Make sure that old versions of the history format are still accepted""" yaml = """ history: - !core/history_entry-1.0.0 description: "Here's a test of old history entries" software: !core/software-1.0.0 name: foo version: 1.2.3 """ buff = yaml_to_asdf(yaml) with asdf.open(buff) as af: assert len(af.tree['history']) == 1
def test_unit(): yaml = """ unit: !unit/unit-1.0.0 "2.1798721 10-18kg m2 s-2" """ buff = helpers.yaml_to_asdf(yaml) with asdf.open(buff) as ff: assert ff.tree['unit'].is_equivalent(u.Ry) buff2 = io.BytesIO() ff.write_to(buff2) buff2.seek(0) with asdf.open(buff2) as ff: assert ff.tree['unit'].is_equivalent(u.Ry)
def test_read_large_literal(): value = 1 << 64 yaml = """integer: {}""".format(value) buff = helpers.yaml_to_asdf(yaml) with pytest.warns(UserWarning) as w: with asdf.open(buff) as af: assert af['integer'] == value # We get two warnings: one for validation time, and one when defaults # are filled. It seems like we could improve this architecture, though... assert len(w) == 2 assert str(w[0].message).startswith('Invalid integer literal value') assert str(w[1].message).startswith('Invalid integer literal value')
def test_tagging_scalars(): yaml = """ unit: !unit/unit-1.0.0 m not_unit: m """ from astropy import units as u buff = helpers.yaml_to_asdf(yaml) with asdf.AsdfFile.open(buff) as ff: assert isinstance(ff.tree['unit'], u.UnitBase) assert not isinstance(ff.tree['not_unit'], u.UnitBase) assert isinstance(ff.tree['not_unit'], str) assert ff.tree == {'unit': u.m, 'not_unit': 'm'}
def test_mismatched_columns(): yaml = """ table: !core/table columns: - !core/column data: !core/ndarray data: [0, 1, 2] name: a - !core/column data: !core/ndarray data: [0, 1, 2, 3] name: b """ buff = helpers.yaml_to_asdf(yaml) with pytest.raises(ValueError): with asdf.AsdfFile.open(buff) as ff: pass
def test_tag_reference_validation(): class DefaultTypeExtension(CustomExtension): @property def types(self): return [TagReferenceType] yaml = """ custom: !<tag:nowhere.org:custom/tag_reference-1.0.0> name: "Something" things: !core/ndarray-1.0.0 data: [1, 2, 3] """ buff = helpers.yaml_to_asdf(yaml) with asdf.AsdfFile.open(buff, extensions=[DefaultTypeExtension()]) as ff: custom = ff.tree['custom'] assert custom['name'] == "Something" assert_array_equal(custom['things'], [1, 2, 3])
def test_tagging_scalars(): yaml = """ unit: !unit/unit-1.0.0 m not_unit: m """ from astropy import units as u buff = helpers.yaml_to_asdf(yaml) with asdf.AsdfFile.open(buff) as ff: assert isinstance(ff.tree['unit'], u.UnitBase) assert not isinstance(ff.tree['not_unit'], u.UnitBase) assert isinstance(ff.tree['not_unit'], str) assert ff.tree == { 'unit': u.m, 'not_unit': 'm' }
def test_missing_extension_warning(): yaml = """ history: extensions: - !core/extension_metadata-1.0.0 extension_class: foo.bar.FooBar software: !core/software-1.0.0 name: foo version: 1.2.3 """ buff = yaml_to_asdf(yaml) with pytest.warns(None) as warnings: with asdf.open(buff) as af: pass assert len(warnings) == 1, display_warnings(warnings) assert str(warnings[0].message).startswith( "File was created with extension 'foo.bar.FooBar'")
def test_foreign_tag_reference_validation(): class ForeignTagReferenceType(asdftypes.CustomType): name = 'foreign_tag_reference' organization = 'nowhere.org' version = (1, 0, 0) standard = 'custom' @classmethod def from_tree(cls, tree, ctx): node = {} node['a'] = yamlutil.tagged_tree_to_custom_tree(tree['a'], ctx) node['b'] = yamlutil.tagged_tree_to_custom_tree(tree['b'], ctx) return node class ForeignTypeExtension(CustomExtension): @property def types(self): return [TagReferenceType, ForeignTagReferenceType] yaml = """ custom: !<tag:nowhere.org:custom/foreign_tag_reference-1.0.0> a: !<tag:nowhere.org:custom/tag_reference-1.0.0> name: "Something" things: !core/ndarray-1.0.0 data: [1, 2, 3] b: !<tag:nowhere.org:custom/tag_reference-1.0.0> name: "Anything" things: !core/ndarray-1.0.0 data: [4, 5, 6] """ buff = helpers.yaml_to_asdf(yaml) with asdf.AsdfFile.open(buff, extensions=ForeignTypeExtension()) as ff: a = ff.tree['custom']['a'] b = ff.tree['custom']['b'] assert a['name'] == 'Something' assert_array_equal(a['things'], [1, 2, 3]) assert b['name'] == 'Anything' assert_array_equal(b['things'], [4, 5, 6])
def test_strict_extension_check(): yaml = """ history: extensions: - !core/extension_metadata-1.0.0 extension_class: foo.bar.FooBar software: !core/software-1.0.0 name: foo version: 1.2.3 """ buff = yaml_to_asdf(yaml) with pytest.raises(RuntimeError): with asdf.open(buff, strict_extension_check=True) as af: pass # Make sure to test for incompatibility with ignore_missing_extensions with pytest.raises(ValueError): with asdf.open(buff, strict_extension_check=True, ignore_missing_extensions=True) as af: pass
def test_mismatched_columns(): yaml = """ table: !<tag:astropy.org:astropy/table/table-1.0.0> columns: - !core/column-1.0.0 data: !core/ndarray-1.0.0 data: [0, 1, 2] name: a - !core/column-1.0.0 data: !core/ndarray-1.0.0 data: [0, 1, 2, 3] name: b colnames: [a, b] """ buff = helpers.yaml_to_asdf(yaml) with pytest.raises(ValueError) as err: with asdf.open(buff) as ff: pass assert 'Inconsistent data column lengths' in str(err)
def test_old_history(tmpdir): """Make sure that old versions of the history format are still accepted""" yaml = """ history: - !core/history_entry-1.0.0 description: "Here's a test of old history entries" software: !core/software-1.0.0 name: foo version: 1.2.3 """ buff = yaml_to_asdf(yaml) with asdf.open(buff) as af: assert len(af.tree['history']) == 1 # Test the history entry retrieval API entries = af.get_history_entries() assert len(entries) == 1 assert isinstance(entries, list) assert isinstance(entries[0], HistoryEntry) assert entries[0]['description'] == "Here's a test of old history entries" assert entries[0]['software']['name'] == 'foo'
def test_ndim_validation(tmpdir): content = """ obj: !<tag:nowhere.org:custom/ndim-1.0.0> a: !core/ndarray-1.0.0 data: [1, 2, 3] """ buff = helpers.yaml_to_asdf(content) with pytest.raises(jsonschema.ValidationError): with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass content = """ obj: !<tag:nowhere.org:custom/ndim-1.0.0> a: !core/ndarray-1.0.0 data: [[1, 2, 3]] """ buff = helpers.yaml_to_asdf(content) with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass content = """ obj: !<tag:nowhere.org:custom/ndim-1.0.0> a: !core/ndarray-1.0.0 shape: [1, 3] data: [[1, 2, 3]] """ buff = helpers.yaml_to_asdf(content) with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass content = """ obj: !<tag:nowhere.org:custom/ndim-1.0.0> b: !core/ndarray-1.0.0 data: [1, 2, 3] """ buff = helpers.yaml_to_asdf(content) with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass content = """ obj: !<tag:nowhere.org:custom/ndim-1.0.0> b: !core/ndarray-1.0.0 data: [[1, 2, 3]] """ buff = helpers.yaml_to_asdf(content) with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass content = """ obj: !<tag:nowhere.org:custom/ndim-1.0.0> b: !core/ndarray-1.0.0 data: [[[1, 2, 3]]] """ buff = helpers.yaml_to_asdf(content) with pytest.raises(jsonschema.ValidationError): with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass
def test_structured_datatype_validation(tmpdir): content = """ obj: !<tag:nowhere.org:custom/datatype-1.0.0> c: !core/ndarray-1.0.0 data: [[1, 'a'], [2, 'b'], [3, 'c']] datatype: - name: a datatype: int8 - name: b datatype: ['ascii', 8] """ buff = helpers.yaml_to_asdf(content) with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass content = """ obj: !<tag:nowhere.org:custom/datatype-1.0.0> c: !core/ndarray-1.0.0 data: [[1, 'a'], [2, 'b'], [3, 'c']] datatype: - name: a datatype: int64 - name: b datatype: ['ascii', 8] """ buff = helpers.yaml_to_asdf(content) with pytest.raises(jsonschema.ValidationError): with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass content = """ obj: !<tag:nowhere.org:custom/datatype-1.0.0> c: !core/ndarray-1.0.0 data: [[1, 'a', 0], [2, 'b', 1], [3, 'c', 2]] datatype: - name: a datatype: int8 - name: b datatype: ['ascii', 8] - name: c datatype: float64 """ buff = helpers.yaml_to_asdf(content) with pytest.raises(jsonschema.ValidationError): with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass content = """ obj: !<tag:nowhere.org:custom/datatype-1.0.0> c: !core/ndarray-1.0.0 data: [1, 2, 3] """ buff = helpers.yaml_to_asdf(content) with pytest.raises(jsonschema.ValidationError): with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass content = """ obj: !<tag:nowhere.org:custom/datatype-1.0.0> d: !core/ndarray-1.0.0 data: [[1, 'a'], [2, 'b'], [3, 'c']] datatype: - name: a datatype: int8 - name: b datatype: ['ascii', 8] """ buff = helpers.yaml_to_asdf(content) with pytest.raises(jsonschema.ValidationError): with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass content = """ obj: !<tag:nowhere.org:custom/datatype-1.0.0> d: !core/ndarray-1.0.0 data: [[1, 'a'], [2, 'b'], [3, 'c']] datatype: - name: a datatype: int16 - name: b datatype: ['ascii', 16] """ buff = helpers.yaml_to_asdf(content) with asdf.AsdfFile.open(buff, extensions=CustomExtension()) as ff: pass