def test_version_mismatch_file(): testfile = str(get_test_data_path('version_mismatch.fits')) with pytest.warns(None) as w: with asdf.open(testfile, ignore_version_mismatch=False) as fits_handle: assert fits_handle.tree['a'] == complex(0j) # This is the warning that we expect from opening the FITS file assert len(w) == 1, display_warnings(w) assert str(w[0].message) == ( "'tag:stsci.edu:asdf/core/complex' with version 7.0.0 found in file " "'{}', but latest supported version is 1.0.0".format(testfile)) # Make sure warning does not occur when warning is ignored (default) with pytest.warns(None) as w: with asdf.open(testfile) as fits_handle: assert fits_handle.tree['a'] == complex(0j) assert len(w) == 0, display_warnings(w) with pytest.warns(None) as w: with fits_embed.AsdfInFits.open(testfile, ignore_version_mismatch=False) as fits_handle: assert fits_handle.tree['a'] == complex(0j) assert len(w) == 1 assert str(w[0].message) == ( "'tag:stsci.edu:asdf/core/complex' with version 7.0.0 found in file " "'{}', but latest supported version is 1.0.0".format(testfile)) # Make sure warning does not occur when warning is ignored (default) with pytest.warns(None) as w: with fits_embed.AsdfInFits.open(testfile) as fits_handle: assert fits_handle.tree['a'] == complex(0j) assert len(w) == 0, display_warnings(w)
def test_no_yaml_end_marker(tmpdir): content = b"""#ASDF 1.0.0 %YAML 1.1 %TAG ! tag:stsci.edu:asdf/ --- !core/asdf-1.0.0 foo: bar...baz baz: 42 """ path = os.path.join(str(tmpdir), 'test.asdf') buff = io.BytesIO(content) with pytest.raises(ValueError): with asdf.open(buff): pass buff.seek(0) fd = generic_io.InputStream(buff, 'r') with pytest.raises(ValueError): with asdf.open(fd): pass with open(path, 'wb') as fd: fd.write(content) with open(path, 'rb') as fd: with pytest.raises(ValueError): with asdf.open(fd): pass
def test_extension_version_warning(): yaml = """ history: extensions: - !core/extension_metadata-1.0.0 extension_class: asdf.extension.BuiltinExtension software: !core/software-1.0.0 name: asdf version: 100.0.3 """ buff = yaml_to_asdf(yaml) with pytest.warns(None) as warnings: with asdf.open(buff) as af: pass assert len(warnings) == 1, display_warnings(warnings) assert str(warnings[0].message).startswith( "File was created with extension 'asdf.extension.BuiltinExtension' " "from package asdf-100.0.3") buff.seek(0) # Make sure suppressing the warning works too with pytest.warns(None) as warnings: with asdf.open(buff, ignore_missing_extensions=True) as af: pass assert len(warnings) == 0, display_warnings(warnings)
def test_array_to_stream(tmpdir): tree = { 'stream': np.array([1, 2, 3, 4], np.int64), } buff = io.BytesIO() ff = asdf.AsdfFile(tree) ff.set_array_storage(tree['stream'], 'streamed') ff.write_to(buff) buff.write(np.array([5, 6, 7, 8], np.int64).tostring()) buff.seek(0) ff = asdf.open(generic_io.InputStream(buff)) assert_array_equal(ff.tree['stream'], [1, 2, 3, 4, 5, 6, 7, 8]) buff.seek(0) ff2 = asdf.AsdfFile(ff) ff2.write_to(buff) assert b"shape: ['*']" in buff.getvalue() with open(os.path.join(str(tmpdir), 'test.asdf'), 'wb') as fd: ff = asdf.AsdfFile(tree) ff.set_array_storage(tree['stream'], 'streamed') ff.write_to(fd) fd.write(np.array([5, 6, 7, 8], np.int64).tostring()) with asdf.open(os.path.join(str(tmpdir), 'test.asdf')) as ff: assert_array_equal(ff.tree['stream'], [1, 2, 3, 4, 5, 6, 7, 8]) ff2 = asdf.AsdfFile(ff) ff2.write_to(buff) assert b"shape: ['*']" in buff.getvalue()
def test_custom_validation_with_definitions_bad(tmpdir): custom_schema_path = helpers.get_test_data_path('custom_schema_definitions.yaml') asdf_file = os.path.join(str(tmpdir), 'out.asdf') # This tree does NOT conform to the custom schema tree = { 'forb': { 'biz': 'hello', 'baz': 'world' } } # Creating file without custom schema should pass with asdf.AsdfFile(tree) as ff: ff.write_to(asdf_file) # Creating file with custom schema should fail with pytest.raises(ValidationError): with asdf.AsdfFile(tree, custom_schema=custom_schema_path) as ff: pass # Opening file without custom schema should pass with asdf.open(asdf_file) as ff: pass # Opening file with custom schema should fail with pytest.raises(ValidationError): with asdf.open(asdf_file, custom_schema=custom_schema_path) as ff: pass
def test_no_asdf_blocks(tmpdir): content = b"""#ASDF 1.0.0 %YAML 1.1 %TAG ! tag:stsci.edu:asdf/ --- !core/asdf-1.0.0 foo: bar ... XXXXXXXX """ path = os.path.join(str(tmpdir), 'test.asdf') buff = io.BytesIO(content) with asdf.open(buff) as ff: assert len(ff.blocks) == 0 buff.seek(0) fd = generic_io.InputStream(buff, 'r') with asdf.open(fd) as ff: assert len(ff.blocks) == 0 with open(path, 'wb') as fd: fd.write(content) with open(path, 'rb') as fd: with asdf.open(fd) as ff: assert len(ff.blocks) == 0
def test_fill_and_remove_defaults(): class DefaultType(dict, types.CustomType): name = 'default' organization = 'nowhere.org' version = (1, 0, 0) standard = 'custom' class DefaultTypeExtension(CustomExtension): @property def types(self): return [DefaultType] yaml = """ custom: !<tag:nowhere.org:custom/default-1.0.0> b: {} """ buff = helpers.yaml_to_asdf(yaml) with asdf.open(buff, extensions=[DefaultTypeExtension()]) as ff: assert 'a' in ff.tree['custom'] assert ff.tree['custom']['a'] == 42 assert ff.tree['custom']['b']['c'] == 82 buff.seek(0) with asdf.open(buff, extensions=[DefaultTypeExtension()], do_not_fill_defaults=True) as ff: assert 'a' not in ff.tree['custom'] assert 'c' not in ff.tree['custom']['b'] ff.fill_defaults() assert 'a' in ff.tree['custom'] assert ff.tree['custom']['a'] == 42 assert 'c' in ff.tree['custom']['b'] assert ff.tree['custom']['b']['c'] == 82 ff.remove_defaults() assert 'a' not in ff.tree['custom'] assert 'c' not in ff.tree['custom']['b']
def test_readonly(tmpdir): tmpfile = str(tmpdir.join('data.asdf')) tree = dict(data=np.ndarray((100))) with asdf.AsdfFile(tree) as af: # Make sure we're actually writing to an internal array for this test af.write_to(tmpfile, all_array_storage='internal') # Opening in read mode (the default) should mean array is readonly with asdf.open(tmpfile) as af: assert af['data'].flags.writeable == False with pytest.raises(ValueError) as err: af['data'][0] = 41 assert str(err) == 'assignment destination is read-only' # This should be perfectly fine with asdf.open(tmpfile, mode='rw') as af: assert af['data'].flags.writeable == True af['data'][0] = 40 # Copying the arrays makes it safe to write to the underlying array with asdf.open(tmpfile, mode='r', copy_arrays=True) as af: assert af['data'].flags.writeable == True af['data'][0] = 42
def test_no_final_newline(tmpdir): content = b"""#ASDF 1.0.0 %YAML 1.1 %TAG ! tag:stsci.edu:asdf/ --- !core/asdf-1.0.0 foo: ...bar... baz: 42 ...""" path = os.path.join(str(tmpdir), 'test.asdf') buff = io.BytesIO(content) with asdf.open(buff) as ff: assert len(ff.tree) == 2 buff.seek(0) fd = generic_io.InputStream(buff, 'r') with asdf.open(fd) as ff: assert len(ff.tree) == 2 with open(path, 'wb') as fd: fd.write(content) with open(path, 'rb') as fd: with asdf.open(fd) as ff: assert len(ff.tree) == 2
def diff(filenames, minimal, iostream=sys.stdout): """Top-level implementation of diff algorithm""" try: with asdf.open(filenames[0], _force_raw_types=True) as asdf0: with asdf.open(filenames[1], _force_raw_types=True) as asdf1: diff_ctx = DiffContext(asdf0, asdf1, iostream, minimal=minimal) compare_trees(diff_ctx, asdf0.tree, asdf1.tree) except ValueError as error: raise RuntimeError(str(error))
def test_supported_versions(): class CustomFlow: def __init__(self, c=None, d=None): self.c = c self.d = d class CustomFlowType(types.CustomType): version = '1.1.0' supported_versions = [(1,0,0), versioning.AsdfSpec('>=1.1.0')] name = 'custom_flow' organization = 'nowhere.org' standard = 'custom' types = [CustomFlow] @classmethod def from_tree(cls, tree, ctx): # Convert old schema to new CustomFlow type if cls.version == '1.0.0': return CustomFlow(c=tree['a'], d=tree['b']) else: return CustomFlow(**tree) return CustomFlow(**kwargs) @classmethod def to_tree(cls, data, ctx): if cls.version == '1.0.0': tree = dict(a=data.c, b=data.d) else: tree = dict(c=data.c, d=data.d) class CustomFlowExtension(CustomExtension): @property def types(self): return [CustomFlowType] new_yaml = """ flow_thing: !<tag:nowhere.org:custom/custom_flow-1.1.0> c: 100 d: 3.14 """ old_yaml = """ flow_thing: !<tag:nowhere.org:custom/custom_flow-1.0.0> a: 100 b: 3.14 """ new_buff = helpers.yaml_to_asdf(new_yaml) new_data = asdf.open(new_buff, extensions=CustomFlowExtension()) assert type(new_data.tree['flow_thing']) == CustomFlow old_buff = helpers.yaml_to_asdf(old_yaml) old_data = asdf.open(old_buff, extensions=CustomFlowExtension()) assert type(old_data.tree['flow_thing']) == CustomFlow
def test_recompression(tmpdir): tree = _get_large_tree() tmpfile = os.path.join(str(tmpdir), 'test1.asdf') afile = asdf.AsdfFile(tree) afile.write_to(tmpfile, all_array_compression='zlib') afile.close() afile = asdf.open(tmpfile) tmpfile = os.path.join(str(tmpdir), 'test2.asdf') afile.write_to(tmpfile, all_array_compression='bzp2') afile.close() afile = asdf.open(tmpfile) helpers.assert_tree_match(tree, afile.tree) afile.close()
def test_open_gzipped(): testfile = get_test_data_path('asdf.fits.gz') # Opening as an HDU should work with fits.open(testfile) as ff: with asdf.open(ff) as af: assert af.tree['stuff'].shape == (20, 20) with fits_embed.AsdfInFits.open(testfile) as af: assert af.tree['stuff'].shape == (20, 20) with asdf.open(testfile) as af: assert af.tree['stuff'].shape == (20, 20)
def test_input(tmpdir): tree = _get_large_tree() tmpfile = os.path.join(str(tmpdir), 'test1.asdf') afile = asdf.AsdfFile(tree) afile.write_to(tmpfile, all_array_compression='zlib') afile.close() afile = asdf.open(tmpfile) tmpfile = os.path.join(str(tmpdir), 'test2.asdf') afile.write_to(tmpfile) afile.close() afile = asdf.open(tmpfile) helpers.assert_tree_match(tree, afile.tree) assert afile.get_array_compression(afile.tree['science_data']) == 'zlib' afile.close()
def test_empty_file(): buff = io.BytesIO(b"#ASDF 1.0.0\n") buff.seek(0) with asdf.open(buff) as ff: assert ff.tree == {} assert len(ff.blocks) == 0 buff = io.BytesIO(b"#ASDF 1.0.0\n#ASDF_STANDARD 1.0.0") buff.seek(0) with asdf.open(buff) as ff: assert ff.tree == {} assert len(ff.blocks) == 0
def test_not_asdf_file(): buff = io.BytesIO(b"SIMPLE") buff.seek(0) with pytest.raises(ValueError): with asdf.open(buff): pass buff = io.BytesIO(b"SIMPLE\n") buff.seek(0) with pytest.raises(ValueError): with asdf.open(buff): pass
def test_no_asdf_header(tmpdir): content = b"What? This ain't no ASDF file" path = os.path.join(str(tmpdir), 'test.asdf') buff = io.BytesIO(content) with pytest.raises(ValueError): asdf.open(buff) with open(path, 'wb') as fd: fd.write(content) with open(path, 'rb') as fd: with pytest.raises(ValueError): asdf.open(fd)
def test_version_mismatch(): yaml = """ a: !core/complex-42.0.0 0j """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(None) as warning: with asdf.open(buff, ignore_version_mismatch=False) as ff: assert isinstance(ff.tree['a'], complex) assert len(warning) == 1 assert str(warning[0].message) == ( "'tag:stsci.edu:asdf/core/complex' with version 42.0.0 found in file, " "but latest supported version is 1.0.0") # Make sure warning is repeatable buff.seek(0) with pytest.warns(None) as warning: with asdf.open(buff, ignore_version_mismatch=False) as ff: assert isinstance(ff.tree['a'], complex) assert len(warning) == 1 assert str(warning[0].message) == ( "'tag:stsci.edu:asdf/core/complex' with version 42.0.0 found in file, " "but latest supported version is 1.0.0") # Make sure the warning does not occur if it is being ignored (default) buff.seek(0) with pytest.warns(None) as warning: with asdf.open(buff) as ff: assert isinstance(ff.tree['a'], complex) assert len(warning) == 0, helpers.display_warnings(warning) # If the major and minor match, there should be no warning. yaml = """ a: !core/complex-1.0.1 0j """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(None) as warning: with asdf.open(buff, ignore_version_mismatch=False) as ff: assert isinstance(ff.tree['a'], complex) assert len(warning) == 0
def test_unsupported_version_warning(): class CustomFlow: pass class CustomFlowType(types.CustomType): version = '1.0.0' supported_versions = [(1,0,0)] name = 'custom_flow' organization = 'nowhere.org' standard = 'custom' types = [CustomFlow] class CustomFlowExtension(CustomExtension): @property def types(self): return [CustomFlowType] yaml = """ flow_thing: !<tag:nowhere.org:custom/custom_flow-1.1.0> c: 100 d: 3.14 """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(None) as _warnings: data = asdf.open(buff, extensions=CustomFlowExtension()) assert len(_warnings) == 1 assert str(_warnings[0].message) == ( "Version 1.1.0 of tag:nowhere.org:custom/custom_flow is not compatible " "with any existing tag implementations")
def test_to_yaml(tmpdir): x = np.arange(0, 10, dtype=np.float) tree = { 'science_data': x, 'subset': x[3:-3], 'skipping': x[::2], 'not_shared': np.arange(10, 0, -1, dtype=np.uint8) } path = os.path.join(str(tmpdir), 'original.asdf') ff = AsdfFile(tree) ff.write_to(path) assert len(ff.blocks) == 2 result = main.main_from_args(['to_yaml', path]) assert result == 0 files = get_file_sizes(str(tmpdir)) assert 'original.asdf' in files assert 'original.yaml' in files with asdf.open(os.path.join(str(tmpdir), 'original.yaml')) as ff: assert_tree_match(ff.tree, tree) assert len(list(ff.blocks.internal_blocks)) == 0
def test_extract(tmpdir): hdulist = HDUList() image = ImageHDU(np.random.random((25, 25))) hdulist.append(image) tree = { 'some_words': 'These are some words', 'nested': { 'a': 100, 'b': 42 }, 'list': [x for x in range(10)], 'image': image.data } asdf_in_fits = str(tmpdir.join('asdf.fits')) with AsdfInFits(hdulist, tree) as aif: aif.write_to(asdf_in_fits) pure_asdf = str(tmpdir.join('extract.asdf')) extract.extract_file(asdf_in_fits, pure_asdf) assert os.path.exists(pure_asdf) with asdf.open(pure_asdf) as af: assert not isinstance(af, AsdfInFits) assert_tree_match(tree, af.tree)
def test_type_missing_dependencies(): class MissingType(types.CustomType): name = 'missing' organization = 'nowhere.org' version = (1, 1, 0) standard = 'custom' types = ['asdfghjkl12345.foo'] requires = ["ASDFGHJKL12345"] class DefaultTypeExtension(CustomExtension): @property def types(self): return [MissingType] yaml = """ custom: !<tag:nowhere.org:custom/missing-1.1.0> b: {foo: 42} """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(None) as w: with asdf.open(buff, extensions=[DefaultTypeExtension()]) as ff: assert ff.tree['custom']['b']['foo'] == 42 assert len(w) == 1
def test_junk_file(): buff = io.BytesIO(b"#ASDF 1.0.0\nFOO") buff.seek(0) with pytest.raises(ValueError): with asdf.open(buff): pass
def load_file(self, filepath): image = AstroImage.AstroImage(logger=self.logger) with asdf.open(filepath) as asdf_f: image.load_asdf(asdf_f) self.fitsimage.set_image(image) self.setWindowTitle(filepath)
def test_invalid_source(small_tree): buff = io.BytesIO() ff = asdf.AsdfFile(small_tree) # Since we're testing with small arrays, force all arrays to be stored # in internal blocks rather than letting some of them be automatically put # inline. ff.write_to(buff, all_array_storage='internal') buff.seek(0) with asdf.open(buff) as ff2: ff2.blocks.get_block(0) with pytest.raises(ValueError): ff2.blocks.get_block(2) with pytest.raises(IOError): ff2.blocks.get_block("http://127.0.0.1/") with pytest.raises(TypeError): ff2.blocks.get_block(42.0) with pytest.raises(ValueError): ff2.blocks.get_source(42.0) block = ff2.blocks.get_block(0) assert ff2.blocks.get_source(block) == 0
def test_stream_real_file(tmpdir): path = os.path.join(str(tmpdir), 'test.asdf') tree = { 'nonstream': np.array([1, 2, 3, 4], np.int64), 'stream': stream.Stream([6, 2], np.float64) } with open(path, 'wb') as fd: ff = asdf.AsdfFile(tree) # Since we're testing with small arrays, force this array to be stored # in an internal block rather than letting it be automatically put # inline. ff.set_array_storage(ff['nonstream'], 'internal') ff.write_to(fd) for i in range(100): fd.write(np.array([i] * 12, np.float64).tostring()) with asdf.open(path) as ff: assert len(ff.blocks) == 1 assert_array_equal(ff.tree['nonstream'], np.array([1, 2, 3, 4], np.int64)) assert ff.tree['stream'].shape == (100, 6, 2) assert len(ff.blocks) == 2 for i, row in enumerate(ff.tree['stream']): assert np.all(row == i)
def _test_defragment(tmpdir, codec): x = np.arange(0, 1000, dtype=np.float) tree = { 'science_data': x, 'subset': x[3:-3], 'skipping': x[::2], 'not_shared': np.arange(100, 0, -1, dtype=np.uint8) } path = os.path.join(str(tmpdir), 'original.asdf') out_path = os.path.join(str(tmpdir), 'original.defragment.asdf') ff = AsdfFile(tree) ff.write_to(path) assert len(ff.blocks) == 2 result = main.main_from_args( ['defragment', path, '-o', out_path, '-c', codec]) assert result == 0 files = get_file_sizes(str(tmpdir)) assert 'original.asdf' in files assert 'original.defragment.asdf' in files assert files['original.defragment.asdf'] < files['original.asdf'] with asdf.open(os.path.join(str(tmpdir), 'original.defragment.asdf')) as ff: assert_tree_match(ff.tree, tree) assert len(list(ff.blocks.internal_blocks)) == 2
def test_version_mismatch_with_supported_versions(): """Make sure that defining the supported_versions field does not affect whether or not schema mismatch warnings are triggered.""" class CustomFlow: pass class CustomFlowType(CustomTestType): version = '1.1.0' supported_versions = ['1.0.0', '1.1.0'] name = 'custom_flow' organization = 'nowhere.org' standard = 'custom' types = [CustomFlow] class CustomFlowExtension(CustomExtension): @property def types(self): return [CustomFlowType] yaml = """ flow_thing: !<tag:nowhere.org:custom/custom_flow-1.0.0> c: 100 d: 3.14 """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(None) as w: data = asdf.open( buff, ignore_version_mismatch=False, extensions=CustomFlowExtension()) assert len(w) == 1, helpers.display_warnings(w) assert str(w[0].message) == ( "'tag:nowhere.org:custom/custom_flow' with version 1.0.0 found in " "file, but latest supported version is 1.1.0")
def test_stream_with_nonstream(): buff = io.BytesIO() tree = { 'nonstream': np.array([1, 2, 3, 4], np.int64), 'stream': stream.Stream([6, 2], np.float64) } ff = asdf.AsdfFile(tree) # Since we're testing with small arrays, force this array to be stored in # an internal block rather than letting it be automatically put inline. ff.set_array_storage(ff['nonstream'], 'internal') ff.write_to(buff) for i in range(100): buff.write(np.array([i] * 12, np.float64).tostring()) buff.seek(0) with asdf.open(buff) as ff: assert len(ff.blocks) == 1 assert_array_equal(ff.tree['nonstream'], np.array([1, 2, 3, 4], np.int64)) assert ff.tree['stream'].shape == (100, 6, 2) assert len(ff.blocks) == 2 for i, row in enumerate(ff.tree['stream']): assert np.all(row == i)
def test_history_to_file(tmpdir): tmpfile = str(tmpdir.join('history.asdf')) with asdf.AsdfFile() as ff: ff.add_history_entry('This happened', {'name': 'my_tool', 'homepage': 'http://nowhere.org', 'author': 'John Doe', 'version': '2.0'}) ff.write_to(tmpfile) with asdf.open(tmpfile) as ff: assert 'entries' in ff.tree['history'] assert 'extensions' in ff.tree['history'] assert len(ff.tree['history']['entries']) == 1 entry = ff.tree['history']['entries'][0] assert entry['description'] == 'This happened' assert entry['software']['name'] == 'my_tool' assert entry['software']['version'] == '2.0' # Test the history entry retrieval API entries = ff.get_history_entries() assert len(entries) == 1 assert isinstance(entries, list) assert isinstance(entries[0], HistoryEntry) assert entries[0]['description'] == "This happened" assert entries[0]['software']['name'] == 'my_tool'
def test_extension_metadata(tmpdir): ff = asdf.AsdfFile() # So far only the base extension has been used assert len(ff.type_index.get_extensions_used()) == 1 tmpfile = str(tmpdir.join('extension.asdf')) ff.write_to(tmpfile) with asdf.open(tmpfile) as af: assert len(af.tree['history']['extensions']) == 1 metadata = af.tree['history']['extensions'][0] assert metadata.extension_class == 'asdf.extension.BuiltinExtension' # Don't bother with testing the version here since it will depend on # how recently the package was built (version is auto-generated) assert metadata.software['name'] == 'asdf'
def test_strict_extension_check(): yaml = """ history: extensions: - !core/extension_metadata-1.0.0 extension_class: foo.bar.FooBar software: !core/software-1.0.0 name: foo version: 1.2.3 """ buff = yaml_to_asdf(yaml) with pytest.raises(RuntimeError): with asdf.open(buff, strict_extension_check=True) as af: pass
def test_tagging_scalars(): yaml = """ unit: !unit/unit-1.0.0 m not_unit: m """ from astropy import units as u buff = helpers.yaml_to_asdf(yaml) with asdf.open(buff) as ff: assert isinstance(ff.tree['unit'], u.UnitBase) assert not isinstance(ff.tree['not_unit'], u.UnitBase) assert isinstance(ff.tree['not_unit'], str) assert ff.tree == {'unit': u.m, 'not_unit': 'm'}
def test_no_blocks_increase_size(tmp_path, create_editor, version): file_path = str(tmp_path / "test.asdf") with asdf.AsdfFile(version=version) as af: af["foo"] = "bar" af.write_to(file_path) new_value = "a" * 32768 os.environ["EDITOR"] = create_editor(r"foo: bar", f"foo: {new_value}") # With no blocks, we can expand the existing file, so this case # shouldn't require confirmation from the user. assert main.main_from_args(["edit", file_path]) == 0 with asdf.open(file_path) as af: assert af["foo"] == new_value
def test_set_array_compression(tmpdir): tmpfile = os.path.join(str(tmpdir), 'compressed.asdf') zlib_data = np.array([x for x in range(1000)]) bzp2_data = np.array([x for x in range(1000)]) tree = dict(zlib_data=zlib_data, bzp2_data=bzp2_data) with asdf.AsdfFile(tree) as af_out: af_out.set_array_compression(zlib_data, 'zlib') af_out.set_array_compression(bzp2_data, 'bzp2') af_out.write_to(tmpfile) with asdf.open(tmpfile) as af_in: assert af_in.get_array_compression(af_in.tree['zlib_data']) == 'zlib' assert af_in.get_array_compression(af_in.tree['bzp2_data']) == 'bzp2'
def test_read_large_literal(): value = 1 << 64 yaml = """integer: {}""".format(value) buff = helpers.yaml_to_asdf(yaml) with pytest.warns(UserWarning) as w: with asdf.open(buff) as af: assert af['integer'] == value # We get two warnings: one for validation time, and one when defaults # are filled. It seems like we could improve this architecture, though... assert len(w) == 2 assert str(w[0].message).startswith('Invalid integer literal value') assert str(w[1].message).startswith('Invalid integer literal value')
def test_array_view(tmp_path): """ Special handling is required when a view over a larger array is assigned to an HDU and referenced from the ASDF tree. """ file_path = tmp_path / "test.fits" data = np.arange(100, dtype=np.float64).reshape(5, 20) data_view = data[:, :10] hdul = fits.HDUList([fits.PrimaryHDU(), fits.ImageHDU(data_view)]) with asdf.fits_embed.AsdfInFits(hdulist=hdul) as af: af["data"] = hdul[-1].data af.write_to(file_path) with asdf.open(file_path) as af: assert_array_equal(af["data"], data_view)
def test_stream(): buff = io.BytesIO() tree = {'stream': stream.Stream([6, 2], np.float64)} ff = asdf.AsdfFile(tree) ff.write_to(buff) for i in range(100): buff.write(np.array([i] * 12, np.float64).tostring()) buff.seek(0) with asdf.open(buff) as ff: assert len(ff.blocks) == 1 assert ff.tree['stream'].shape == (100, 6, 2) for i, row in enumerate(ff.tree['stream']): assert np.all(row == i)
def test_open_stream(tmp_path): file_path = tmp_path / "test.asdf" with asdf.AsdfFile() as af: af["foo"] = "bar" af.write_to(file_path) class StreamWrapper: def __init__(self, fd): self._fd = fd def read(self, size=-1): return self._fd.read(size) with file_path.open("rb") as fd: with asdf.open(StreamWrapper(fd)) as af: assert af["foo"] == "bar"
def test_serialize_custom_type(tmpdir): with asdf.config_context() as config: config.add_resource_mapping({FOO_SCHEMA_URI: FOO_SCHEMA}) config.add_extension(FooExtension()) path = str(tmpdir / "test.asdf") af = asdf.AsdfFile() af["foo"] = Foo("bar") af.write_to(path) with asdf.open(path) as af2: assert af2["foo"].value == "bar" with pytest.raises(asdf.ValidationError): af["foo"] = Foo(12) af.write_to(path)
def test_to_fits_sip(): y, x = np.mgrid[:1024:10, :1024:10] xflat = np.ravel(x[1:-1, 1:-1]) yflat = np.ravel(y[1:-1, 1:-1]) af = asdf.open(get_pkg_data_filename('data/miriwcs.asdf')) miriwcs = af.tree['wcs'] bounding_box = ((0, 1024), (0, 1024)) mirisip = miriwcs.to_fits_sip(bounding_box, max_inv_pix_error=0.1) fitssip = astwcs.WCS(mirisip) fitsvalx, fitsvaly = fitssip.all_pix2world(xflat + 1, yflat + 1, 1) gwcsvalx, gwcsvaly = miriwcs(xflat, yflat) assert_allclose(gwcsvalx, fitsvalx, atol=1e-10, rtol=0) assert_allclose(gwcsvaly, fitsvaly, atol=1e-10, rtol=0) fits_inverse_valx, fits_inverse_valy = fitssip.all_world2pix( fitsvalx, fitsvaly, 1) assert_allclose(xflat, fits_inverse_valx - 1, atol=0.1, rtol=0) assert_allclose(yflat, fits_inverse_valy - 1, atol=0.1, rtol=0)
def test_version_mismatch_with_supported_versions(): """Make sure that defining the supported_versions field does not affect whether or not schema mismatch warnings are triggered.""" class CustomFlow(object): pass class CustomFlowType(CustomTestType): version = '1.1.0' supported_versions = ['1.0.0', '1.1.0'] name = 'custom_flow' organization = 'nowhere.org' standard = 'custom' types = [CustomFlow] class CustomFlowExtension(object): @property def types(self): return [CustomFlowType] @property def tag_mapping(self): return [('tag:nowhere.org:custom', 'http://nowhere.org/schemas/custom{tag_suffix}')] @property def url_mapping(self): return [ ('http://nowhere.org/schemas/custom/', util.filepath_to_url(TEST_DATA_PATH) + '/{url_suffix}.yaml') ] yaml = """ flow_thing: !<tag:nowhere.org:custom/custom_flow-1.0.0> c: 100 d: 3.14 """ buff = helpers.yaml_to_asdf(yaml) with pytest.warns(None) as w: data = asdf.open(buff, ignore_version_mismatch=False, extensions=CustomFlowExtension()) assert len(w) == 1, helpers.display_warnings(w) assert str(w[0].message) == ( "'tag:nowhere.org:custom/custom_flow' with version 1.0.0 found in " "file, but latest supported version is 1.1.0")
def test_extra_properties(): yaml = """ metadata: !core/extension_metadata-1.0.0 extension_class: foo.extension.FooExtension software: !core/software-1.0.0 name: FooSoft version: "1.5" extension_uri: http://foo.biz/extensions/foo-1.0.0 """ buff = helpers.yaml_to_asdf(yaml) with asdf.open(buff) as af: af["metadata"].extension_class == "foo.extension.FooExtension" af["metadata"].software["name"] == "FooSoft" af["metadata"].software["version"] == "1.5" af["metadata"]["extension_uri"] == "http://foo.biz/extensions/foo-1.0.0"
def read_table(filename, data_key=None, find_table=None, **kwargs): """ Read a `~astropy.table.Table` object from an ASDF file This requires `asdf <https://pypi.org/project/asdf/>`_ to be installed. By default, this function will look for a Table object with the key of ``data`` in the top-level ASDF tree. The parameters ``data_key`` and ``find_key`` can be used to override the default behavior. This function is registered as the Table reader for ASDF files with the unified I/O interface. Parameters ---------- filename : str or :class:`py.lath:local` Name of the file to be read data_key : str Optional top-level key to use for finding the Table in the tree. If not provided, uses ``data`` by default. Use of this parameter is not compatible with ``find_table``. find_table : function Optional function to be used for locating the Table in the tree. The function takes a single parameter, which is a dictionary representing the top of the ASDF tree. The function must return a `~astropy.table.Table` instance. Returns ------- table : `~astropy.table.Table` `~astropy.table.Table` instance """ try: import asdf except ImportError: raise Exception( "The asdf module is required to read and write ASDF files") if data_key and find_table: raise ValueError( "Options 'data_key' and 'find_table' are not compatible") with asdf.open(filename, **kwargs) as af: if find_table: return find_table(af.tree) else: return af[data_key or 'data']
def test_cartesian_spherical_asdf(tmpdir): s2c0 = geometry.SphericalToCartesian(wrap_lon_at=360) c2s0 = geometry.CartesianToSpherical(wrap_lon_at=180) # asdf round-trip test: assert_model_roundtrip(c2s0, tmpdir) assert_model_roundtrip(s2c0, tmpdir) # create file object f = asdf.AsdfFile({'c2s': c2s0, 's2c': s2c0}) # write to... buf = io.BytesIO() f.write_to(buf) # read back: buf.seek(0) f = asdf.open(buf) # retrieve transformations: c2s = f['c2s'] s2c = f['s2c'] pcoords = [(45.0, -90.0), (45.0, -45.0), (45, 0.0), (45.0, 45), (45.0, 90.0), (135.0, -90.0), (135.0, -45.0), (135.0, 0.0), (135.0, 45.0), (135.0, 90.0)] ncoords = [(225.0, -90.0), (225.0, -45.0), (225.0, 0.0), (225.0, 45.0), (225.0, 90.0), (315.0, -90.0), (315.0, -45.0), (315.0, 0.0), (315.0, 45.0), (315.0, 90.0)] for lon, lat in pcoords: xyz = s2c(lon, lat) assert xyz == s2c0(lon, lat) lon2, lat2 = c2s(*xyz) assert lon2, lat2 == c2s0(*xyz) assert np.allclose((lon, lat), (lon2, lat2)) for lon, lat in ncoords: xyz = s2c(lon, lat) assert xyz == s2c0(lon, lat) lon2, lat2 = c2s(*xyz) lon3, lat3 = s2c.inverse(*xyz) assert lon2, lat2 == c2s0(*xyz) assert np.allclose((lon, lat), (lon2 + 360, lat2)) assert np.allclose((lon, lat), (lon3, lat2))
def test_copy(tmpdir): tmpdir = str(tmpdir) my_array = np.random.rand(8, 8) tree = {'my_array': my_array, 'foo': {'bar': 'baz'}} ff = asdf.AsdfFile(tree) ff.write_to(os.path.join(tmpdir, 'test.asdf')) with asdf.open(os.path.join(tmpdir, 'test.asdf')) as ff: ff2 = ff.copy() ff2.tree['my_array'] *= 2 ff2.tree['foo']['bar'] = 'boo' assert np.all(ff2.tree['my_array'] == ff.tree['my_array'] * 2) assert ff.tree['foo']['bar'] == 'baz' assert_array_equal(ff2.tree['my_array'], ff2.tree['my_array'])
def test_serialize_table(tmpdir): tmpfile = str(tmpdir.join('table.fits')) data = np.random.random((10, 10)) table = Table(data) hdu = fits.BinTableHDU(table) hdulist = fits.HDUList() hdulist.append(hdu) tree = {'my_table': hdulist[1].data} with fits_embed.AsdfInFits(hdulist, tree) as ff: ff.write_to(tmpfile) with asdf.open(tmpfile) as ff: data = ff.tree['my_table'] assert data._source.startswith('fits:')
def test_dots_but_no_block_index(): # This puts `...` at the end of the file, so we sort of think # we might have a block index, but as it turns out, we don't # after reading a few chunks from the end of the file. buff = io.BytesIO() tree = {'array': np.ones((8, 8))} ff = asdf.AsdfFile(tree) ff.write_to(buff, include_block_index=False) buff.write(b'A' * 64000) buff.write(b'...\n') buff.seek(0) with asdf.open(buff) as ff: assert len(ff.blocks) == 1
def test_exploded_stream_read(tmpdir, small_tree): # Reading from an exploded input file should fail, but only once # the data block is accessed. This behavior is important so that # the tree can still be accessed even if the data is missing. path = os.path.join(str(tmpdir), 'test.asdf') ff = asdf.AsdfFile(small_tree) ff.write_to(path, all_array_storage='external') with open(path, 'rb') as fd: # This should work, so we can get the tree content x = generic_io.InputStream(fd, 'r') with asdf.open(x) as ff: # It's only when trying to access external data that an error occurs with pytest.raises(ValueError): ff.tree['science_data'][:]
def imaging_distortion(input_model, reference_files): """ Create the "detector" to "v2v3" transform for imaging mode. Parameters ---------- input_model : `~jwst.datamodel.DataModel` Input datamodel for processing reference_files : dict The dictionary of reference file names and their associated files. Returns ------- The transform model """ dist = DistortionModel(reference_files['distortion']) transform = dist.model try: transform.bounding_box except NotImplementedError: # Check if the transform in the reference file has a ``bounding_box``. # If not set a ``bounding_box`` equal to the size of the image. transform.bounding_box = transform_bbox_from_shape( input_model.data.shape) dist.close() # Add an offset for the filter if reference_files['filteroffset'] is not None: obsfilter = input_model.meta.instrument.filter obspupil = input_model.meta.instrument.pupil with asdf.open(reference_files['filteroffset']) as filter_offset: filters = filter_offset.tree['filters'] match_keys = {'filter': obsfilter, 'pupil': obspupil} row = find_row(filters, match_keys) if row is not None: col_offset = row.get('col_offset', 'N/A') row_offset = row.get('row_offset', 'N/A') if col_offset != 'N/A' and row_offset != 'N/A': transform = Shift(col_offset) & Shift(row_offset) | transform return transform
def read_grid(filename, fmt=None): '''Read a grid from a file. Parameters ---------- filename : string The path of the file you want to read the grid from. fmt : string The file format. If it is not given, the file format will be guessed from the file extension. Returns ------- Grid The read grid. Raises ------ ValueError If the file format could not be guessed from the file extension. NotImplementedError If the file format was not yet implemented. ''' from ..field import Grid if fmt is None: fmt = _guess_file_format(filename) if fmt is None: raise ValueError('Format not given and could not be guessed based on the file extension.') if fmt in ['asdf', 'fits']: import asdf f = asdf.open(filename) grid = Grid.from_dict(f.tree['grid']) f.close() return grid elif fmt == 'pickle': import pickle with open(filename, 'rb') as f: return pickle.load(f) else: raise NotImplementedError('The "%s" file format has not been implemented.' % fmt)
def test_access_tree_outside_handler(tmpdir): tempname = str(tmpdir.join('test.asdf')) tree = {'random': np.random.random(10)} ff = asdf.AsdfFile(tree) ff.write_to(str(tempname)) with asdf.open(tempname) as newf: pass # Accessing array data outside of handler should fail with pytest.raises(OSError): repr(newf.tree['random']) # Using the top-level getattr should also fail with pytest.raises(OSError): repr(newf['random'])
def test_masked_array_stay_open_bug(tmpdir): psutil = pytest.importorskip('psutil') tmppath = os.path.join(str(tmpdir), 'masked.asdf') tree = {'test': np.ma.array([1, 2, 3], mask=[False, True, False])} f = asdf.AsdfFile(tree) f.write_to(tmppath) p = psutil.Process() orig_open = p.open_files() for i in range(3): with asdf.open(tmppath) as f2: np.sum(f2.tree['test']) assert len(p.open_files()) == len(orig_open)
def test_dont_load_data(): x = np.arange(0, 10, dtype=np.float) tree = {'science_data': x, 'subset': x[3:-3], 'skipping': x[::2]} ff = asdf.AsdfFile(tree) buff = io.BytesIO() ff.write_to(buff) buff.seek(0) with asdf.open(buff) as ff: ff.run_hook('reserve_blocks') # repr and str shouldn't load data str(ff.tree['science_data']) repr(ff.tree) for block in ff.blocks.internal_blocks: assert block._data is None
def assert_selector_roundtrip(s, tmpdir, version=None): """ Assert that a selector can be written to an ASDF file and read back in without losing any of its essential properties. """ path = str(tmpdir / "test.asdf") with asdf.AsdfFile({"selector": s}, version=version) as af: af.write_to(path) with asdf.open(path) as af: rs = af["selector"] if isinstance(s, selector.RegionsSelector): _assert_selector_equal(s, rs) elif isinstance(s, selector._LabelMapper): _assert_mapper_equal(s, rs) else: assert False
def test_domain_orthopoly(tmpdir): model1d = astmodels.Chebyshev1D(2, c0=2, c1=3, c2=0.5, domain=[-2, 2]) model2d = astmodels.Chebyshev2D(1, 1, c0_0=1, c0_1=2, c1_0=3, x_domain=[-2, 2], y_domain=[-2, 2]) fa = AsdfFile() fa.tree['model1d'] = model1d fa.tree['model2d'] = model2d file_path = str(tmpdir.join('orthopoly_domain.asdf')) fa.write_to(file_path) with asdf.open(file_path) as f: assert f.tree['model1d'](1.8) == model1d(1.8) assert f.tree['model2d'](1.8, -1.5) == model2d(1.8, -1.5)
def test_missing_extension_warning(): yaml = """ history: extensions: - !core/extension_metadata-1.0.0 extension_class: foo.bar.FooBar software: !core/software-1.0.0 name: foo version: 1.2.3 """ buff = yaml_to_asdf(yaml) with pytest.warns( AsdfWarning, match="File was created with extension class 'foo.bar.FooBar'"): with asdf.open(buff): pass
def test_export_config(step_obj, expected, tmp_path): """Test retrieving of configuration parameters""" config_path = tmp_path / "config.asdf" step_obj.export_config(config_path) with asdf.open(config_path) as af: # StepConfig has an __eq__ implementation but we can't use it # due to differences between asdf 2.7 and 2.8 in serializing None # values. This can be simplified once the minimum asdf requirement # is changed to >= 2.8. # assert StepConfig.from_asdf(af) == expected config = StepConfig.from_asdf(af) assert config.class_name == expected.class_name assert config.name == expected.name assert config.steps == expected.steps parameters = set(expected.parameters.keys()).union(set(config.parameters.keys())) for parameter in parameters: assert config.parameters.get(parameter) == expected.parameters.get(parameter)
def explode(input, output=None): """ Explode a given ASDF file so each data block is in a separate file. Parameters ---------- input : str or file-like object The input file. output : str of file-like object The output file. """ if output is None: base, ext = os.path.splitext(input) output = base + '_exploded' + '.asdf' with asdf.open(input) as ff: ff.write_to(output, all_array_storage='external')
def read_metadata(filename): """ Reads JWST data product and extracts metadata fields of interest Parameters ---------- filename : `str` Path of JWST data file to be summarized Returns ------- dict : `dict` contaning JWST metadata fields of interest """ with asdf.open(filename, ignore_unrecognized_tag=True, ignore_missing_extensions=True) as af: return parse_metadata(af)