def test_array_to_stream(tmpdir):
tree = {
'stream': np.array([1, 2, 3, 4], np.int64),
}
buff = io.BytesIO()
ff = asdf.AsdfFile(tree)
ff.set_array_storage(tree['stream'], 'streamed')
ff.write_to(buff)
buff.write(np.array([5, 6, 7, 8], np.int64).tostring())
buff.seek(0)
ff = asdf.open(generic_io.InputStream(buff))
assert_array_equal(ff.tree['stream'], [1, 2, 3, 4, 5, 6, 7, 8])
buff.seek(0)
ff2 = asdf.AsdfFile(ff)
ff2.write_to(buff)
assert b"shape: ['*']" in buff.getvalue()
with open(os.path.join(str(tmpdir), 'test.asdf'), 'wb') as fd:
ff = asdf.AsdfFile(tree)
ff.set_array_storage(tree['stream'], 'streamed')
ff.write_to(fd)
fd.write(np.array([5, 6, 7, 8], np.int64).tostring())
with asdf.open(os.path.join(str(tmpdir), 'test.asdf')) as ff:
assert_array_equal(ff.tree['stream'], [1, 2, 3, 4, 5, 6, 7, 8])
ff2 = asdf.AsdfFile(ff)
ff2.write_to(buff)
assert b"shape: ['*']" in buff.getvalue()
def test_no_asdf_blocks(tmpdir):
content = b"""#ASDF 1.0.0
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
foo: bar
...
XXXXXXXX
"""
path = os.path.join(str(tmpdir), 'test.asdf')
buff = io.BytesIO(content)
with asdf.AsdfFile.open(buff) as ff:
assert len(ff.blocks) == 0
buff.seek(0)
fd = generic_io.InputStream(buff, 'r')
with asdf.AsdfFile.open(fd) as ff:
assert len(ff.blocks) == 0
with open(path, 'wb') as fd:
fd.write(content)
with open(path, 'rb') as fd:
with asdf.AsdfFile.open(fd) as ff:
assert len(ff.blocks) == 0
def test_no_final_newline(tmpdir):
content = b"""#ASDF 1.0.0
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
foo: ...bar...
baz: 42
..."""
path = os.path.join(str(tmpdir), 'test.asdf')
buff = io.BytesIO(content)
with asdf.AsdfFile.open(buff) as ff:
assert len(ff.tree) == 2
buff.seek(0)
fd = generic_io.InputStream(buff, 'r')
with asdf.AsdfFile.open(fd) as ff:
assert len(ff.tree) == 2
with open(path, 'wb') as fd:
fd.write(content)
with open(path, 'rb') as fd:
with asdf.AsdfFile.open(fd) as ff:
assert len(ff.tree) == 2
def test_no_yaml_end_marker(tmpdir):
content = b"""#ASDF 1.0.0
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
foo: bar...baz
baz: 42
"""
path = os.path.join(str(tmpdir), 'test.asdf')
buff = io.BytesIO(content)
with pytest.raises(ValueError):
with asdf.AsdfFile.open(buff):
pass
buff.seek(0)
fd = generic_io.InputStream(buff, 'r')
with pytest.raises(ValueError):
with asdf.AsdfFile.open(fd):
pass
with open(path, 'wb') as fd:
fd.write(content)
with open(path, 'rb') as fd:
with pytest.raises(ValueError):
with asdf.AsdfFile.open(fd):
pass
def test_streams2():
buff = io.BytesIO(b'\0' * 60)
buff.seek(0)
fd = generic_io.InputStream(buff, 'r')
x = fd._peek(10)
x = fd.read()
assert len(x) == 60
def test_exploded_stream_read(tmpdir, small_tree):
# Reading from an exploded input file should fail, but only once
# the data block is accessed. This behavior is important so that
# the tree can still be accessed even if the data is missing.
path = os.path.join(str(tmpdir), 'test.asdf')
ff = asdf.AsdfFile(small_tree)
ff.write_to(path, all_array_storage='external')
with open(path, 'rb') as fd:
# This should work, so we can get the tree content
x = generic_io.InputStream(fd, 'r')
with asdf.open(x) as ff:
# It's only when trying to access external data that an error occurs
with pytest.raises(ValueError):
ff.tree['science_data'][:]
def _roundtrip(tmpdir,
tree,
compression=None,
write_options={},
read_options={}):
tmpfile = os.path.join(str(tmpdir), 'test.asdf')
ff = asdf.AsdfFile(tree)
ff.set_array_compression(tree['science_data'], compression)
ff.write_to(tmpfile, **write_options)
with asdf.AsdfFile.open(tmpfile, mode="rw") as ff:
ff.update(**write_options)
with asdf.AsdfFile.open(tmpfile, **read_options) as ff:
helpers.assert_tree_match(tree, ff.tree)
# Also test saving to a buffer
buff = io.BytesIO()
ff = asdf.AsdfFile(tree)
ff.set_array_compression(tree['science_data'], compression)
ff.write_to(buff, **write_options)
buff.seek(0)
with asdf.AsdfFile.open(buff, **read_options) as ff:
helpers.assert_tree_match(tree, ff.tree)
# Test saving to a non-seekable buffer
buff = io.BytesIO()
ff = asdf.AsdfFile(tree)
ff.set_array_compression(tree['science_data'], compression)
ff.write_to(generic_io.OutputStream(buff), **write_options)
buff.seek(0)
with asdf.AsdfFile.open(generic_io.InputStream(buff),
**read_options) as ff:
helpers.assert_tree_match(tree, ff.tree)
return ff
def test_seek_until_on_block_boundary():
# Create content where the first block begins on a
# file-reading-block boundary.
content = b"""#ASDF 1.0.0
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
foo : bar
...
"""
content += (b'\0' * (io.DEFAULT_BUFFER_SIZE - 2) + constants.BLOCK_MAGIC +
b'\0\x30' + b'\0' * 50)
buff = io.BytesIO(content)
ff = asdf.open(buff)
assert len(ff.blocks) == 1
buff.seek(0)
fd = generic_io.InputStream(buff, 'r')
ff = asdf.open(fd)
assert len(ff.blocks) == 1
def test_stream_to_stream():
tree = {
'nonstream': np.array([1, 2, 3, 4], np.int64),
'stream': stream.Stream([6, 2], np.float64)
}
buff = io.BytesIO()
fd = generic_io.OutputStream(buff)
ff = asdf.AsdfFile(tree)
ff.write_to(fd)
for i in range(100):
fd.write(np.array([i] * 12, np.float64).tostring())
buff.seek(0)
with asdf.AsdfFile().open(generic_io.InputStream(buff, 'r')) as ff:
assert len(ff.blocks) == 2
assert_array_equal(ff.tree['nonstream'], np.array([1, 2, 3, 4], np.int64))
assert ff.tree['stream'].shape == (100, 6, 2)
for i, row in enumerate(ff.tree['stream']):
assert np.all(row == i)
def get_read_fd():
buff.seek(0)
return generic_io.InputStream(buff, 'rw')