def test_twofile_multiblock(tmpdir):
fn1 = os.path.join(tmpdir, 'one.avro')
fn2 = os.path.join(tmpdir, 'two.avro')
with open(fn1, 'wb') as f:
fastavro.writer(f, records=expected[:500], schema=schema,
sync_interval=100)
with open(fn2, 'wb') as f:
fastavro.writer(f, records=expected[500:], schema=schema,
sync_interval=100)
b = db.read_avro(os.path.join(tmpdir, '*.avro'), blocksize=None)
assert b.npartitions == 2
assert b.compute() == expected
b = db.read_avro(os.path.join(tmpdir, '*.avro'), blocksize=1000)
assert b.npartitions > 2
assert b.compute() == expected
def test_onefile_oneblock(tmpdir):
fn = os.path.join(tmpdir, 'one.avro')
with open(fn, 'wb') as f:
fastavro.writer(f, records=expected, schema=schema)
b = db.read_avro(fn, blocksize=None)
assert b.npartitions == 1
assert b.compute() == expected
def test_onefile_oneblock(tmpdir):
fn = os.path.join(tmpdir, 'one.avro')
with open(fn, 'wb') as f:
fastavro.writer(f, records=expected, schema=schema)
b = db.read_avro(fn, blocksize=None)
assert b.npartitions == 1
assert b.compute() == expected
def test_roundtrip(tmpdir, codec):
if codec == 'snappy':
pytest.importorskip('snappy')
fn = os.path.join(tmpdir, 'out*.avro')
b = db.from_sequence(expected, npartitions=3)
b.to_avro(fn, schema=schema, codec=codec)
b2 = db.read_avro(fn)
assert b.compute() == b2.compute()
def test_roundtrip(tmpdir, codec):
if codec == 'snappy':
pytest.importorskip('snappy')
fn = os.path.join(tmpdir, 'out*.avro')
b = db.from_sequence(expected, npartitions=3)
b.to_avro(fn, schema=schema, codec=codec)
b2 = db.read_avro(fn)
assert b.compute() == b2.compute()
def test_roundtrip(tmpdir, codec):
tmpdir = str(tmpdir)
if codec == "snappy":
pytest.importorskip("snappy")
fn = os.path.join(tmpdir, "out*.avro")
b = db.from_sequence(expected, npartitions=3)
b.to_avro(fn, schema=schema, codec=codec)
b2 = db.read_avro(fn)
assert b.compute() == b2.compute()
def to_dask_by_urlpath(self):
"""Create lazy dask bag object"""
import dask.bag as db
self._get_schema()
actual_urlpath = self._storage_options['actual_urlpath']
del self._storage_options['actual_urlpath']
return db.read_avro(actual_urlpath,
storage_options=self._storage_options)
def test_twofile_oneblock(tmpdir):
fn1 = os.path.join(tmpdir, 'one.avro')
fn2 = os.path.join(tmpdir, 'two.avro')
with open(fn1, 'wb') as f:
fastavro.writer(f, records=expected[:500], schema=schema)
with open(fn2, 'wb') as f:
fastavro.writer(f, records=expected[500:], schema=schema)
b = db.read_avro(os.path.join(tmpdir, '*.avro'), blocksize=None)
assert b.npartitions == 2
assert b.compute() == expected
def test_twofile_oneblock(tmpdir):
tmpdir = str(tmpdir)
fn1 = os.path.join(tmpdir, "one.avro")
fn2 = os.path.join(tmpdir, "two.avro")
with open(fn1, "wb") as f:
fastavro.writer(f, records=expected[:500], schema=schema)
with open(fn2, "wb") as f:
fastavro.writer(f, records=expected[500:], schema=schema)
b = db.read_avro(os.path.join(tmpdir, "*.avro"), blocksize=None)
assert b.npartitions == 2
assert b.compute() == expected
def test_twofile_oneblock(tmpdir):
tmpdir = str(tmpdir)
fn1 = os.path.join(tmpdir, 'one.avro')
fn2 = os.path.join(tmpdir, 'two.avro')
with open(fn1, 'wb') as f:
fastavro.writer(f, records=expected[:500], schema=schema)
with open(fn2, 'wb') as f:
fastavro.writer(f, records=expected[500:], schema=schema)
b = db.read_avro(os.path.join(tmpdir, '*.avro'), blocksize=None)
assert b.npartitions == 2
assert b.compute() == expected
def _get_schema(self):
if self._bag is None:
from dask.bag import read_avro
self._bag = read_avro(self._urlpath,
blocksize=self._bs,
storage_options=self._storage_options)
self.npartitions = self._bag.npartitions
return base.Schema(datashape=None,
dtype=None,
shape=None,
npartitions=self._bag.npartitions,
extra_metadata={})
def test_roundtrip_simple(tmpdir):
from dask.delayed import Delayed
fn = os.path.join(tmpdir, 'out*.avro')
b = db.from_sequence([{'a': i} for i in [1, 2, 3, 4, 5]], npartitions=2)
schema = {
'name': 'Test',
'type': 'record',
'fields': [
{'name': 'a', 'type': 'int'}, ]}
out = b.to_avro(fn, schema, compute=False)
assert isinstance(out[0], Delayed)
out = b.to_avro(fn, schema)
assert len(out) == 2
b2 = db.read_avro(fn)
assert b.compute() == b2.compute()
def test_roundtrip_simple(tmpdir):
from dask.delayed import Delayed
tmpdir = str(tmpdir)
fn = os.path.join(tmpdir, "out*.avro")
b = db.from_sequence([{"a": i} for i in [1, 2, 3, 4, 5]], npartitions=2)
schema = {
"name": "Test",
"type": "record",
"fields": [{
"name": "a",
"type": "int"
}],
}
out = b.to_avro(fn, schema, compute=False)
assert isinstance(out[0], Delayed)
out = b.to_avro(fn, schema)
assert len(out) == 2
b2 = db.read_avro(fn)
assert b.compute() == b2.compute()
def test_roundtrip_simple(tmpdir):
from dask.delayed import Delayed
fn = os.path.join(tmpdir, 'out*.avro')
b = db.from_sequence([{'a': i} for i in [1, 2, 3, 4, 5]], npartitions=2)
schema = {
'name': 'Test',
'type': 'record',
'fields': [
{
'name': 'a',
'type': 'int'
},
]
}
out = b.to_avro(fn, schema, compute=False)
assert isinstance(out[0], Delayed)
out = b.to_avro(fn, schema)
assert len(out) == 2
b2 = db.read_avro(fn)
assert b.compute() == b2.compute()