async def run():
async with asyncstream.open('samples/animals.txt', mode='rb') as fd:
async with asyncstream.open('samples/animals.txt.gz',
mode='wb',
compression='gzip') as gzfd:
async for line in fd:
await gzfd.write(line)
async def run():
async with asyncstream.open('samples/animals.txt.gz',
'rb',
compression='gzip') as inc_fd:
async with asyncstream.open('samples/animals.txt.snappy',
'wb',
compression='snappy') as outc_fd:
async for line in inc_fd:
await outc_fd.write(line)
async def test_open_read_parquet_with_filename(compression: str):
baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv')
with NamedTemporaryFile(delete=False) as tmpfd:
tmpfd.write(encode_parquet(baby_name_filename, compression))
tmpfd.flush()
async with asyncstream.open(tmpfd.name, mode='rt', encoding='parquet') as fd:
assert get_raw_lines(baby_name_filename) == await async_gen_to_list(fd)
async def run():
session = aiobotocore.get_session()
async with session.create_client('s3') as s3:
obj = await s3.get_object(Bucket='test-bucket', Key='path/to/file.gz')
async with asyncstream.open(obj['Body'], 'rt', compression='bzip2') as fd:
async for line in fd:
print(line)
async def test_open_read(compression: str):
baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv')
with NamedTemporaryFile() as tmpfd:
tmpfd.write(compress(baby_name_filename, compression))
tmpfd.flush()
async with aiofiles.open(tmpfd.name, 'rb') as cfd:
async with asyncstream.open(cfd, mode='rt', compression=compression) as fd:
assert get_raw_lines(baby_name_filename) == await async_gen_to_list(fd)
async def test_writer(compression: str):
baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv')
with NamedTemporaryFile(mode='wb', delete=False) as tmpfd:
async with aiofiles.open(baby_name_filename, 'rb') as rfd:
async with aiofiles.open(tmpfd.name, 'wb') as owfd:
async with asyncstream.open(owfd, mode='wb', compression=compression) as wfd:
async for line in rfd:
await wfd.write(line)
assert get_raw_lines(baby_name_filename) == get_raw_lines(tmpfd.name, compression)
async def test_open_read_with_filename(compression: str):
baby_name_filename = os.path.join(os.path.dirname(__file__), 'data',
'baby_names.csv')
with NamedTemporaryFile() as tmpfd:
async with aiofiles.open(baby_name_filename, 'rb') as cfd:
async with asyncstream.open(tmpfd.name,
mode='wb',
compression=compression) as fd:
await fd.write(await cfd.read())
assert get_raw_lines(baby_name_filename) == get_raw_lines(
tmpfd.name, compression)