def test_registered_open_text_files(s3):
from dask.bytes.core import open_text_files
myfiles = open_text_files('s3://' + test_bucket_name + '/test/accounts.*.json',
s3=s3)
assert len(myfiles) == len(files)
data = compute(*[file.read() for file in myfiles])
assert list(data) == [files[k].decode() for k in sorted(files)]
def test_registered_open_text_files(encoding):
from dask.bytes.core import open_text_files
with filetexts(files, mode='b'):
myfiles = open_text_files('.test.accounts.*', encoding=encoding)
assert len(myfiles) == len(files)
data = compute(*[file.read() for file in myfiles])
assert list(data) == [files[k].decode(encoding) for k in sorted(files)]
def test_registered_open_text_files(encoding):
from dask.bytes.core import open_text_files
with filetexts(files, mode='b'):
myfiles = open_text_files('.test.accounts.*', encoding=encoding)
assert len(myfiles) == len(files)
data = compute(*[file.read() for file in myfiles])
assert list(data) == [files[k].decode(encoding)
for k in sorted(files)]
def test_registered_open_text_files(s3):
myfiles = open_text_files('s3://%s/test/accounts.*.json' % test_bucket_name)
assert len(myfiles) == len(files)
data = []
for file in myfiles:
with file as f:
data.append(f.read())
assert list(data) == [files[k].decode() for k in sorted(files)]
def test_registered_open_text_files(s3):
from dask.bytes.core import open_text_files
myfiles = open_text_files('s3://%s/test/accounts.*.json' %
test_bucket_name,
s3=s3)
assert len(myfiles) == len(files)
data = compute(*[file.read() for file in myfiles])
assert list(data) == [files[k].decode() for k in sorted(files)]
def test_registered_open_text_files(s3):
myfiles = open_text_files('s3://%s/test/accounts.*.json' % test_bucket_name)
assert len(myfiles) == len(files)
data = []
for file in myfiles:
with file as f:
data.append(f.read())
assert list(data) == [files[k].decode() for k in sorted(files)]
def test_compression_text(fmt):
files2 = valmap(compression.compress[fmt], files)
with filetexts(files2, mode='b'):
myfiles = open_text_files('.test.accounts.*', compression=fmt)
data = []
for file in myfiles:
with file as f:
data.append(f.read())
assert list(data) == [files[k].decode() for k in sorted(files)]
def test_compression_text(fmt):
files2 = valmap(compression.compress[fmt], files)
with filetexts(files2, mode='b'):
myfiles = open_text_files('.test.accounts.*', compression=fmt)
data = []
for file in myfiles:
with file as f:
data.append(f.read())
assert list(data) == [files[k].decode() for k in sorted(files)]
