def test_registered_open_text_files(s3): from dask.bytes.core import open_text_files myfiles = open_text_files('s3://' + test_bucket_name + '/test/accounts.*.json', s3=s3) assert len(myfiles) == len(files) data = compute(*[file.read() for file in myfiles]) assert list(data) == [files[k].decode() for k in sorted(files)]
def test_registered_open_text_files(encoding): from dask.bytes.core import open_text_files with filetexts(files, mode='b'): myfiles = open_text_files('.test.accounts.*', encoding=encoding) assert len(myfiles) == len(files) data = compute(*[file.read() for file in myfiles]) assert list(data) == [files[k].decode(encoding) for k in sorted(files)]
def test_registered_open_text_files(s3): myfiles = open_text_files('s3://%s/test/accounts.*.json' % test_bucket_name) assert len(myfiles) == len(files) data = [] for file in myfiles: with file as f: data.append(f.read()) assert list(data) == [files[k].decode() for k in sorted(files)]
def test_registered_open_text_files(s3): from dask.bytes.core import open_text_files myfiles = open_text_files('s3://%s/test/accounts.*.json' % test_bucket_name, s3=s3) assert len(myfiles) == len(files) data = compute(*[file.read() for file in myfiles]) assert list(data) == [files[k].decode() for k in sorted(files)]
def test_compression_text(fmt): files2 = valmap(compression.compress[fmt], files) with filetexts(files2, mode='b'): myfiles = open_text_files('.test.accounts.*', compression=fmt) data = [] for file in myfiles: with file as f: data.append(f.read()) assert list(data) == [files[k].decode() for k in sorted(files)]