def test__read_text(e, s, a, b):
with make_hdfs() as hdfs:
with hdfs.open('/tmp/test/text.1.txt', 'wb') as f:
f.write('Alice 100\nBob 200\nCharlie 300'.encode())
with hdfs.open('/tmp/test/text.2.txt', 'wb') as f:
f.write('Dan 400\nEdith 500\nFrank 600'.encode())
with hdfs.open('/tmp/test/other.txt', 'wb') as f:
f.write('a b\nc d'.encode())
b = read_text('/tmp/test/text.*.txt', collection=True, lazy=True)
yield gen.sleep(0.5)
assert not s.tasks
future = e.compute(b.str.strip().str.split().map(len))
result = yield future._result()
assert result == [2, 2, 2, 2, 2, 2]
b = read_text('/tmp/test/other.txt', collection=True, lazy=False)
future = e.compute(b.str.split().concat())
result = yield future._result()
assert result == ['a', 'b', 'c', 'd']
L = read_text('/tmp/test/text.*.txt', collection=False, lazy=False)
assert all(isinstance(x, Future) for x in L)
L = read_text('/tmp/test/text.*.txt', collection=False, lazy=True)
assert all(isinstance(x, Value) for x in L)
def test__read_text(e, s, a, b):
with make_hdfs() as hdfs:
with hdfs.open('/tmp/test/text.1.txt', 'wb') as f:
f.write('Alice 100\nBob 200\nCharlie 300'.encode())
with hdfs.open('/tmp/test/text.2.txt', 'wb') as f:
f.write('Dan 400\nEdith 500\nFrank 600'.encode())
with hdfs.open('/tmp/test/other.txt', 'wb') as f:
f.write('a b\nc d'.encode())
b = read_text('/tmp/test/text.*.txt',
collection=True, lazy=True)
yield gen.sleep(0.5)
assert not s.tasks
future = e.compute(b.str.strip().str.split().map(len))
result = yield future._result()
assert result == [2, 2, 2, 2, 2, 2]
b = read_text('/tmp/test/other.txt',
collection=True, lazy=False)
future = e.compute(b.str.split().concat())
result = yield future._result()
assert result == ['a', 'b', 'c', 'd']
L = read_text('/tmp/test/text.*.txt',
collection=False, lazy=False)
assert all(isinstance(x, Future) for x in L)
L = read_text('/tmp/test/text.*.txt',
collection=False, lazy=True)
assert all(isinstance(x, Value) for x in L)
def test_read_text_sync(loop):
with make_hdfs() as hdfs:
with hdfs.open('/tmp/test/data.txt', 'wb') as f:
f.write(b'hello\nworld')
with cluster(nworkers=3) as (s, [a, b, c]):
with Executor(('127.0.0.1', s['port']), loop=loop) as e:
b = read_text('/tmp/test/*.txt', lazy=False)
assert list(b.str.upper()) == ['HELLO', 'WORLD']
def test_read_text_sync(loop):
with make_hdfs() as hdfs:
with hdfs.open('/tmp/test/data.txt', 'wb') as f:
f.write(b'hello\nworld')
with cluster(nworkers=3) as (s, [a, b, c]):
with Executor(('127.0.0.1', s['port']), loop=loop) as e:
b = read_text('/tmp/test/*.txt', lazy=False)
assert list(b.str.upper()) == ['HELLO', 'WORLD']
def test__read_text_json_endline(e, s, a):
import json
with make_hdfs() as hdfs:
with hdfs.open('/tmp/test/text.1.txt', 'wb') as f:
f.write(b'{"x": 1}\n{"x": 2}\n')
b = read_text('/tmp/test/text.1.txt').map(json.loads)
result = yield e.compute(b)._result()
assert result == [{"x": 1}, {"x": 2}]
def test__read_text_json_endline(e, s, a):
import json
with make_hdfs() as hdfs:
with hdfs.open('/tmp/test/text.1.txt', 'wb') as f:
f.write(b'{"x": 1}\n{"x": 2}\n')
b = read_text('/tmp/test/text.1.txt').map(json.loads)
result = yield e.compute(b)._result()
assert result == [{"x": 1}, {"x": 2}]
def test__read_text_unicode(e, s, a, b):
fn = '/tmp/test/data.txt'
data = b'abcd\xc3\xa9'
with make_hdfs() as hdfs:
with hdfs.open(fn, 'wb') as f:
f.write(b'\n'.join([data, data]))
f = read_text(fn, collection=False, lazy=False)
result = yield f[0]._result()
assert len(result) == 2
assert list(map(unicode.strip, result)) == [data.decode('utf-8')] * 2
assert len(result[0]) == 5
def test__read_text_unicode(e, s, a, b):
fn = '/tmp/test/data.txt'
data = b'abcd\xc3\xa9'
with make_hdfs() as hdfs:
with hdfs.open(fn, 'wb') as f:
f.write(b'\n'.join([data, data]))
f = read_text(fn, collection=False, lazy=False)
result = yield f[0]._result()
assert len(result) == 2
assert list(map(unicode.strip, result)) == [data.decode('utf-8')] * 2
assert len(result[0]) == 5