def test_convert_logfiles_to_bag():
with filetexts({'a1.log': 'Hello\nWorld', 'a2.log': 'Hola\nMundo'}) as fns:
logs = chunks(TextFile)(list(map(TextFile, fns)))
b = convert(Bag, logs)
assert isinstance(b, Bag)
assert 'a1.log' in str(b.dask.values())
assert convert(list, b) == convert(list, logs)
def test_first_csv_establishes_consistent_dshape():
d = {'accounts1.csv': 'name,when\nAlice,one\nBob,two',
'accounts2.csv': 'name,when\nAlice,300\nBob,400'}
with filetexts(d) as fns:
L = into(list, 'accounts*.csv')
assert len(L) == 4
assert all(isinstance(val, (str, unicode)) for name, val in L)
def test_glob():
d = {'accounts1.csv': 'name,when\nAlice,100\nBob,200',
'accounts2.csv': 'name,when\nAlice,300\nBob,400'}
with filetexts(d) as fns:
r = resource('accounts*.csv', has_header=True)
assert convert(list, r) == [('Alice', 100), ('Bob', 200),
('Alice', 300), ('Bob', 400)]
def test_first_csv_establishes_consistent_dshape():
d = {
'accounts1.csv': 'name,when\nAlice,one\nBob,two',
'accounts2.csv': 'name,when\nAlice,300\nBob,400'
}
with filetexts(d) as fns:
L = into(list, 'accounts*.csv')
assert len(L) == 4
assert all(isinstance(val, (str, unicode)) for name, val in L)
def test_glob():
d = {
'accounts1.csv': 'name,when\nAlice,100\nBob,200',
'accounts2.csv': 'name,when\nAlice,300\nBob,400'
}
with filetexts(d) as fns:
r = resource('accounts*.csv', has_header=True)
assert convert(list, r) == [('Alice', 100), ('Bob', 200),
('Alice', 300), ('Bob', 400)]
def test_chunks_textfile():
with filetexts({'a1.log': 'Hello\nWorld', 'a2.log': 'Hola\nMundo'}) as fns:
logs = chunks(TextFile)(list(map(TextFile, fns)))
assert set(map(str.strip, convert(list, logs))) == \
set(['Hello', 'World', 'Hola', 'Mundo'])
def test_from_filenames():
with filetexts({'a1.log': 'A\nB', 'a2.log': 'C\nD'}) as fns:
assert set(line.strip() for line in db.from_filenames(fns)) == \
set('ABCD')
assert set(line.strip() for line in db.from_filenames('a*.log')) == \
set('ABCD')
