def test_temp_csv():
csv = into(Temp(CSV)('_test_temp_csv.csv'), df)
assert isinstance(csv, CSV)
assert into(list, csv) == into(list, df)
del csv
import gc
gc.collect()
assert not os.path.exists('_test_temp_csv.csv')
def test_convert_to_csv():
csv = into(Temp(CSV), df)
assert isinstance(csv, CSV)
assert into(list, csv) == into(list, df)
assert isinstance(csv, _Temp)
def iterator_to_temporary_jsonlines(data, **kwargs):
fn = '.%s.json' % uuid.uuid1()
target = Temp(JSONLines)(fn)
return append(target, data, **kwargs)
from __future__ import absolute_import, division, print_function
import uuid
from odo import JSONLines, Temp, convert, append
from collections import Iterator
@convert.register(Temp(JSONLines), Iterator)
def iterator_to_temporary_jsonlines(data, **kwargs):
fn = '.%s.json' % uuid.uuid1()
target = Temp(JSONLines)(fn)
return append(target, data, **kwargs)
def list_to_temporary_bson(data, **kwargs):
fn = '.%s.bson' % uuid.uuid1()
target = Temp(BSON)(fn)
return append(target, data, **kwargs)
@discover.register(BSON)
def discover_bson(b, n=10, **kwargs):
with bson_lines(b.path) as lines:
data = list(take(n, lines))
if len(data) < n:
ds = discover(data)
else:
ds = var * discover(data).subshape[0]
return ds
# return date_to_datetime_dshape(ds)
@convert.register(list, (BSON, Temp(BSON)))
def bson_to_list(b, dshape=None, **kwargs):
with bson_lines(b.path) as lines:
return list(lines)
@convert.register(Iterator, (BSON, Temp(BSON)))
def bson_to_iterator(b, **kwargs):
with bson_lines(b.path, **kwargs) as bs:
for line in bs:
yield line
@append.register(BSON, object)
def object_to_bson(b, o, **kwargs):
return append(b, convert(Iterator, o, **kwargs), **kwargs)
def pre_compute(expr, data, **kwargs):
return pre_compute(expr, into(Temp(CSV), data, **kwargs), **kwargs)