def hive_table(host):
name = ('temp' + str(uuid.uuid1()).replace('-', ''))[:30]
uri = 'hive://hdfs@%s:10000/default::%s' % (host, name)
try:
yield uri
finally:
with ignoring(Exception):
drop(uri)
def hive_table(host):
name = ('temp' + str(uuid.uuid1()).replace('-', ''))[:30]
uri = 'hive://hdfs@%s:10000/default::%s' % (host, name)
try:
yield uri
finally:
with ignoring(Exception):
drop(uri)
def test_multiple_jsonlines():
a, b = '_test_a1.json', '_test_a2.json'
try:
with ignoring(OSError):
os.remove(a)
with ignoring(OSError):
os.remove(b)
with open(a, 'w') as f:
json.dump(dat, f)
with open(b'_test_a2.json', 'w') as f:
json.dump(dat, f)
r = resource('_test_a*.json')
result = convert(list, r)
assert len(result) == len(dat) * 2
finally:
with ignoring(OSError):
os.remove(a)
with ignoring(OSError):
os.remove(b)
def test_multiple_jsonlines():
a, b = '_test_a1.json', '_test_a2.json'
try:
with ignoring(OSError):
os.remove(a)
with ignoring(OSError):
os.remove(b)
with open(a, 'w') as f:
json.dump(dat, f)
with open(b'_test_a2.json', 'w') as f:
json.dump(dat, f)
r = resource('_test_a*.json')
result = convert(list, r)
assert len(result) == len(dat) * 2
finally:
with ignoring(OSError):
os.remove(a)
with ignoring(OSError):
os.remove(b)
def test_into_sqlite():
data = [('Alice', 100), ('Bob', 200)]
ds = datashape.dshape('var * {name: string, amount: int}')
with tmpfile('.db') as dbpath:
with tmpfile('.csv') as csvpath:
csv = into(csvpath, data, dshape=ds, has_header=False)
sql = resource('sqlite:///%s::mytable' % dbpath, dshape=ds)
with ignoring(NotImplementedError):
append_csv_to_sql_table(sql, csv)
assert into(list, sql) == data
def tmpbcolz(*args, **kwargs):
fn = '.%s.bcolz' % str(uuid.uuid1())
r = resource(fn, *args, **kwargs)
try:
yield r
finally:
with ignoring(Exception):
r.flush()
if os.path.exists(fn):
shutil.rmtree(fn)
def file(x):
with tmpfile('.hdf5') as fn:
f = h5py.File(fn)
data = f.create_dataset('/data', data=x, chunks=True,
maxshape=(None,) + x.shape[1:])
try:
yield fn, f, data
finally:
with ignoring(Exception):
f.close()
def test_drop_group():
with tmpfile('.hdf5') as fn:
f = h5py.File(fn)
try:
f.create_dataset('/group/data', data=x, chunks=True,
maxshape=(None,) + x.shape[1:])
drop(f['/group'])
assert '/group' not in f.keys()
finally:
with ignoring(Exception):
f.close()
def test_into_sqlite():
data = [('Alice', 100), ('Bob', 200)]
ds = datashape.dshape('var * {name: string, amount: int}')
with tmpfile('.db') as dbpath:
with tmpfile('.csv') as csvpath:
csv = into(csvpath, data, dshape=ds, has_header=False)
sql = resource('sqlite:///%s::mytable' % dbpath, dshape=ds)
with ignoring(NotImplementedError):
append_csv_to_sql_table(sql, csv)
assert into(list, sql) == data
def tmpbcolz(*args, **kwargs):
fn = '.%s.bcolz' % str(uuid.uuid1())
r = resource(fn, *args, **kwargs)
try:
yield r
finally:
with ignoring(Exception):
r.flush()
if os.path.exists(fn):
shutil.rmtree(fn)
def test_drop_group():
with tmpfile('.hdf5') as fn:
f = h5py.File(fn)
try:
f.create_dataset('/group/data',
data=x,
chunks=True,
maxshape=(None, ) + x.shape[1:])
drop(f['/group'])
assert '/group' not in f.keys()
finally:
with ignoring(Exception):
f.close()
def file(x):
with tmpfile('.hdf5') as fn:
f = h5py.File(fn)
data = f.create_dataset('/data',
data=x,
chunks=True,
maxshape=(None, ) + x.shape[1:])
try:
yield fn, f, data
finally:
with ignoring(Exception):
f.close()
def jslines(n=3):
d = tempfile.mkdtemp()
files = []
dfc = df.copy()
for i in range(n):
_, fn = tempfile.mkstemp(suffix='.json', dir=d)
dfc['id'] += i
odo(dfc, 'jsonlines://%s' % fn)
files.append(fn)
yield d
with ignoring(OSError):
shutil.rmtree(d)
def jslines(n=3):
d = tempfile.mkdtemp()
files = []
dfc = df.copy()
for i in range(n):
_, fn = tempfile.mkstemp(suffix='.json', dir=d)
dfc['id'] += i
odo(dfc, 'jsonlines://%s' % fn)
files.append(fn)
yield d
with ignoring(OSError):
shutil.rmtree(d)
def test_discover_on_data_with_object_in_record_name():
data = np.array([(u'a', 1), (u'b', 2)],
dtype=[('lrg_object', unicode_dtype), ('an_int', 'int64')])
with tmpfile('.hdf5') as fn:
f = h5py.File(fn)
try:
f.create_dataset('/data', data=data)
except:
raise
else:
assert (discover(f['data']) == datashape.dshape(
'2 * {lrg_object: string, an_int: int64}'))
finally:
with ignoring(Exception):
f.close()
def test_discover_on_data_with_object_in_record_name():
data = np.array([(u'a', 1), (u'b', 2)], dtype=[('lrg_object',
unicode_dtype),
('an_int', 'int64')])
with tmpfile('.hdf5') as fn:
f = h5py.File(fn)
try:
f.create_dataset('/data', data=data)
except:
raise
else:
assert (discover(f['data']) ==
datashape.dshape('2 * {lrg_object: string, an_int: int64}'))
finally:
with ignoring(Exception):
f.close()
def _spider(resource_path, ignore, followlinks, hidden):
resources = {}
for filename in (os.path.join(resource_path, x)
for x in os.listdir(resource_path)):
basename = os.path.basename(filename)
if (basename.startswith(os.curdir) and not hidden or
os.path.islink(filename) and not followlinks):
continue
if os.path.isdir(filename):
new_resources = _spider(filename, ignore=ignore,
followlinks=followlinks, hidden=hidden)
if new_resources:
resources[basename] = new_resources
else:
with ignoring(*ignore):
resources[basename] = resource(filename)
return resources
from pandas import DataFrame, Series, Timestamp
from .expr import Expr, Symbol, ndim
from .dispatch import dispatch
from .compatibility import _strtypes
__all__ = ['Data', 'Table', 'into', 'to_html']
names = ('_%d' % i for i in itertools.count(1))
not_an_iterator = []
with ignoring(ImportError):
import bcolz
not_an_iterator.append(bcolz.carray)
with ignoring(ImportError):
import pymongo
not_an_iterator.append(pymongo.collection.Collection)
not_an_iterator.append(pymongo.database.Database)
class InteractiveSymbol(Symbol):
"""Interactive data.
The ``Data`` object presents a familiar view onto a variety of forms of
data. This user-level object provides an interactive experience to using
import numpy as np
from odo import resource, odo
from odo.utils import ignoring, copydoc
from odo.compatibility import unicode
from pandas import DataFrame, Series, Timestamp
from .expr import Expr, Symbol, ndim
from .dispatch import dispatch
from .compatibility import _strtypes
__all__ = ['Data', 'Table', 'into', 'to_html']
names = ('_%d' % i for i in itertools.count(1))
not_an_iterator = []
with ignoring(ImportError):
import bcolz
not_an_iterator.append(bcolz.carray)
with ignoring(ImportError):
import pymongo
not_an_iterator.append(pymongo.collection.Collection)
not_an_iterator.append(pymongo.database.Database)
class InteractiveSymbol(Symbol):
"""Interactive data.
The ``Data`` object presents a familiar view onto a variety of forms of
data. This user-level object provides an interactive experience to using
Blaze's abstract expressions.