def test_stack(stack_data):
descriptors = [CSV(fn, schema='2 * int32') for fn in sorted(stack_data)]
dd = Stack(descriptors)
assert dd.dshape == 3 * descriptors[0].dshape
expected = (((1, 1), (2, 2)),
((3, 3), (4, 4)),
((5, 5), (6, 6)))
assert tuplify(tuple(dd.as_py())) == expected
result = dd.as_dynd()
expected2 = nd.array(expected, dtype='int32')
assert nd.as_py(result) == nd.as_py(expected2)
assert tuplify(tuple(dd)) == expected
assert tuplify(tuple(dd)) == expected # Not one use only
chunks = dd.chunks()
assert all(isinstance(chunk, nd.array) for chunk in chunks)
assert tuple(dd[[0, 2], 0, 0]) == (1, 5)
assert tuplify(tuple(dd[0])) == ((1, 1), (2, 2))
res = dd[0, :, [1]]
x = tuple(res)
assert tuplify(x) == ((1,), (2,))
assert tuplify(tuple(dd[0])) == expected[0]
assert isinstance(dd[:, 0], Iterator)
assert isinstance(dd[:], Iterator)
def test_Stack(self):
with filetexts(self.data) as filenames:
descriptors = [CSV(fn, schema='2 * int32')
for fn in sorted(filenames)]
dd = Stack(descriptors)
self.assertEqual(dd.dshape, 3 * descriptors[0].dshape)
expected = (((1, 1), (2, 2)),
((3, 3), (4, 4)),
((5, 5), (6, 6)))
self.assertEqual(tuplify(tuple(dd.as_py())), expected)
result = dd.as_dynd()
expected2 = nd.array(expected, dtype='int32')
self.assertEqual(nd.as_py(result),
nd.as_py(expected2))
self.assertEqual(tuplify(tuple(dd)), expected)
self.assertEqual(tuplify(tuple(dd)), expected) # Not one use only
chunks = dd.chunks()
assert all(isinstance(chunk, nd.array) for chunk in chunks)
self.assertEqual(tuple(dd[[0, 2], 0, 0]), (1, 5))
self.assertEqual(tuplify(tuple(dd[0])), ((1, 1), (2, 2)))
self.assertEqual(tuplify(tuple(dd[0, :, [1]])), ((1,), (2,)))
self.assertEqual(tuplify(tuple(dd[0])), expected[0])
assert isinstance(dd[:, 0], Iterator)
assert isinstance(dd[:], Iterator)
def test_stack(stack_data):
descriptors = [CSV(fn, schema='2 * int32') for fn in sorted(stack_data)]
dd = Stack(descriptors)
assert dd.dshape == 3 * descriptors[0].dshape
expected = (((1, 1), (2, 2)), ((3, 3), (4, 4)), ((5, 5), (6, 6)))
assert tuplify(tuple(dd.as_py())) == expected
result = dd.as_dynd()
expected2 = nd.array(expected, dtype='int32')
assert nd.as_py(result) == nd.as_py(expected2)
assert tuplify(tuple(dd)) == expected
assert tuplify(tuple(dd)) == expected # Not one use only
chunks = dd.chunks()
assert all(isinstance(chunk, nd.array) for chunk in chunks)
assert tuple(dd[[0, 2], 0, 0]) == (1, 5)
assert tuplify(tuple(dd[0])) == ((1, 1), (2, 2))
res = dd[0, :, [1]]
x = tuple(res)
assert tuplify(x) == ((1, ), (2, ))
assert tuplify(tuple(dd[0])) == expected[0]
assert isinstance(dd[:, 0], Iterator)
assert isinstance(dd[:], Iterator)
def test_Stack(self):
with filetexts(self.text) as filenames:
descriptors = [JSON_Streaming(fn, schema="{x: int32, y: int32}") for fn in sorted(filenames)]
dd = Stack(descriptors)
expected = (((1, 2), (3, 4)), ((5, 6), (7, 8)), ((9, 10), (11, 12)))
self.assertEqual(tuplify(dd.as_py()), expected)
self.assertEqual(tuplify(dd.py[::2, 1, :]), ((3, 4), (11, 12)))
self.assertEqual(tuplify(dd.py[::2, 1, "x"]), (3, 11))
def json_data():
data = {
'a.csv': [{
'x': 1,
'y': 2
}, {
'x': 3,
'y': 4
}],
'b.csv': [{
'x': 5,
'y': 6
}, {
'x': 7,
'y': 8
}],
'c.csv': [{
'x': 9,
'y': 10
}, {
'x': 11,
'y': 12
}]
}
text = dict(
(fn, '\n'.join(map(json.dumps, dicts))) for fn, dicts in data.items())
with filetexts(text) as filenames:
descriptors = [
JSON_Streaming(fn, schema='{x: int32, y: int32}')
for fn in sorted(filenames)
]
yield Stack(descriptors)
def test_gzip_json_files(self):
with filetexts(texts, open=gzip.open) as filenames:
descriptors = [JSON(fn, dshape=schema, open=gzip.open)
for fn in sorted(filenames)]
dd = Stack(descriptors)
self.assertEqual(sorted(dd), sorted(tuples))
self.assertEqual(dd.schema, dshape(schema))
