def test_argmax_2d(self):
for axis in [1]: #[None, 0, 1]:
x = expr.arange((TEST_SIZE, TEST_SIZE), dtype=np.int)
nx = np.arange(TEST_SIZE * TEST_SIZE, dtype=np.int).reshape((TEST_SIZE, TEST_SIZE))
y = x.argmax(axis=axis)
val = expr.glom(y)
Assert.all_eq(val, nx.argmax(axis=axis))
def test_reshape3(self): a = expr.arange((100, 100)) b = expr.reshape(a, (10000,)) c = expr.reshape(b, (10000, 1)) d = expr.reshape(c, (1, 10000)) e = expr.arange((1, 10000)) Assert.all_eq(d.glom(), e.glom())
def test_slices_from_slices(self):
slices1 = (slice(0, 50), slice(0, 50))
slices2 = (slice(0, 50), slice(50, 100))
slices3 = (slice(50, 100), slice(0, 50))
slices4 = (slice(50, 100), slice(50, 100))
t1 = expr.randn(100, 100)
t2 = expr.randn(100, 100)
t3 = expr.write(t2, slices1, t1, slices1)
t4 = expr.write(t3, slices2, t1, slices2)
t5 = expr.write(t4, slices3, t1, slices3)
t6 = expr.write(t5, slices4, t1, slices4)
Assert.all_eq(t1.glom(), t6.glom())
dst = np.arange(0, 2500).reshape(50, 50)
t11 = expr.write(t1, slices1, dst, slices1)
t12 = expr.write(t11, slices2, dst, slices1)
t13 = expr.write(t12, slices3, dst, slices1)
t14 = expr.write(t13, slices4, dst, slices1)
tmp = expr.write(expr.randn(100, 100), slices4, dst, slices1)
t21 = expr.write(t2, slices1, tmp, slices4)
t22 = expr.write(t21, slices2, tmp, slices4)
t23 = expr.write(t22, slices3, tmp, slices4)
t24 = expr.write(t23, slices4, tmp, slices4)
Assert.all_eq(t14.glom(), t24.glom())
def test_fio_path(self):
self.create_path()
t1 = expr.randn(100, 100).evaluate()
expr.save(t1, "fiotest1", self.test_dir2, False)
expr.pickle(t1, "fiotest2", self.test_dir2, False)
Assert.all_eq(t1.glom(), expr.load("fiotest1", self.test_dir2, False).glom())
Assert.all_eq(t1.glom(), expr.unpickle("fiotest2", self.test_dir2, False).glom())
def test_creation(self):
A = spartan.eye(100, 10)
nA = np.eye(100, 10)
B = spartan.identity(100)
nB = np.identity(100)
Assert.all_eq(A.glom(), nA)
Assert.all_eq(B.glom(), nB)
def test_sum_2d(self):
x = expr.arange((TEST_SIZE, TEST_SIZE), dtype=np.int)
nx = np.arange(TEST_SIZE * TEST_SIZE, dtype=np.int).reshape((TEST_SIZE, TEST_SIZE))
for axis in [None, 0, 1]:
y = x.sum(axis)
val = y.glom()
Assert.all_eq(val, nx.sum(axis))
def _test_optimization_ordered(self):
na = np.random.rand(1000, 1000)
nb = np.random.rand(1000, 1000)
a = expr.from_numpy(na)
b = expr.from_numpy(nb)
c = a - b
d = a + c
f = c[200:900, 200:900]
g = d[200:900, 200:900]
h = f - g
i = f + h
j = h[100:500, 100:500]
k = i[100:500, 100:500]
l = expr.dot(j, k)
m = j + k
n = k - l
o = n - m
q = o[100:200, 100:200]
nc = na - nb
nd = na + nc
nf = nc[200:900, 200:900]
ng = nd[200:900, 200:900]
nh = nf - ng
ni = nf + nh
nj = nh[100:500, 100:500]
nk = ni[100:500, 100:500]
nl = np.dot(nj, nk)
nm = nj + nk
nn = nk - nl
no = nn - nm
nq = no[100:200, 100:200]
Assert.all_eq(nq, q.optimized().glom(), tolerance = 1e-10)
def test_slice_reduce(self): x = expr.arange((TEST_SIZE, TEST_SIZE, TEST_SIZE), dtype=np.int) nx = np.arange(TEST_SIZE * TEST_SIZE * TEST_SIZE, dtype=np.int).reshape((TEST_SIZE, TEST_SIZE, TEST_SIZE)) y = x[:, :, 0].sum() val = y.glom() Assert.all_eq(val, nx[:, :, 0].sum())
def test_reshape5(self): a = expr.arange((35511, )) b = expr.reshape(a, (133, 267)) c = expr.reshape(b, (267, 133)) d = expr.reshape(c, (1, 35511)) e = expr.arange((1, 35511)) Assert.all_eq(d.glom(), e.glom())
def test_assign_array_like(self): a = np.zeros((20, 10)) b = np.ones((10, )) region = np.s_[10, ] sp_a = assign(from_numpy(a), region, b).glom() a[region] = b Assert.all_eq(sp_a, a)
def test_transpose3(self): t1 = expr.sparse_diagonal((107, 401)).evaluate() t2 = expr.sparse_diagonal((401, 107)).evaluate() a = expr.transpose(t1) b = expr.transpose(t2) Assert.all_eq(a.glom().todense(), sp.eye(107, 401).transpose().todense()) Assert.all_eq(b.glom().todense(), sp.eye(401, 107).transpose().todense())
def test_reshape6(self): a = expr.arange((12319, )) b = expr.reshape(a, (127, 97)) c = expr.reshape(b, (97, 127)) d = expr.reshape(c, (1, 12319)) e = expr.arange((1, 12319)) Assert.all_eq(d.glom(), e.glom())
def test_reshape8(self): t1 = expr.sparse_diagonal((137, 113)) t2 = expr.sparse_diagonal((113, 137)) a = expr.reshape(t1, (113, 137)) b = expr.reshape(t2, (137, 113)) Assert.all_eq(a.glom().todense(), sp.eye(137, 113).tolil().reshape((113, 137)).todense()) Assert.all_eq(b.glom().todense(), sp.eye(113, 137).tolil().reshape((137, 113)).todense())
def test_optimization_map_with_location(self):
FLAGS.opt_parakeet_gen = 1
def mapper(tile, ex):
return tile + 10
a = expr.map_with_location(expr.ones((5, 5)), mapper) + expr.ones((5, 5))
Assert.isinstance(a.optimized().op, expr.local.ParakeetExpr)
def test_argmin_2d(self):
for axis in [1]: #[None, 0, 1]:
x = expr.arange((TEST_SIZE, TEST_SIZE), dtype=np.int)
nx = np.arange(TEST_SIZE * TEST_SIZE, dtype=np.int).reshape(
(TEST_SIZE, TEST_SIZE))
y = x.argmin(axis=axis)
val = expr.glom(y)
Assert.all_eq(val, nx.argmin(axis=axis))
def test_sum_2d(self):
x = expr.arange((TEST_SIZE, TEST_SIZE), dtype=np.int)
nx = np.arange(TEST_SIZE * TEST_SIZE, dtype=np.int).reshape(
(TEST_SIZE, TEST_SIZE))
for axis in [None, 0, 1]:
y = x.sum(axis)
val = y.glom()
Assert.all_eq(val, nx.sum(axis))
def test_argmax_3d(self):
x = expr.arange((TEST_SIZE, TEST_SIZE, TEST_SIZE), dtype=np.int64)
nx = np.arange(TEST_SIZE * TEST_SIZE * TEST_SIZE, dtype=np.int64).reshape((TEST_SIZE, TEST_SIZE, TEST_SIZE))
for axis in [None, 0, 1, 2]:
y = x.argmax(axis)
val = y.glom()
Assert.all_eq(val, nx.argmax(axis))
def _step(): yp = expr.dot(x, w) Assert.all_eq(yp.shape, y.shape) diff = x * (yp - y) grad = expr.sum(diff, axis=0).glom().reshape((N_DIM, 1)) wprime = w - grad * 1e-6 expr.force(wprime)
def test_numpy_vec_vec(self):
av = expr.arange(stop=100)
bv = np.arange(100)
na = np.arange(100)
nb = np.arange(100)
Assert.all_eq(expr.dot(av, bv).glom(),
np.dot(na, nb))
def test_slice_shuffle(self): x = expr.arange((TEST_SIZE, TEST_SIZE)) z = x[5:8, 5:8] z = expr.shuffle(z, add_one_extent) val = z.force() nx = np.arange(TEST_SIZE*TEST_SIZE).reshape(TEST_SIZE, TEST_SIZE) Assert.all_eq(val.glom(), nx[5:8, 5:8] + 1)
def test_numpy_2d_vec(self):
av = expr.arange((77, 100))
bv = np.arange(100)
na = np.arange(7700).reshape(77, 100)
nb = np.arange(100)
Assert.all_eq(expr.dot(av, bv).glom(),
np.dot(na, nb))
def test_update_sparse_to_sparse(self):
t = tile.from_shape(ARRAY_SIZE, dtype=np.float32, tile_type=tile.TYPE_SPARSE)
update = sp.lil_matrix(ARRAY_SIZE, dtype=np.float32)
for i in range(UPDATE_SHAPE[0]):
update[i,i] = 1
t.update(UPDATE_SUBSLICE, update, None)
Assert.eq(sp.issparse(t.data), True)
print t.data.todense()
def test_numpy_vec_2d(self):
av = expr.arange(stop = 100)
bv = np.arange(7700).reshape(100, 77)
na = np.arange(100)
nb = np.arange(7700).reshape(100, 77)
Assert.all_eq(expr.dot(av, bv).glom(),
np.dot(na, nb))
def test_slice_map(self): x = expr.arange((TEST_SIZE, TEST_SIZE)) z = x[5:8, 5:8] z = expr.map(z, add_one_tile) print z nx = np.arange(TEST_SIZE*TEST_SIZE).reshape(TEST_SIZE, TEST_SIZE) Assert.all_eq(z.glom(), nx[5:8, 5:8] + 1)
def test_diag(self):
import random
dim = random.randint(0, 99)
np_array = np.random.randn(dim, dim)
Assert.all_eq(spartan.diag(spartan.from_numpy(np_array)).glom(), np.diag(np_array))
np_array2 = np.random.randn(dim, dim)
Assert.all_eq(spartan.diag(spartan.diag(spartan.from_numpy(np_array2))).glom(), np.diag(np.diag(np_array2)))
def test_reshape8(self):
t1 = expr.sparse_diagonal((137, 113))
t2 = expr.sparse_diagonal((113, 137))
a = expr.reshape(t1, (113, 137))
b = expr.reshape(t2, (137, 113))
Assert.all_eq(a.glom().todense(),
sp.eye(137, 113).tolil().reshape((113, 137)).todense())
Assert.all_eq(b.glom().todense(),
sp.eye(113, 137).tolil().reshape((137, 113)).todense())
def test_reshape4(self):
a = expr.arange((10000, ))
b = expr.reshape(a, (10, 1000))
c = expr.reshape(b, (1000, 10))
d = expr.reshape(c, (20, 500))
e = expr.reshape(d, (500, 20))
f = expr.reshape(e, (1, 10000))
g = expr.arange((1, 10000))
Assert.all_eq(f.glom(), g.glom())
def test_add_many(self):
a = expr.ones((TEST_SIZE, TEST_SIZE))
b = expr.ones((TEST_SIZE, TEST_SIZE))
add_many = (a + b + a + b + a + b + a + b + a + b)
#print add_many
#print add_many.dag()
Assert.all_eq(add_many.glom(),
np.ones((TEST_SIZE, TEST_SIZE)) * 10)
def test_optimization_map_with_location(self):
FLAGS.opt_parakeet_gen = 1
def mapper(tile, ex):
return tile + 10
a = expr.map_with_location(expr.ones((5, 5)), mapper) + expr.ones(
(5, 5))
Assert.isinstance(a.optimized().op, expr.operator.local.ParakeetExpr)
def test_optimization_region_map(self):
def mapper(tile, ex):
return tile + 10
ex = array.extent.create((0, 0), (1, 5), (5, 5))
a = expr.region_map(expr.ones((5, 5)), ex, mapper) + expr.ones((5, 5))*10
for child in a.optimized().op.deps:
Assert.true(not isinstance(child, expr.local.LocalInput))
def test_reshape4(self): a = expr.arange((10000, )) b = expr.reshape(a, (10, 1000)) c = expr.reshape(b, (1000, 10)) d = expr.reshape(c, (20, 500)) e = expr.reshape(d, (500, 20)) f = expr.reshape(e, (1, 10000)) g = expr.arange((1, 10000)) Assert.all_eq(f.glom(), g.glom())
def test_slice_map2(self): x = expr.arange((10, 10, 10), dtype=np.int) nx = np.arange(10 * 10 * 10, dtype=np.int).reshape((10, 10, 10)) y = x[:, :, 0] z = expr.map(y, lambda tile: tile + 13) val = z.glom() Assert.all_eq(val.reshape(10, 10), nx[:, :, 0] + 13)
def test_min(self):
src = np.asarray([1, 1, 1, 2, 2, 5, 5, 10])
Assert.all_eq(
spartan.min(spartan.from_numpy(src)).glom(),
np.min(src))
src = np.arange(100).reshape(10, 10)
Assert.all_eq(
spartan.min(spartan.from_numpy(src), axis=1).glom(),
np.min(src, axis=1))
def test_transpose3(self):
t1 = expr.sparse_diagonal((107, 401)).evaluate()
t2 = expr.sparse_diagonal((401, 107)).evaluate()
a = expr.transpose(t1)
b = expr.transpose(t2)
Assert.all_eq(a.glom().todense(),
sp.eye(107, 401).transpose().todense())
Assert.all_eq(b.glom().todense(),
sp.eye(401, 107).transpose().todense())
def test_slice_sub(self):
a = expr.arange((TEST_SIZE,), dtype=np.int)
v = (a[1:] - a[:-1])
print optimize.optimize(v)
v = v.glom()
print v
na = np.arange(TEST_SIZE, dtype=np.int)
nv = na[1:] - na[:-1]
Assert.all_eq(v, nv)
def test_index(self):
a = expr.arange((TEST_SIZE, TEST_SIZE))
b = expr.ones((10,), dtype=np.int)
z = a[b]
val = expr.evaluate(z)
nx = np.arange(TEST_SIZE * TEST_SIZE).reshape(TEST_SIZE, TEST_SIZE)
ny = np.ones((10,), dtype=np.int)
Assert.all_eq(val.glom(), nx[ny])
def test_argmax_3d(self):
x = expr.arange((TEST_SIZE, TEST_SIZE, TEST_SIZE), dtype=np.int64)
nx = np.arange(TEST_SIZE * TEST_SIZE * TEST_SIZE,
dtype=np.int64).reshape(
(TEST_SIZE, TEST_SIZE, TEST_SIZE))
for axis in [None, 0, 1, 2]:
y = x.argmax(axis)
val = y.glom()
Assert.all_eq(val, nx.argmax(axis))
def test_update_sparse_to_sparse(self):
t = tile.from_shape(ARRAY_SIZE,
dtype=np.float32,
tile_type=tile.TYPE_SPARSE)
update = sp.lil_matrix(ARRAY_SIZE, dtype=np.float32)
for i in range(UPDATE_SHAPE[0]):
update[i, i] = 1
t.update(UPDATE_SUBSLICE, update, None)
Assert.eq(sp.issparse(t.data), True)
print t.data.todense()
def test_unravel():
for i in range(100):
shp = (20, 77)
ul = (random.randint(0, 19), random.randint(0, 76))
lr = (random.randint(ul[0] + 1, 20), random.randint(ul[1] + 1, 77))
a = extent.create(ul, lr, shp)
ravelled = a.ravelled_pos()
unravelled = extent.unravelled_pos(ravelled, a.array_shape)
Assert.eq(a.ul, unravelled)
def test_index(self):
a = expr.arange((TEST_SIZE, TEST_SIZE))
b = expr.ones((10, ), dtype=np.int)
z = a[b]
val = expr.evaluate(z)
nx = np.arange(TEST_SIZE * TEST_SIZE).reshape(TEST_SIZE, TEST_SIZE)
ny = np.ones((10, ), dtype=np.int)
Assert.all_eq(val.glom(), nx[ny])
def test_optimization_region_map(self):
def mapper(tile, ex):
return tile + 10
ex = array.extent.create((0, 0), (1, 5), (5, 5))
a = expr.region_map(expr.ones((5, 5)), ex, mapper) + expr.ones(
(5, 5)) * 10
for child in a.optimized().op.deps:
Assert.true(not isinstance(child, expr.operator.local.LocalInput))
def test_arange_stop(self):
# Arange with stop.
Assert.all_eq(spartan.arange(stop=10).glom(), np.arange(10))
# Arange with start, stop
Assert.all_eq(spartan.arange(-1, 10).glom(), np.arange(-1, 10))
Assert.all_eq(spartan.arange(1, 10).glom(), np.arange(1, 10))
# Arange with start, stop, step
Assert.all_eq(spartan.arange(-1, 19, 2).glom(), np.arange(-1, 19, 2))
Assert.all_eq(spartan.arange(1, 21, 2).glom(), np.arange(1, 21, 2))
def test_slices_from_np(self): npa = np.random.random(10000).reshape(100, 100) slices1 = (slice(0, 50), slice(0, 50)) slices2 = (slice(0, 50), slice(50, 100)) slices3 = (slice(50, 100), slice(0, 50)) slices4 = (slice(50, 100), slice(50, 100)) t1 = expr.randn(100, 100) t2 = expr.write(t1, slices1, npa, slices1) t3 = expr.write(t2, slices2, npa, slices2) t4 = expr.write(t3, slices3, npa, slices3) t5 = expr.write(t4, slices4, npa, slices4) Assert.all_eq(t5.glom(), npa)
def test_broadcast(self):
a = expr.ones((100, 1, 100, 100)).force()
b = expr.ones((10, 100, 1)).force()
a, b = broadcast.broadcast((a, b))
c = expr.add(a, b).force()
d = expr.sub(a, b).force()
n = np.ones((100, 10, 100, 100))
n1 = n + n
n2 = n - n
Assert.all_eq(n1, c.glom())
Assert.all_eq(n2, d.glom())
def test_diag(self):
import random
dim = random.randint(0, 99)
np_array = np.random.randn(dim, dim)
Assert.all_eq(
spartan.diag(spartan.from_numpy(np_array)).glom(),
np.diag(np_array))
np_array2 = np.random.randn(dim, dim)
Assert.all_eq(
spartan.diag(spartan.diag(spartan.from_numpy(np_array2))).glom(),
np.diag(np.diag(np_array2)))
def test_distarray(self):
A = spartan.arange(40000, dtype=np.int32).reshape(100, 400).evaluate()
nA = np.arange(40000).reshape(100, 400)
B = A.transpose().evaluate()
nB = nA.transpose()
C = B.T.evaluate()
nC = nB.T
D = (C / 100).evaluate()
nD = nC / 100
E = D.all()
nE = nD.all()
Assert.all_eq(E.glom(), nE)
def test_matmul(self):
x = expr.arange(XDIM, dtype=np.int).astype(np.float64)
y = expr.arange(YDIM, dtype=np.int).astype(np.float64)
z = expr.dot(x, y)
nx = np.arange(np.prod(XDIM),
dtype=np.int).reshape(XDIM).astype(np.float64)
ny = np.arange(np.prod(YDIM),
dtype=np.int).reshape(YDIM).astype(np.float64)
nz = np.dot(nx, ny)
Assert.all_eq(z.glom(), nz)
def test_logic(self):
# Arange with no parameters.
A = spartan.arange(40000, dtype=np.int32).reshape(100, 400)
nA = np.arange(40000).reshape(100, 400)
B = A.T
nB = nA.T
C = B / 1000
nC = nB / 1000
D = spartan.all(C)
nD = np.all(nC)
E = spartan.any(C)
nE = np.any(nC)
Assert.all_eq(D.glom(), nD)
Assert.all_eq(E.glom(), nE)
def test_assign_1d(self):
b = np.random.randn(100)
sp_b = from_numpy(b)
#a[:] = b[:] copy entire array
a = np.random.randn(100)
region_a = np.s_[0:100]
region_b = np.s_[0:100]
sp_a = assign(from_numpy(a), region_a, sp_b[region_b]).glom()
a[region_a] = b[region_b]
Assert.all_eq(sp_a, a)
# a[0] = b[1] copy one value
a = np.random.randn(100)
region_a = np.s_[0]
region_b = np.s_[1]
sp_a = assign(from_numpy(a), region_a, sp_b[region_b]).glom()
a[region_a] = b[region_b]
Assert.all_eq(sp_a, a)
# a[0:10] = b[20:30] copy range of values
a = np.random.randn(100)
region_a = np.s_[0:10]
region_b = np.s_[20:30]
sp_a = assign(from_numpy(a), region_a, sp_b[region_b]).glom()
a[region_a] = b[region_b]
Assert.all_eq(sp_a, a)
# a[30:60] = b[:30] copy range of values, not starting from 0.
a = np.random.randn(100)
region_a = np.s_[0:10]
region_b = np.s_[20:30]
sp_a = assign(from_numpy(a), region_a, sp_b[region_b]).glom()
a[region_a] = b[region_b]
Assert.all_eq(sp_a, a)
def test_newaxis(self):
na = np.arange(100).reshape(10, 10)
a = expr.from_numpy(na)
Assert.all_eq(na[np.newaxis, 2:7, 4:8].shape, a[expr.newaxis, 2:7,
4:8].shape)
Assert.all_eq(na[np.newaxis, 2:7, np.newaxis, 4:8].shape,
a[expr.newaxis, 2:7, expr.newaxis, 4:8].shape)
Assert.all_eq(
na[np.newaxis, 2:7, np.newaxis, 4:8, np.newaxis].shape,
a[expr.newaxis, 2:7, expr.newaxis, 4:8, expr.newaxis].shape)
#Extreme case
Assert.all_eq(
na[np.newaxis, np.newaxis, np.newaxis, np.newaxis, 2:7, np.newaxis,
np.newaxis, np.newaxis, 4:8, np.newaxis, np.newaxis,
np.newaxis].shape,
a[expr.newaxis, expr.newaxis, expr.newaxis, expr.newaxis, 2:7,
expr.newaxis, expr.newaxis, expr.newaxis, 4:8, expr.newaxis,
expr.newaxis, expr.newaxis].shape)
util.log_info(
'\na.shape: %s \nna.shape: %s',
a[expr.newaxis, 2:7, expr.newaxis, 4:8, expr.newaxis, expr.newaxis,
expr.newaxis].shape, na[np.newaxis, 2:7, np.newaxis, 4:8,
np.newaxis, np.newaxis,
np.newaxis].shape)
def test_ndimension(self):
for case in xrange(5):
dim = np.random.randint(low=2, high=6)
shape = np.random.randint(low=5, high=11, size=dim)
util.log_info('Test Case #%s: DIM(%s) shape%s', case + 1, dim, shape)
na = new_ndarray(shape)
a = expr.from_numpy(na)
for axis in xrange(dim):
Assert.all_eq(expr.sort(a, axis).glom(),
np.sort(na, axis))
Assert.all_eq(expr.argsort(a, axis).glom(),
np.argsort(na, axis))
def test_maximum(self):
# Test arrays of equal length.
np_a = np.random.randn(10, 10)
np_b = np.random.randn(10, 10)
sp_a = spartan.from_numpy(np_a)
sp_b = spartan.from_numpy(np_b)
Assert.all_eq(
spartan.maximum(sp_a, sp_b).glom(),
np.maximum(np_a, np_b))
# Test broadcasting.
Assert.all_eq(
spartan.maximum(sp_a, 0).glom(),
np.maximum(np_a, 0))
def test_2d_vec(self):
# Test with row > col
av = expr.arange((100, 77))
bv = expr.arange(stop=77)
na = np.arange(7700).reshape(100, 77)
nb = np.arange(77)
Assert.all_eq(expr.dot(av, bv).glom(), np.dot(na, nb))
# Test with col > row
av = expr.arange((77, 100))
bv = expr.arange(stop=100)
na = np.arange(7700).reshape(77, 100)
nb = np.arange(100)
Assert.all_eq(expr.dot(av, bv).glom(), np.dot(na, nb))
def test1(self):
a = expr.ones(ARRAY_SIZE)
b = expr.ones(ARRAY_SIZE)
c = expr.ones(ARRAY_SIZE)
x = a + b + c
y = x + x
z = y + y
z = expr.checkpoint(z, mode='disk')
z.evaluate()
failed_worker_id = 0
ctx = blob_ctx.get()
ctx.local_worker.mark_failed_worker(failed_worker_id)
res = z + z
Assert.all_eq(res.glom(), np.ones(ARRAY_SIZE)*24)
def test_ravelled_pos():
a = extent.create((2, 2), (7, 7), (10, 10))
for i in range(0, 10):
for j in range(0, 10):
assert extent.ravelled_pos((i, j), a.array_shape) == 10 * i + j
Assert.eq(a.to_global(0, axis=None), 22)
Assert.eq(a.to_global(10, axis=None), 42)
Assert.eq(a.to_global(11, axis=None), 43)
Assert.eq(a.to_global(20, axis=None), 62)
def test_del_dim(self):
na = np.arange(100).reshape(10, 10)
a = expr.from_numpy(na)
Assert.all_eq(na[2:7, 8], a[2:7, 8].glom())
Assert.all_eq(na[3:9, 4].shape, a[3:9, 4].shape)
Assert.all_eq(na[2:7, -1], a[2:7, -1].glom())
Assert.all_eq(na[-1, 3:9].shape, a[-1, 3:9].shape)
util.log_info('\na.shape: %s \nna.shape %s', a[3:9, 4].shape,
na[3:9, 4].shape)
def test_sparse_scan(self):
# np.cumsum does not support sparse matrices, so they must be converted
# to dense matrices first.
np_sparse = sparse.eye(*ARRAY_SIZE).todense()
sp_sparse = sparse_diagonal(ARRAY_SIZE, np.float32, tile_hint)
Assert.all_eq(
scan(sp_sparse,
reduce_fn=lambda x, **kw: x.sum(axis=kw['axis']),
scan_fn=lambda x, **kw: x.cumsum(axis=kw['axis']),
axis=None).glom(),
np.cumsum(np_sparse).reshape(ARRAY_SIZE))
for axis in range(len(ARRAY_SIZE)):
Assert.all_eq(
scan(sp_sparse,
reduce_fn=lambda x, **kw: x.sum(axis=kw['axis']),
scan_fn=lambda x, **kw: x.cumsum(axis=kw['axis']),
axis=axis).glom(),
np.cumsum(np_sparse, axis).reshape(ARRAY_SIZE))
def test_optimization_reduced(self):
na = np.random.rand(1000, 1000)
nb = np.random.rand(1000, 1000)
a = expr.from_numpy(na)
b = expr.from_numpy(nb)
c = a - b
d = a + c
f = c[200:900, 200:900]
g = d[200:900, 200:900]
h = f - g
i = f + h
j = h[100:500, 100:500]
k = i[100:500, 100:500]
l = expr.dot(j, k)
m = j + k
n = k - l
o = n - m
q = n + o
r = q - m
s = expr.sum(r)
nc = na - nb
nd = na + nc
nf = nc[200:900, 200:900]
ng = nd[200:900, 200:900]
nh = nf - ng
ni = nf + nh
nj = nh[100:500, 100:500]
nk = ni[100:500, 100:500]
nl = np.dot(nj, nk)
nm = nj + nk
nn = nk - nl
no = nn - nm
nq = nn + no
nr = nq - nm
ns = np.sum(nr)
# Our sum seems to reduce precision
Assert.all_eq(ns, s.optimized().glom(), tolerance=1e-6)