def test_reshape6(self):
a = expr.arange((12319, ))
b = expr.reshape(a, (127, 97))
c = expr.reshape(b, (97, 127))
d = expr.reshape(c, (1, 12319))
e = expr.arange((1, 12319))
Assert.all_eq(d.glom(), e.glom())
def test_sum_2d(self):
x = expr.arange((TEST_SIZE, TEST_SIZE), dtype=np.int)
nx = np.arange(TEST_SIZE * TEST_SIZE, dtype=np.int).reshape((TEST_SIZE, TEST_SIZE))
for axis in [None, 0, 1]:
y = x.sum(axis)
val = y.glom()
Assert.all_eq(val, nx.sum(axis))
def test_creation(self):
A = spartan.eye(100, 10)
nA = np.eye(100, 10)
B = spartan.identity(100)
nB = np.identity(100)
Assert.all_eq(A.glom(), nA)
Assert.all_eq(B.glom(), nB)
def test_reshape3(self): a = expr.arange((100, 100)) b = expr.reshape(a, (10000,)) c = expr.reshape(b, (10000, 1)) d = expr.reshape(c, (1, 10000)) e = expr.arange((1, 10000)) Assert.all_eq(d.glom(), e.glom())
def test_assign_array_like(self): a = np.zeros((20, 10)) b = np.ones((10, )) region = np.s_[10, ] sp_a = assign(from_numpy(a), region, b).glom() a[region] = b Assert.all_eq(sp_a, a)
def _test_optimization_ordered(self):
na = np.random.rand(1000, 1000)
nb = np.random.rand(1000, 1000)
a = expr.from_numpy(na)
b = expr.from_numpy(nb)
c = a - b
d = a + c
f = c[200:900, 200:900]
g = d[200:900, 200:900]
h = f - g
i = f + h
j = h[100:500, 100:500]
k = i[100:500, 100:500]
l = expr.dot(j, k)
m = j + k
n = k - l
o = n - m
q = o[100:200, 100:200]
nc = na - nb
nd = na + nc
nf = nc[200:900, 200:900]
ng = nd[200:900, 200:900]
nh = nf - ng
ni = nf + nh
nj = nh[100:500, 100:500]
nk = ni[100:500, 100:500]
nl = np.dot(nj, nk)
nm = nj + nk
nn = nk - nl
no = nn - nm
nq = no[100:200, 100:200]
Assert.all_eq(nq, q.optimized().glom(), tolerance=1e-10)
def test_argmax_2d(self):
for axis in [1]: #[None, 0, 1]:
x = expr.arange((TEST_SIZE, TEST_SIZE), dtype=np.int)
nx = np.arange(TEST_SIZE * TEST_SIZE, dtype=np.int).reshape((TEST_SIZE, TEST_SIZE))
y = x.argmax(axis=axis)
val = expr.glom(y)
Assert.all_eq(val, nx.argmax(axis=axis))
def test_transpose3(self): t1 = expr.sparse_diagonal((107, 401)).evaluate() t2 = expr.sparse_diagonal((401, 107)).evaluate() a = expr.transpose(t1) b = expr.transpose(t2) Assert.all_eq(a.glom().todense(), sp.eye(107, 401).transpose().todense()) Assert.all_eq(b.glom().todense(), sp.eye(401, 107).transpose().todense())
def test_creation(self):
A = spartan.eye(100, 10)
nA = np.eye(100, 10)
B = spartan.identity(100)
nB = np.identity(100)
Assert.all_eq(A.glom(), nA)
Assert.all_eq(B.glom(), nB)
def test_numpy_2d_vec(self):
av = expr.arange((77, 100))
bv = np.arange(100)
na = np.arange(7700).reshape(77, 100)
nb = np.arange(100)
Assert.all_eq(expr.dot(av, bv).glom(), np.dot(na, nb))
def test_reshape6(self): a = expr.arange((12319, )) b = expr.reshape(a, (127, 97)) c = expr.reshape(b, (97, 127)) d = expr.reshape(c, (1, 12319)) e = expr.arange((1, 12319)) Assert.all_eq(d.glom(), e.glom())
def test_numpy_vec_vec(self):
av = expr.arange(stop=100)
bv = np.arange(100)
na = np.arange(100)
nb = np.arange(100)
Assert.all_eq(expr.dot(av, bv).glom(), np.dot(na, nb))
def _(axis):
util.log_info('Testing sum over axis %s', axis)
a = expr.ones((TEST_SIZE, TEST_SIZE)) + expr.ones(
(TEST_SIZE, TEST_SIZE))
b = a.sum(axis=axis)
Assert.all_eq(b.glom(), 2 * np.ones(
(TEST_SIZE, TEST_SIZE)).sum(axis))
def test_reshape8(self): t1 = expr.sparse_diagonal((137, 113)) t2 = expr.sparse_diagonal((113, 137)) a = expr.reshape(t1, (113, 137)) b = expr.reshape(t2, (137, 113)) Assert.all_eq(a.glom().todense(), sp.eye(137, 113).tolil().reshape((113, 137)).todense()) Assert.all_eq(b.glom().todense(), sp.eye(113, 137).tolil().reshape((137, 113)).todense())
def test_fio_path(self):
self.create_path()
t1 = expr.randn(100, 100).evaluate()
expr.save(t1, "fiotest1", self.test_dir2, False)
expr.pickle(t1, "fiotest2", self.test_dir2, False)
Assert.all_eq(t1.glom(), expr.load("fiotest1", self.test_dir2, False).glom())
Assert.all_eq(t1.glom(), expr.unpickle("fiotest2", self.test_dir2, False).glom())
def test_reshape5(self): a = expr.arange((35511, )) b = expr.reshape(a, (133, 267)) c = expr.reshape(b, (267, 133)) d = expr.reshape(c, (1, 35511)) e = expr.arange((1, 35511)) Assert.all_eq(d.glom(), e.glom())
def test_reshape5(self):
a = expr.arange((35511, ))
b = expr.reshape(a, (133, 267))
c = expr.reshape(b, (267, 133))
d = expr.reshape(c, (1, 35511))
e = expr.arange((1, 35511))
Assert.all_eq(d.glom(), e.glom())
def test_slice_reduce(self): x = expr.arange((TEST_SIZE, TEST_SIZE, TEST_SIZE), dtype=np.int) nx = np.arange(TEST_SIZE * TEST_SIZE * TEST_SIZE, dtype=np.int).reshape((TEST_SIZE, TEST_SIZE, TEST_SIZE)) y = x[:, :, 0].sum() val = y.glom() Assert.all_eq(val, nx[:, :, 0].sum())
def test_min(self):
src = np.asarray([1, 1, 1, 2, 2, 5, 5, 10])
Assert.all_eq(spartan.min(spartan.from_numpy(src)).glom(), np.min(src))
src = np.arange(100).reshape(10, 10)
Assert.all_eq(
spartan.min(spartan.from_numpy(src), axis=1).glom(),
np.min(src, axis=1))
def test_assign_array_like(self):
a = np.zeros((20, 10))
b = np.ones((10, ))
region = np.s_[10, ]
sp_a = assign(from_numpy(a), region, b).glom()
a[region] = b
Assert.all_eq(sp_a, a)
def test_reshape3(self):
a = expr.arange((100, 100))
b = expr.reshape(a, (10000, ))
c = expr.reshape(b, (10000, 1))
d = expr.reshape(c, (1, 10000))
e = expr.arange((1, 10000))
Assert.all_eq(d.glom(), e.glom())
def test_slices_from_slices(self):
slices1 = (slice(0, 50), slice(0, 50))
slices2 = (slice(0, 50), slice(50, 100))
slices3 = (slice(50, 100), slice(0, 50))
slices4 = (slice(50, 100), slice(50, 100))
t1 = expr.randn(100, 100)
t2 = expr.randn(100, 100)
t3 = expr.write(t2, slices1, t1, slices1)
t4 = expr.write(t3, slices2, t1, slices2)
t5 = expr.write(t4, slices3, t1, slices3)
t6 = expr.write(t5, slices4, t1, slices4)
Assert.all_eq(t1.glom(), t6.glom())
dst = np.arange(0, 2500).reshape(50, 50)
t11 = expr.write(t1, slices1, dst, slices1)
t12 = expr.write(t11, slices2, dst, slices1)
t13 = expr.write(t12, slices3, dst, slices1)
t14 = expr.write(t13, slices4, dst, slices1)
tmp = expr.write(expr.randn(100, 100), slices4, dst, slices1)
t21 = expr.write(t2, slices1, tmp, slices4)
t22 = expr.write(t21, slices2, tmp, slices4)
t23 = expr.write(t22, slices3, tmp, slices4)
t24 = expr.write(t23, slices4, tmp, slices4)
Assert.all_eq(t14.glom(), t24.glom())
def _test_optimization_ordered(self):
na = np.random.rand(1000, 1000)
nb = np.random.rand(1000, 1000)
a = expr.from_numpy(na)
b = expr.from_numpy(nb)
c = a - b
d = a + c
f = c[200:900, 200:900]
g = d[200:900, 200:900]
h = f - g
i = f + h
j = h[100:500, 100:500]
k = i[100:500, 100:500]
l = expr.dot(j, k)
m = j + k
n = k - l
o = n - m
q = o[100:200, 100:200]
nc = na - nb
nd = na + nc
nf = nc[200:900, 200:900]
ng = nd[200:900, 200:900]
nh = nf - ng
ni = nf + nh
nj = nh[100:500, 100:500]
nk = ni[100:500, 100:500]
nl = np.dot(nj, nk)
nm = nj + nk
nn = nk - nl
no = nn - nm
nq = no[100:200, 100:200]
Assert.all_eq(nq, q.optimized().glom(), tolerance = 1e-10)
def test_numpy_2d_vec(self):
av = expr.arange((77, 100))
bv = np.arange(100)
na = np.arange(7700).reshape(77, 100)
nb = np.arange(100)
Assert.all_eq(expr.dot(av, bv).glom(),
np.dot(na, nb))
def test_argmax_2d(self):
for axis in [1]: #[None, 0, 1]:
x = expr.arange((TEST_SIZE, TEST_SIZE), dtype=np.int)
nx = np.arange(TEST_SIZE * TEST_SIZE, dtype=np.int).reshape(
(TEST_SIZE, TEST_SIZE))
y = x.argmax(axis=axis)
val = expr.glom(y)
Assert.all_eq(val, nx.argmax(axis=axis))
def _step():
yp = expr.dot(x, w)
Assert.all_eq(yp.shape, y.shape)
diff = x * (yp - y)
grad = expr.sum(diff, axis=0).glom().reshape((N_DIM, 1))
wprime = w - grad * 1e-6
wprime.evaluate()
def _step(): yp = expr.dot(x, w) Assert.all_eq(yp.shape, y.shape) diff = x * (yp - y) grad = expr.sum(diff, axis=0).glom().reshape((N_DIM, 1)) wprime = w - grad * 1e-6 expr.force(wprime)
def test_add_many(self):
a = expr.ones((TEST_SIZE, TEST_SIZE))
b = expr.ones((TEST_SIZE, TEST_SIZE))
add_many = (a + b + a + b + a + b + a + b + a + b)
#print add_many
#print add_many.dag()
Assert.all_eq(add_many.glom(), np.ones((TEST_SIZE, TEST_SIZE)) * 10)
def test_numpy_vec_2d(self):
av = expr.arange(stop = 100)
bv = np.arange(7700).reshape(100, 77)
na = np.arange(100)
nb = np.arange(7700).reshape(100, 77)
Assert.all_eq(expr.dot(av, bv).glom(),
np.dot(na, nb))
def test_slice_shuffle(self): x = expr.arange((TEST_SIZE, TEST_SIZE)) z = x[5:8, 5:8] z = expr.shuffle(z, add_one_extent) val = z.force() nx = np.arange(TEST_SIZE*TEST_SIZE).reshape(TEST_SIZE, TEST_SIZE) Assert.all_eq(val.glom(), nx[5:8, 5:8] + 1)
def test_slice_map(self): x = expr.arange((TEST_SIZE, TEST_SIZE)) z = x[5:8, 5:8] z = expr.map(z, add_one_tile) print z nx = np.arange(TEST_SIZE*TEST_SIZE).reshape(TEST_SIZE, TEST_SIZE) Assert.all_eq(z.glom(), nx[5:8, 5:8] + 1)
def test_sum_2d(self):
x = expr.arange((TEST_SIZE, TEST_SIZE), dtype=np.int)
nx = np.arange(TEST_SIZE * TEST_SIZE, dtype=np.int).reshape(
(TEST_SIZE, TEST_SIZE))
for axis in [None, 0, 1]:
y = x.sum(axis)
val = y.glom()
Assert.all_eq(val, nx.sum(axis))
def test_numpy_vec_vec(self):
av = expr.arange(stop=100)
bv = np.arange(100)
na = np.arange(100)
nb = np.arange(100)
Assert.all_eq(expr.dot(av, bv).glom(),
np.dot(na, nb))
def test_argmax_3d(self):
x = expr.arange((TEST_SIZE, TEST_SIZE, TEST_SIZE), dtype=np.int64)
nx = np.arange(TEST_SIZE * TEST_SIZE * TEST_SIZE, dtype=np.int64).reshape((TEST_SIZE, TEST_SIZE, TEST_SIZE))
for axis in [None, 0, 1, 2]:
y = x.argmax(axis)
val = y.glom()
Assert.all_eq(val, nx.argmax(axis))
def test_diag(self):
import random
dim = random.randint(0, 99)
np_array = np.random.randn(dim, dim)
Assert.all_eq(spartan.diag(spartan.from_numpy(np_array)).glom(), np.diag(np_array))
np_array2 = np.random.randn(dim, dim)
Assert.all_eq(spartan.diag(spartan.diag(spartan.from_numpy(np_array2))).glom(), np.diag(np.diag(np_array2)))
def test_add_many(self):
a = expr.ones((TEST_SIZE, TEST_SIZE))
b = expr.ones((TEST_SIZE, TEST_SIZE))
add_many = (a + b + a + b + a + b + a + b + a + b)
#print add_many
#print add_many.dag()
Assert.all_eq(add_many.glom(),
np.ones((TEST_SIZE, TEST_SIZE)) * 10)
def test_min(self):
src = np.asarray([1, 1, 1, 2, 2, 5, 5, 10])
Assert.all_eq(
spartan.min(spartan.from_numpy(src)).glom(),
np.min(src))
src = np.arange(100).reshape(10, 10)
Assert.all_eq(
spartan.min(spartan.from_numpy(src), axis=1).glom(),
np.min(src, axis=1))
def test_transpose3(self):
t1 = expr.sparse_diagonal((107, 401)).evaluate()
t2 = expr.sparse_diagonal((401, 107)).evaluate()
a = expr.transpose(t1)
b = expr.transpose(t2)
Assert.all_eq(a.glom().todense(),
sp.eye(107, 401).transpose().todense())
Assert.all_eq(b.glom().todense(),
sp.eye(401, 107).transpose().todense())
def test_reshape4(self):
a = expr.arange((10000, ))
b = expr.reshape(a, (10, 1000))
c = expr.reshape(b, (1000, 10))
d = expr.reshape(c, (20, 500))
e = expr.reshape(d, (500, 20))
f = expr.reshape(e, (1, 10000))
g = expr.arange((1, 10000))
Assert.all_eq(f.glom(), g.glom())
def test_slice_map2(self): x = expr.arange((10, 10, 10), dtype=np.int) nx = np.arange(10 * 10 * 10, dtype=np.int).reshape((10, 10, 10)) y = x[:, :, 0] z = expr.map(y, lambda tile: tile + 13) val = z.glom() Assert.all_eq(val.reshape(10, 10), nx[:, :, 0] + 13)
def test_reshape8(self):
t1 = expr.sparse_diagonal((137, 113))
t2 = expr.sparse_diagonal((113, 137))
a = expr.reshape(t1, (113, 137))
b = expr.reshape(t2, (137, 113))
Assert.all_eq(a.glom().todense(),
sp.eye(137, 113).tolil().reshape((113, 137)).todense())
Assert.all_eq(b.glom().todense(),
sp.eye(113, 137).tolil().reshape((137, 113)).todense())
def test_reshape4(self): a = expr.arange((10000, )) b = expr.reshape(a, (10, 1000)) c = expr.reshape(b, (1000, 10)) d = expr.reshape(c, (20, 500)) e = expr.reshape(d, (500, 20)) f = expr.reshape(e, (1, 10000)) g = expr.arange((1, 10000)) Assert.all_eq(f.glom(), g.glom())
def test_argmax_3d(self):
x = expr.arange((TEST_SIZE, TEST_SIZE, TEST_SIZE), dtype=np.int64)
nx = np.arange(TEST_SIZE * TEST_SIZE * TEST_SIZE,
dtype=np.int64).reshape(
(TEST_SIZE, TEST_SIZE, TEST_SIZE))
for axis in [None, 0, 1, 2]:
y = x.argmax(axis)
val = y.glom()
Assert.all_eq(val, nx.argmax(axis))
def test_matmul(self):
x = expr.arange(XDIM, dtype=np.int).astype(np.float64)
y = expr.arange(YDIM, dtype=np.int).astype(np.float64)
z = expr.dot(x, y)
nx = np.arange(np.prod(XDIM), dtype=np.int).reshape(XDIM).astype(np.float64)
ny = np.arange(np.prod(YDIM), dtype=np.int).reshape(YDIM).astype(np.float64)
nz = np.dot(nx, ny)
Assert.all_eq(z.glom(), nz)
def test_index(self):
a = expr.arange((TEST_SIZE, TEST_SIZE))
b = expr.ones((10,), dtype=np.int)
z = a[b]
val = expr.evaluate(z)
nx = np.arange(TEST_SIZE * TEST_SIZE).reshape(TEST_SIZE, TEST_SIZE)
ny = np.ones((10,), dtype=np.int)
Assert.all_eq(val.glom(), nx[ny])
def test_slice_sub(self):
a = expr.arange((TEST_SIZE,), dtype=np.int)
v = (a[1:] - a[:-1])
print optimize.optimize(v)
v = v.glom()
print v
na = np.arange(TEST_SIZE, dtype=np.int)
nv = na[1:] - na[:-1]
Assert.all_eq(v, nv)
def test_index(self):
a = expr.arange((TEST_SIZE, TEST_SIZE))
b = expr.ones((10, ), dtype=np.int)
z = a[b]
val = expr.evaluate(z)
nx = np.arange(TEST_SIZE * TEST_SIZE).reshape(TEST_SIZE, TEST_SIZE)
ny = np.ones((10, ), dtype=np.int)
Assert.all_eq(val.glom(), nx[ny])
def test_arange_stop(self):
# Arange with stop.
Assert.all_eq(spartan.arange(stop=10).glom(), np.arange(10))
# Arange with start, stop
Assert.all_eq(spartan.arange(-1, 10).glom(), np.arange(-1, 10))
Assert.all_eq(spartan.arange(1, 10).glom(), np.arange(1, 10))
# Arange with start, stop, step
Assert.all_eq(spartan.arange(-1, 19, 2).glom(), np.arange(-1, 19, 2))
Assert.all_eq(spartan.arange(1, 21, 2).glom(), np.arange(1, 21, 2))
def test_arange_stop(self):
# Arange with stop.
Assert.all_eq(spartan.arange(stop=10).glom(), np.arange(10))
# Arange with start, stop
Assert.all_eq(spartan.arange(None, -1, 10).glom(), np.arange(-1, 10))
Assert.all_eq(spartan.arange(None, 1, 10).glom(), np.arange(1, 10))
# Arange with start, stop, step
Assert.all_eq(spartan.arange(None, -1, 19, 2).glom(), np.arange(-1, 19, 2))
Assert.all_eq(spartan.arange(None, 1, 21, 2).glom(), np.arange(1, 21, 2))
def test_matmul(self):
x = expr.arange(XDIM, dtype=np.int).astype(np.float64)
y = expr.arange(YDIM, dtype=np.int).astype(np.float64)
z = expr.dot(x, y)
nx = np.arange(np.prod(XDIM),
dtype=np.int).reshape(XDIM).astype(np.float64)
ny = np.arange(np.prod(YDIM),
dtype=np.int).reshape(YDIM).astype(np.float64)
nz = np.dot(nx, ny)
Assert.all_eq(z.glom(), nz)
def test_slices_from_np(self): npa = np.random.random(10000).reshape(100, 100) slices1 = (slice(0, 50), slice(0, 50)) slices2 = (slice(0, 50), slice(50, 100)) slices3 = (slice(50, 100), slice(0, 50)) slices4 = (slice(50, 100), slice(50, 100)) t1 = expr.randn(100, 100) t2 = expr.write(t1, slices1, npa, slices1) t3 = expr.write(t2, slices2, npa, slices2) t4 = expr.write(t3, slices3, npa, slices3) t5 = expr.write(t4, slices4, npa, slices4) Assert.all_eq(t5.glom(), npa)
def test_broadcast(self):
a = expr.ones((100, 1, 100, 100)).force()
b = expr.ones((10, 100, 1)).force()
a, b = broadcast.broadcast((a, b))
c = expr.add(a, b).force()
d = expr.sub(a, b).force()
n = np.ones((100, 10, 100, 100))
n1 = n + n
n2 = n - n
Assert.all_eq(n1, c.glom())
Assert.all_eq(n2, d.glom())
def test_broadcast(self):
a = expr.ones((100, 1, 100, 100)).force()
b = expr.ones((10, 100, 1)).force()
a, b = broadcast.broadcast((a, b))
c = expr.add(a, b).force()
d = expr.sub(a, b).force()
n = np.ones((100, 10, 100, 100))
n1 = n + n
n2 = n - n
Assert.all_eq(n1, c.glom())
Assert.all_eq(n2, d.glom())
